Merge pull request #1 from Ostorlab/bootstrap

Bootstrap implementation of the WhoisIP agent.

Dockerfile

@@ -12,4 +12,4 @@ ENV PYTHONPATH=/app
 COPY agent /app/agent
 COPY ostorlab.yaml /app/agent/ostorlab.yaml
 WORKDIR /app
-CMD ["python3", "/app/agent/template_agent.py"]
+CMD ["python3", "/app/agent/whois_ip_agent.py"]

README.md

@@ -1,13 +1,85 @@
-# Template to create agents.
+<h1 align="center">Agent Whois IP</h1>
 
-## How to :
-* Create a repository from base_agent template.
-* git remote add template https://github.com/Ostorlab/template_agent.git
+<p align="center">
+<img src="https://img.shields.io/badge/License-Apache_2.0-brightgreen.svg">
+<img src="https://img.shields.io/github/languages/top/ostorlab/agent_whois_ip">
+<img src="https://img.shields.io/github/stars/ostorlab/agent_whois_ip">
+<img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg">
+</p>
 
+_Agent responsible for retrieving WHOIS information of an IP such as it's AS Number, contact information, registrar, and
+address._
 
-**In case new changes are made to the template**
+---
 
+<p align="center">
+<img src="https://github.com/Ostorlab/agent_whois_ip/blob/main/images/logo.png" alt="agent-whois-ip" />
+</p>
 
-* git fetch template
-* git checkout [branch-to-merge-to]
-* git merge template/main
+This repository is an implementation of [Ostorlab Agent](https://pypi.org/project/ostorlab/)
+for [ipwhois](https://pypi.org/project/ipwhois/).
+
+## Getting Started
+
+To perform your first scan, simply run the following command.
+
+```shell
+ostorlab scan run --install --agent agent/ostorlab/whois_ip ip 8.8.8.8
+```
+
+This command will download and install `agent/ostorlab/whois_ip`.
+For more information, please refer to
+the [Ostorlab Documentation](https://github.com/Ostorlab/ostorlab/blob/main/README.md)
+
+## Usage
+
+Agent Whois can be installed directly from the ostorlab agent store or built from this repository.
+
+### Install directly from ostorlab agent store
+
+ ```shell
+ ostorlab agent install agent/ostorlab/whois_ip
+ ```
+
+You can then run the agent with the following command:
+
+```shell
+ostorlab scan run --agent agent/ostorlab/whois_ip domain-name tesla.com
+```
+
+### Build directly from the repository
+
+1. To build the whois_ip agent you need to have [ostorlab](https://pypi.org/project/ostorlab/) installed in your
+   machine. if you have already installed ostorlab, you can skip this step.
+
+```shell
+pip3 install ostorlab
+```
+
+2. Clone this repository.
+
+```shell
+git clone https://github.com/Ostorlab/agent_whois_ip.git && cd agent_whois_ip
+```
+
+3. Build the agent image using ostorlab cli.
+
+ ```shell
+ ostortlab agent build --file=ostorlab.yaml
+ ```
+
+You can pass the optional flag `--organization` to specify your organisation. The organization is empty by default.
+
+1. Run the agent using on of the following commands:
+    * If you did not specify an organization when building the image:
+     ```shell
+     ostorlab scan run --agent agent//whois_ip ip 8.8.8.8
+     ```
+    * If you specified an organization when building the image:
+     ```shell
+     ostorlab scan run --agent agent/[ORGANIZATION]/whois_ip ip 8.8.8.8
+     ```
+
+## License
+
+[Apache-2](./LICENSE)

agent/whois_ip_agent.py

@@ -0,0 +1,70 @@
+"""WhoisIP agent implementation"""
+import logging
+from rich import logging as rich_logging
+import ipwhois
+from ostorlab.agent import agent
+from ostorlab.agent import message as m
+
+logging.basicConfig(
+    format='%(message)s',
+    datefmt='[%X]',
+    level='INFO',
+    force=True,
+    handlers=[rich_logging.RichHandler(rich_tracebacks=True)]
+)
+logger = logging.getLogger(__name__)
+
+
+class WhoisIPAgent(agent.Agent):
+    """WhoisIP agent that collect IP registry and AS information using the RDAP protocol."""
+
+    def process(self, message: m.Message) -> None:
+        logger.info('processing IP %s', message.data.get('host'))
+        record = ipwhois.IPWhois(message.data.get('host')).lookup_rdap()
+        logger.debug('record\n%s', record)
+
+        whois_message = {
+            'host': message.data.get('host'),
+            'mask': message.data.get('mask'),
+            'version': message.data.get('version'),
+            'asn_registry': record.get('asn_registry'),
+            'asn_number': record.get('asn_number'),
+            'asn_country_code': record.get('asn_country_code'),
+            'asn_date': record.get('asn_date'),
+            'asn_description': record.get('asn_description'),
+            'network': {
+                'cidr': record.get('network', {}).get('cidr'),
+                'name': record.get('network', {}).get('name'),
+                'handle': record.get('network', {}).get('handle'),
+                'parent_handle': record.get('network', {}).get('parent_handle'),
+            },
+            'entities': [
+                {
+                    'name': e.get('handle'),
+                    'contact': {
+                        'name': e.get('contact', {}).get('name'),
+                        'kind': e.get('contact', {}).get('kind'),
+                        'address': self._get_entity_address(e),
+                    }
+                } for e in record.get('objects', {}).values()
+            ],
+
+        }
+
+        if message.data.get('version') == 4:
+            self.emit('v3.asset.ip.v4.whois', whois_message)
+        elif message.data.get('version') == 6:
+            self.emit('v3.asset.ip.v6.whois', whois_message)
+        else:
+            logger.error('unsupported version %s', message.data.get('version'))
+
+    def _get_entity_address(self, e):
+        addresses = e.get('contact', {}).get('address', [])
+        if addresses is None:
+            return None
+        return ' '.join(a.get('value') for a in addresses)
+
+
+if __name__ == '__main__':
+    logger.info('starting agent ...')
+    WhoisIPAgent.main()

ostorlab.yaml

@@ -1,9 +1,80 @@
 kind: Agent
-name: template_agent # Agent name, must be unique by organisation to be published on the store.
-version: 0.0.0
-description: Agent description. # Support for Markdown format.
-in_selectors: # List of input selectors.
-  - v3.healthcheck.ping
-out_selectors: [] # List of output selectors.
-docker_file_path : Dockerfile # Dockerfile path for automated release build.
-docker_build_root : . # Docker build dir for automated release build.
+name: whois_ip
+version: 0.0.1
+image: images/cover.png
+description: |
+  This repository is an implementation of [Ostorlab Agent](https://pypi.org/project/ostorlab/)
+  for [ipwhois](https://pypi.org/project/ipwhois/).
+
+  ## Getting Started
+
+  To perform your first scan, simply run the following command.
+
+  ```shell
+  ostorlab scan run --install --agent agent/ostorlab/whois_ip ip 8.8.8.8
+  ```
+
+  This command will download and install `agent/ostorlab/whois_ip`.
+  For more information, please refer to
+  the [Ostorlab Documentation](https://github.com/Ostorlab/ostorlab/blob/main/README.md)
+
+  ## Usage
+
+  Agent Whois can be installed directly from the ostorlab agent store or built from this repository.
+
+  ### Install directly from ostorlab agent store
+
+   ```shell
+   ostorlab agent install agent/ostorlab/whois_ip
+   ```
+
+  You can then run the agent with the following command:
+
+  ```shell
+  ostorlab scan run --agent agent/ostorlab/whois_ip domain-name tesla.com
+  ```
+
+  ### Build directly from the repository
+
+  1. To build the whois_ip agent you need to have [ostorlab](https://pypi.org/project/ostorlab/) installed in your
+     machine. if you have already installed ostorlab, you can skip this step.
+
+  ```shell
+  pip3 install ostorlab
+  ```
+
+  2. Clone this repository.
+
+  ```shell
+  git clone https://github.com/Ostorlab/agent_whois_ip.git && cd agent_whois_ip
+  ```
+
+  3. Build the agent image using ostorlab cli.
+
+   ```shell
+   ostortlab agent build --file=ostorlab.yaml
+   ```
+
+  You can pass the optional flag `--organization` to specify your organisation. The organization is empty by default.
+
+  1. Run the agent using on of the following commands:
+      * If you did not specify an organization when building the image:
+       ```shell
+       ostorlab scan run --agent agent//whois_ip domain-name tesla.com
+       ```
+      * If you specified an organization when building the image:
+       ```shell
+       ostorlab scan run --agent agent/[ORGANIZATION]/whois_ip ip 8.8.8.8
+       ```
+
+  ## License
+
+  [Apache-2](./LICENSE)
+in_selectors:
+  - v3.asset.ip.v4
+  - v3.asset.ip.v6
+out_selectors:
+  - v3.asset.ip.v4.whois
+  - v3.asset.ip.v6.whois
+docker_file_path : Dockerfile
+docker_build_root : .

requirement.txt

@@ -1,2 +1,3 @@
 ostorlab[agent]
 rich
+ipwhois

tests/conftest.py

@@ -1,10 +1,48 @@
-"""
-    Dummy conftest.py for template_agent.
+"""Pytest fixture for the WhoisIP agent."""
+import pathlib
+import random
 
-    If you don't know what this is for, just leave it empty.
-    Read more about conftest.py under:
-    - https://docs.pytest.org/en/stable/fixture.html
-    - https://docs.pytest.org/en/stable/writing_plugins.html
-"""
+import pytest
+from ostorlab.agent import definitions as agent_definitions
+from ostorlab.agent import message
+from ostorlab.runtimes import definitions as runtime_definitions
 
-# import pytest
+from agent import whois_ip_agent
+
+
+@pytest.fixture
+def scan_message_ipv4():
+    """Creates a dummy message of IPv4 asset.
+    """
+    selector = 'v3.asset.ip.v4'
+    msg_data = {
+        'host': '8.8.8.8',
+        'version': 4
+    }
+    return message.Message.from_data(selector, data=msg_data)
+
+
+@pytest.fixture
+def scan_message_ipv6():
+    """Creates a dummy message of IPv6 asset.
+    """
+    selector = 'v3.asset.ip.v6'
+    msg_data = {
+        'host': '2a00:1450:4006:80e::200e',
+        'version': 6
+    }
+    return message.Message.from_data(selector, data=msg_data)
+
+
+@pytest.fixture
+def test_agent():
+    with (pathlib.Path(__file__).parent.parent / 'ostorlab.yaml').open() as yaml_o:
+        definition = agent_definitions.AgentDefinition.from_yaml(yaml_o)
+        settings = runtime_definitions.AgentSettings(
+            key='agent/ostorlab/whois_ip',
+            bus_url='NA',
+            bus_exchange_topic='NA',
+            redis_url='redis://redis',
+            args=[],
+            healthcheck_port=random.randint(4000, 5000))
+        return whois_ip_agent.WhoisIPAgent(definition, settings)

tests/test-requirement.txt

@@ -1,4 +1,5 @@
 pylint
 pylint-quotes
 pytest
-pytest-cov
+pytest-cov
+pytest-mock