From 3e2d70224ef2adeb4884df231d8eb10d74ce7dc9 Mon Sep 17 00:00:00 2001 From: pyzcool Date: Wed, 1 Feb 2023 16:08:51 +0100 Subject: [PATCH 1/9] Fix whois crash: 'str' object has no attribute extend --- agent/result_parser.py | 6 ++++-- tests/whois_domain_agent_test.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/agent/result_parser.py b/agent/result_parser.py index f7472d2..8309a28 100644 --- a/agent/result_parser.py +++ b/agent/result_parser.py @@ -42,13 +42,15 @@ def parse_results(results: whois.parser.WhoisCom) -> Iterator[Dict[str, Any]]: "expiration_date": get_isoformat( scan_output_dict.get("expiration_date", []) ), - "name": name, + "name": _format_str(name) if name is not None else name, "emails": get_list_from_string(scan_output_dict.get("emails", "")), "status": get_list_from_string(scan_output_dict.get("status", "")), "name_servers": get_list_from_string( scan_output_dict.get("name_servers", "") ), - "contact_name": contact_name, + "contact_name": _format_str(contact_name) + if contact_name is not None + else contact_name, "dnssec": get_list_from_string(scan_output_dict.get("dnssec", "")), } for field in OPTIONAL_FIELDS: diff --git a/tests/whois_domain_agent_test.py b/tests/whois_domain_agent_test.py index a2ebd77..df1b633 100644 --- a/tests/whois_domain_agent_test.py +++ b/tests/whois_domain_agent_test.py @@ -26,7 +26,7 @@ ], "emails": ["abuse@godaddy.com"], "dnssec": "unsigned", - "name": "REDACTED FOR PRIVACY", + "name": ["Catherine Shapiro", "Ivan SLY"], "org": "Contact Privacy Inc. Customer 0139267634", "address": "REDACTED FOR PRIVACY", "city": "REDACTED FOR PRIVACY", @@ -54,7 +54,7 @@ ], "emails": ["abuse@godaddy.com"], "dnssec": "unsigned", - "name": "REDACTED FOR PRIVACY", + "name": ["Catherine Shapiro", "Ivan SLY"], "org": "Contact Privacy Inc. Customer 0139267634", "address": "REDACTED FOR PRIVACY", "city": "REDACTED FOR PRIVACY", @@ -82,7 +82,7 @@ ], "emails": ["compliance@tucows.com", "easydns@myprivacy.ca"], "dnssec": "unsigned", - "name": "REDACTED FOR PRIVACY", + "name": ["Catherine Shapiro", "Ivan SLY"], "org": "Contact Privacy Inc. Customer 0139267634", "address": "REDACTED FOR PRIVACY", "city": "REDACTED FOR PRIVACY", From 5bb2ccec71489f6eb3e692f2aaadf87adc58d256 Mon Sep 17 00:00:00 2001 From: pyzcool Date: Wed, 1 Feb 2023 16:24:58 +0100 Subject: [PATCH 2/9] Fix comments. --- agent/result_parser.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/agent/result_parser.py b/agent/result_parser.py index 8309a28..05ddc89 100644 --- a/agent/result_parser.py +++ b/agent/result_parser.py @@ -42,15 +42,14 @@ def parse_results(results: whois.parser.WhoisCom) -> Iterator[Dict[str, Any]]: "expiration_date": get_isoformat( scan_output_dict.get("expiration_date", []) ), - "name": _format_str(name) if name is not None else name, + "name": name, "emails": get_list_from_string(scan_output_dict.get("emails", "")), "status": get_list_from_string(scan_output_dict.get("status", "")), "name_servers": get_list_from_string( scan_output_dict.get("name_servers", "") ), - "contact_name": _format_str(contact_name) - if contact_name is not None - else contact_name, + # TODO(ticket: os-3017): change the proto of v3.asset.domain_name.whois to send contact names. + "contact_name": contact_name[0] if isinstance(contact_name, list) else contact_name, "dnssec": get_list_from_string(scan_output_dict.get("dnssec", "")), } for field in OPTIONAL_FIELDS: @@ -61,7 +60,7 @@ def parse_results(results: whois.parser.WhoisCom) -> Iterator[Dict[str, Any]]: def get_isoformat( - date_name: Union[datetime.datetime, List[datetime.datetime]] + date_name: Union[datetime.datetime, List[datetime.datetime]] ) -> List[str]: """Converts dates to ISO fomat From 6c609ebf363c6d7f3d4c53e083ea2f608f160ea1 Mon Sep 17 00:00:00 2001 From: pyzcool Date: Wed, 1 Feb 2023 16:26:56 +0100 Subject: [PATCH 3/9] Apply black. --- agent/result_parser.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/agent/result_parser.py b/agent/result_parser.py index 05ddc89..718f9df 100644 --- a/agent/result_parser.py +++ b/agent/result_parser.py @@ -49,7 +49,9 @@ def parse_results(results: whois.parser.WhoisCom) -> Iterator[Dict[str, Any]]: scan_output_dict.get("name_servers", "") ), # TODO(ticket: os-3017): change the proto of v3.asset.domain_name.whois to send contact names. - "contact_name": contact_name[0] if isinstance(contact_name, list) else contact_name, + "contact_name": contact_name[0] + if isinstance(contact_name, list) + else contact_name, "dnssec": get_list_from_string(scan_output_dict.get("dnssec", "")), } for field in OPTIONAL_FIELDS: @@ -60,7 +62,7 @@ def parse_results(results: whois.parser.WhoisCom) -> Iterator[Dict[str, Any]]: def get_isoformat( - date_name: Union[datetime.datetime, List[datetime.datetime]] + date_name: Union[datetime.datetime, List[datetime.datetime]] ) -> List[str]: """Converts dates to ISO fomat From 4b1539517c373b43f862a7c52c70d1d768b99dea Mon Sep 17 00:00:00 2001 From: pyzcool Date: Thu, 2 Feb 2023 11:26:30 +0100 Subject: [PATCH 4/9] Add unit test, fix bug: whois send one of two different keys email or emails. --- agent/result_parser.py | 6 +++- tests/whois_domain_agent_test.py | 56 ++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/agent/result_parser.py b/agent/result_parser.py index 718f9df..f85ba22 100644 --- a/agent/result_parser.py +++ b/agent/result_parser.py @@ -43,7 +43,11 @@ def parse_results(results: whois.parser.WhoisCom) -> Iterator[Dict[str, Any]]: scan_output_dict.get("expiration_date", []) ), "name": name, - "emails": get_list_from_string(scan_output_dict.get("emails", "")), + "emails": get_list_from_string( + scan_output_dict.get("email", "") + if scan_output_dict.get("email", "") is not "" + else scan_output_dict.get("emails", "") + ), "status": get_list_from_string(scan_output_dict.get("status", "")), "name_servers": get_list_from_string( scan_output_dict.get("name_servers", "") diff --git a/tests/whois_domain_agent_test.py b/tests/whois_domain_agent_test.py index df1b633..a72efa6 100644 --- a/tests/whois_domain_agent_test.py +++ b/tests/whois_domain_agent_test.py @@ -24,7 +24,7 @@ "clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited", "clientTransferProhibited https://icann.org/epp#clientTransferProhibited", ], - "emails": ["abuse@godaddy.com"], + "email": ["abuse@godaddy.com"], "dnssec": "unsigned", "name": ["Catherine Shapiro", "Ivan SLY"], "org": "Contact Privacy Inc. Customer 0139267634", @@ -52,7 +52,7 @@ "clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited", "clientTransferProhibited https://icann.org/epp#clientTransferProhibited", ], - "emails": ["abuse@godaddy.com"], + "email": ["abuse@godaddy.com"], "dnssec": "unsigned", "name": ["Catherine Shapiro", "Ivan SLY"], "org": "Contact Privacy Inc. Customer 0139267634", @@ -80,7 +80,7 @@ "clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited", "clientTransferProhibited https://icann.org/epp#clientTransferProhibited", ], - "emails": ["compliance@tucows.com", "easydns@myprivacy.ca"], + "email": ["compliance@tucows.com", "easydns@myprivacy.ca"], "dnssec": "unsigned", "name": ["Catherine Shapiro", "Ivan SLY"], "org": "Contact Privacy Inc. Customer 0139267634", @@ -91,6 +91,25 @@ "country": "CA", } +SCAN_OUTPUT_MULTIPLE_CONTACT_NAMES = { + "domain_name": "marksandspencer.at", + "registrar": "Key-Systems GmbH ( https://nic.at/registrar/404 )", + "name": ["Catherine Shapiro", "Ivan SLY"], + "org": ["Marks And Spencer P.l.c.", "IP TWINS S.A.S."], + "address": ["Waterside House", "35 North Wharf Road", "78 rue de Turbigo"], + "registrant_postal_code": ["W2 1NW", "75003"], + "city": ["London", "PARIS"], + "country": ["United Kingdom of Great Britain and Northern Ireland (the)", "France"], + "phone": ["+442087186494", "+33142789312"], + "fax": "+440207487267", + "updated_date": [ + datetime.datetime(2021, 6, 23, 10, 10, 57), + datetime.datetime(2021, 6, 23, 10, 7, 2), + datetime.datetime(2023, 1, 4, 19, 30, 24), + ], + "email": ["externaldnssupport@marks-and-spencer.com", "ivan.sly@iptwins.com"], +} + def testAgentWhois_whenDomainNameAsset_emitsMessages( scan_message: message.Message, @@ -115,6 +134,37 @@ def testAgentWhois_whenDomainNameAsset_emitsMessages( assert agent_mock[0].data["emails"] == ["abuse@godaddy.com"] +def testAgentWhois_whenMultipleContactNames_emitsMessages( + scan_message: message.Message, + test_agent: whois_domain_agent.AgentWhoisDomain, + agent_persist_mock: Any, + mocker: plugin.MockerFixture, + agent_mock: List[message.Message], +) -> None: + """Tests running the agent and emitting vulnerabilities.""" + del agent_persist_mock + + mock_whois = mocker.patch( + "whois.whois", return_value=SCAN_OUTPUT_MULTIPLE_CONTACT_NAMES + ) + test_agent.start() + test_agent.process(scan_message) + mock_whois.assert_called_once() + assert len(agent_mock) > 0 + assert agent_mock[0].selector == "v3.asset.domain_name.whois" + assert agent_mock[0].data["name"] == "marksandspencer.at" + assert agent_mock[0].data["updated_date"] == [ + "2021-06-23T10:10:57", + "2021-06-23T10:07:02", + "2023-01-04T19:30:24", + ] + assert agent_mock[0].data["emails"] == [ + "externaldnssupport@marks-and-spencer.com", + "ivan.sly@iptwins.com", + ] + assert agent_mock[0].data["contact_name"] == "Catherine Shapiro" + + def testAgentWhois_whenDomainNameISEmpty_NotEmitsMessages( scan_message: message.Message, test_agent: whois_domain_agent.AgentWhoisDomain, From 607b2bd31c68afbeb078f1198051173b845e6b4a Mon Sep 17 00:00:00 2001 From: pyzcool Date: Thu, 2 Feb 2023 14:25:41 +0100 Subject: [PATCH 5/9] Fix linting --- agent/result_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/result_parser.py b/agent/result_parser.py index f85ba22..956414b 100644 --- a/agent/result_parser.py +++ b/agent/result_parser.py @@ -45,7 +45,7 @@ def parse_results(results: whois.parser.WhoisCom) -> Iterator[Dict[str, Any]]: "name": name, "emails": get_list_from_string( scan_output_dict.get("email", "") - if scan_output_dict.get("email", "") is not "" + if scan_output_dict.get("email", "") != "" else scan_output_dict.get("emails", "") ), "status": get_list_from_string(scan_output_dict.get("status", "")), From 1a4dbca768441a35ae276734fa4664072ea5c9d0 Mon Sep 17 00:00:00 2001 From: pyzcool Date: Thu, 2 Feb 2023 18:07:57 +0100 Subject: [PATCH 6/9] Resolve conflict. --- tests/whois_domain_agent_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/whois_domain_agent_test.py b/tests/whois_domain_agent_test.py index 79cfac7..6fcacb3 100644 --- a/tests/whois_domain_agent_test.py +++ b/tests/whois_domain_agent_test.py @@ -133,6 +133,7 @@ def testAgentWhois_whenDomainNameAsset_emitsMessages( assert agent_mock[0].data["expiration_date"] == ["2023-01-26T23:59:59"] assert agent_mock[0].data["emails"] == ["abuse@godaddy.com"] + def testAgentWhois_whenMultipleContactNames_emitsMessages( scan_message: message.Message, test_agent: whois_domain_agent.AgentWhoisDomain, @@ -163,6 +164,7 @@ def testAgentWhois_whenMultipleContactNames_emitsMessages( ] assert agent_mock[0].data["contact_name"] == "Catherine Shapiro" + def testAgentWhois_whenDomainNameInputIsEmpty_NotEmitsMessages( scan_message_not_valid: message.Message, test_agent: whois_domain_agent.AgentWhoisDomain, @@ -179,6 +181,7 @@ def testAgentWhois_whenDomainNameInputIsEmpty_NotEmitsMessages( mock_whois.assert_not_called() assert len(agent_mock) == 0 + def testAgentWhois_whenDomainNameISEmpty_NotEmitsMessages( scan_message: message.Message, test_agent: whois_domain_agent.AgentWhoisDomain, From acead500962dce1bb84ef18e8bfe4bae77068544 Mon Sep 17 00:00:00 2001 From: Alaeddine Mesbahi Date: Thu, 2 Feb 2023 22:39:15 +0100 Subject: [PATCH 7/9] Update tests/whois_domain_agent_test.py --- tests/whois_domain_agent_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/whois_domain_agent_test.py b/tests/whois_domain_agent_test.py index 6fcacb3..bde00f7 100644 --- a/tests/whois_domain_agent_test.py +++ b/tests/whois_domain_agent_test.py @@ -182,7 +182,7 @@ def testAgentWhois_whenDomainNameInputIsEmpty_NotEmitsMessages( assert len(agent_mock) == 0 -def testAgentWhois_whenDomainNameISEmpty_NotEmitsMessages( +def testAgentWhois_whenDomainNameIsEmpty_notEmitsMessages( scan_message: message.Message, test_agent: whois_domain_agent.AgentWhoisDomain, agent_persist_mock: Any, From 9cf3aa042839e52ced70f3f1bf5e1efa6733a8c7 Mon Sep 17 00:00:00 2001 From: Alaeddine Mesbahi Date: Thu, 2 Feb 2023 22:39:21 +0100 Subject: [PATCH 8/9] Update tests/whois_domain_agent_test.py --- tests/whois_domain_agent_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/whois_domain_agent_test.py b/tests/whois_domain_agent_test.py index bde00f7..d89965e 100644 --- a/tests/whois_domain_agent_test.py +++ b/tests/whois_domain_agent_test.py @@ -150,6 +150,7 @@ def testAgentWhois_whenMultipleContactNames_emitsMessages( test_agent.start() test_agent.process(scan_message) mock_whois.assert_called_once() + assert len(agent_mock) > 0 assert agent_mock[0].selector == "v3.asset.domain_name.whois" assert agent_mock[0].data["name"] == "marksandspencer.at" From df441d3ca56e167217ad518ad97853fbc70afa0b Mon Sep 17 00:00:00 2001 From: mohsinenar Date: Fri, 3 Feb 2023 10:22:41 +0100 Subject: [PATCH 9/9] Apply black. --- tests/whois_domain_agent_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/whois_domain_agent_test.py b/tests/whois_domain_agent_test.py index d89965e..dfd3825 100644 --- a/tests/whois_domain_agent_test.py +++ b/tests/whois_domain_agent_test.py @@ -150,7 +150,7 @@ def testAgentWhois_whenMultipleContactNames_emitsMessages( test_agent.start() test_agent.process(scan_message) mock_whois.assert_called_once() - + assert len(agent_mock) > 0 assert agent_mock[0].selector == "v3.asset.domain_name.whois" assert agent_mock[0].data["name"] == "marksandspencer.at"