From 6e94027dcb7d5df26b405c0e27a78e58fb343573 Mon Sep 17 00:00:00 2001 From: Tal Zichlinsky <35036457+talzich@users.noreply.github.com> Date: Mon, 9 Sep 2024 18:15:35 +0300 Subject: [PATCH] Fix yml files for incremental feeds (#35937) * Get updates from pt1 * Replace created file path with a temp file * Replace created file path with a temp file * Remove teardown and setup functions for TestCreateFile * Add RN * Add RN --- Packs/CortexXDR/.pack-ignore | 2 +- .../Integrations/XDR_iocs/XDR_iocs.py | 16 +++---- .../Integrations/XDR_iocs/XDR_iocs.yml | 3 +- .../Integrations/XDR_iocs/XDR_iocs_test.py | 48 +++++++++---------- Packs/CortexXDR/ReleaseNotes/6_1_65.md | 5 ++ Packs/CortexXDR/ReleaseNotes/6_1_69.md | 6 +++ Packs/CortexXDR/pack_metadata.json | 2 +- .../FeedElasticsearch/FeedElasticsearch.yml | 7 +++ Packs/FeedElasticsearch/ReleaseNotes/1_1_7.md | 6 +++ Packs/FeedElasticsearch/pack_metadata.json | 2 +- .../Integrations/FeedTAXII/FeedTAXII.yml | 9 +++- Packs/FeedTAXII/ReleaseNotes/1_2_18.md | 7 +++ Packs/FeedTAXII/pack_metadata.json | 2 +- 13 files changed, 75 insertions(+), 40 deletions(-) create mode 100644 Packs/CortexXDR/ReleaseNotes/6_1_69.md create mode 100644 Packs/FeedElasticsearch/ReleaseNotes/1_1_7.md create mode 100644 Packs/FeedTAXII/ReleaseNotes/1_2_18.md diff --git a/Packs/CortexXDR/.pack-ignore b/Packs/CortexXDR/.pack-ignore index 863b475b6ce9..e904eaf5d751 100644 --- a/Packs/CortexXDR/.pack-ignore +++ b/Packs/CortexXDR/.pack-ignore @@ -5,7 +5,7 @@ ignore=PA116 ignore=IN126,IN139 [file:XDR_iocs.yml] -ignore=BA108,BA109 +ignore=BA108,BA109,IN122 [file:XDRSyncScript_5_9_9.yml] ignore=BA108,BA109,BA110 diff --git a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py index f3d2b5a7345a..aa031efef4b6 100644 --- a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py +++ b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py @@ -121,7 +121,7 @@ def get_headers(params: dict) -> dict: api_key: str = params.get('apikey_creds', {}).get('password', '') or str(params.get('apikey')) api_key_id: str = params.get('apikey_id_creds', {}).get('password', '') or str(params.get('apikey_id')) nonce: str = "".join([secrets.choice(string.ascii_letters + string.digits) for _ in range(64)]) - timestamp: str = str(int(datetime.now(timezone.utc).timestamp()) * 1000) + timestamp: str = str(int(datetime.now(timezone.utc).timestamp()) * 1000) # noqa: UP017 auth_key = f"{api_key}{nonce}{timestamp}" auth_key = auth_key.encode("utf-8") api_key_hash: str = hashlib.sha256(auth_key).hexdigest() @@ -214,7 +214,7 @@ def demisto_expiration_to_xdr(expiration) -> int: try: expiration_date = parse(expiration) assert expiration_date is not None, f'could not parse {expiration}' - return int(expiration_date.astimezone(timezone.utc).timestamp() * 1000) + return int(expiration_date.astimezone(timezone.utc).timestamp() * 1000) # noqa: UP017 except (ValueError, AssertionError): pass return -1 @@ -387,7 +387,7 @@ def sync(client: Client): demisto.info("executing sync") temp_file_path: str = get_temp_file() try: - sync_time = datetime.now(timezone.utc) + sync_time = datetime.now(timezone.utc) # noqa: UP017 create_file_sync(temp_file_path) # may end up empty requests_kwargs: dict = get_requests_kwargs(file_path=temp_file_path) path: str = 'sync_tim_iocs' @@ -614,7 +614,7 @@ def get_changes(client: Client): def module_test(client: Client): - ts = int(datetime.now(timezone.utc).timestamp() * 1000) - 1 + ts = int(datetime.now(timezone.utc).timestamp() * 1000) - 1 # noqa: UP017 path, requests_kwargs = prepare_get_changes(ts) requests_kwargs: dict = get_requests_kwargs(_json=requests_kwargs) demisto.debug(f"calling endpoint {path} with {requests_kwargs=}") @@ -655,7 +655,7 @@ def set_new_iocs_to_keep_time(): offset = secrets.randbelow(115) hour, minute = divmod(offset, 60) hour += 1 - last_ioc_to_keep = datetime.now(timezone.utc) + last_ioc_to_keep = datetime.now(timezone.utc) # noqa: UP017 last_ioc_to_keep = last_ioc_to_keep.replace(hour=hour, minute=minute) + timedelta( days=1 ) @@ -680,10 +680,10 @@ def is_iocs_to_keep_time(): set_new_iocs_to_keep_time() next_iocs_to_keep_time = get_integration_context().get("next_iocs_to_keep_time") - time_now = datetime.now(timezone.utc) + time_now = datetime.now(timezone.utc) # noqa: UP017 if ( time_now.hour in range(1, 3) - and time_now > datetime.strptime(next_iocs_to_keep_time, DEMISTO_TIME_FORMAT).replace(tzinfo=timezone.utc) + and time_now > datetime.strptime(next_iocs_to_keep_time, DEMISTO_TIME_FORMAT).replace(tzinfo=timezone.utc) # noqa: UP017 ): return True @@ -723,7 +723,7 @@ def get_indicator_xdr_score(indicator: str, xdr_server: int): def get_sync_file(set_time: bool = False, zip: bool = False) -> None: temp_file_path = get_temp_file() - timestamp = datetime.now(timezone.utc) + timestamp = datetime.now(timezone.utc) # noqa: UP017 demisto.debug(f"creating sync file with {timestamp=!s}") try: create_file_sync(temp_file_path) diff --git a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml index 984e2e0ebd3b..b2c868e9cdcb 100644 --- a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml +++ b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml @@ -111,7 +111,6 @@ configuration: - never - interval - indicatorType - - suddenDeath required: false - display: '' name: feedExpirationInterval @@ -220,7 +219,7 @@ script: required: true description: Disables IOCs in the XDR server. name: xdr-iocs-disable - dockerimage: demisto/python3:3.10.13.89009 + dockerimage: demisto/python3:3.11.9.107902 feed: true runonce: false script: '-' diff --git a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py index c4afc860e5ec..9cb608d51473 100644 --- a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py +++ b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py @@ -1,3 +1,5 @@ +import tempfile + from XDR_iocs import * import pytest from freezegun import freeze_time @@ -180,17 +182,6 @@ class TestCreateFile: ('File_iocs', 'File_iocs_to_keep_file') ] - @classmethod - def setup_method(cls): - # creates the file - with open(TestCreateFile.path, 'w') as _file: - _file.write('') - - @classmethod - def teardown_method(cls): - # removes the file when done - os.remove(TestCreateFile.path) - @staticmethod def get_file(path): with open(path) as _file: @@ -221,8 +212,9 @@ def test_create_file_sync_without_iocs(self, mocker): - Verify sync file data. """ mocker.patch.object(demisto, 'searchIndicators', return_value={"total": 0}) - create_file_sync(TestCreateFile.path) - data = self.get_file(TestCreateFile.path) + with tempfile.NamedTemporaryFile(mode='w') as temp_file: + create_file_sync(temp_file.name) + data = self.get_file(temp_file.name) expected_data = '' assert data == expected_data, f'create_file_sync with no iocs\n\tcreates: {data}\n\tinstead: {expected_data}' @@ -237,8 +229,9 @@ def test_create_file_sync(self, in_iocs, out_iocs, mocker): - Verify sync file data. """ mocker.patch.object(demisto, 'searchIndicators', return_value=json.loads(self.get_file(f'test_data/{in_iocs}.json'))) # noqa: E501 - create_file_sync(TestCreateFile.path) - data = self.get_file(TestCreateFile.path) + with tempfile.NamedTemporaryFile(mode='w') as temp_file: + create_file_sync(temp_file.name) + data = self.get_file(temp_file.name) expected_data = self.get_file(f'test_data/{out_iocs}.txt') assert data == expected_data, f'create_file_sync with {in_iocs} iocs\n\tcreates: {data}\n\tinstead: {expected_data}' @@ -253,8 +246,9 @@ def test_create_file_sync_all_types(self, mocker): """ all_iocs, expected_data = self.get_all_iocs(self.data_test_create_file_sync, 'txt') mocker.patch.object(demisto, 'searchIndicators', return_value=all_iocs) - create_file_sync(TestCreateFile.path) - data = self.get_file(TestCreateFile.path) + with tempfile.NamedTemporaryFile(mode='w') as temp_file: + create_file_sync(temp_file.name) + data = self.get_file(temp_file.name) assert data == expected_data, f'create_file_sync with all iocs\n\tcreates: {data}\n\tinstead: {expected_data}' data_test_create_file_with_empty_indicators = [ @@ -277,8 +271,9 @@ def test_create_file_sync_with_empty_indicators(self, defective_indicator, mocke all_iocs['iocs'].append(defective_indicator) all_iocs['total'] += 1 mocker.patch.object(demisto, 'searchIndicators', return_value=all_iocs) - create_file_sync(TestCreateFile.path) - data = self.get_file(TestCreateFile.path) + with tempfile.NamedTemporaryFile(mode='w') as temp_file: + create_file_sync(temp_file.name) + data = self.get_file(temp_file.name) assert data == expected_data, f'create_file_sync with all iocs\n\tcreates: {data}\n\tinstead: {expected_data}' def test_create_file_iocs_to_keep_without_iocs(self, mocker): @@ -292,8 +287,9 @@ def test_create_file_iocs_to_keep_without_iocs(self, mocker): """ mocker.patch.object(demisto, 'searchIndicators', return_value={"total": 0}) - create_file_iocs_to_keep(TestCreateFile.path) - data = self.get_file(TestCreateFile.path) + with tempfile.NamedTemporaryFile(mode='w') as temp_file: + create_file_iocs_to_keep(temp_file.name) + data = self.get_file(temp_file.name) expected_data = ' ' assert data == expected_data, f'create_file_iocs_to_keep with no iocs\n\tcreates: {data}\n\tinstead: {expected_data}' @@ -309,8 +305,9 @@ def test_create_file_iocs_to_keep(self, in_iocs, out_iocs, mocker): """ mocker.patch.object(demisto, 'searchIndicators', return_value=json.loads( self.get_file(f'test_data/{in_iocs}.json'))) - create_file_iocs_to_keep(TestCreateFile.path) - data = self.get_file(TestCreateFile.path) + with tempfile.NamedTemporaryFile(mode='w') as temp_file: + create_file_iocs_to_keep(temp_file.name) + data = self.get_file(temp_file.name) expected_data = self.get_file(f'test_data/{out_iocs}.txt') assert data == expected_data, f'create_file_iocs_to_keep with {in_iocs} iocs\n\tcreates: {data}\n\tinstead: {expected_data}' # noqa: E501 @@ -325,8 +322,9 @@ def test_create_file_iocs_to_keep_all_types(self, mocker): """ all_iocs, expected_data = self.get_all_iocs(self.data_test_create_file_iocs_to_keep, 'txt') mocker.patch.object(demisto, 'searchIndicators', return_value=all_iocs) - create_file_iocs_to_keep(TestCreateFile.path) - data = self.get_file(TestCreateFile.path) + with tempfile.NamedTemporaryFile('w') as temp_file: + create_file_iocs_to_keep(temp_file.name) + data = self.get_file(temp_file.name) assert data == expected_data, f'create_file_iocs_to_keep with all iocs\n\tcreates: {data}\n\tinstead: {expected_data}' diff --git a/Packs/CortexXDR/ReleaseNotes/6_1_65.md b/Packs/CortexXDR/ReleaseNotes/6_1_65.md index c6ed5d5d1e84..440ecfd7c80d 100644 --- a/Packs/CortexXDR/ReleaseNotes/6_1_65.md +++ b/Packs/CortexXDR/ReleaseNotes/6_1_65.md @@ -1,6 +1,11 @@ #### Integrations +##### Cortex XDR - IOC + +- Removed *When removed from the feed* option from *Indicator Expiration Method* +- Updated the Docker image to: *demisto/python3:3.11.9.107902*. +- ##### Palo Alto Networks Cortex XDR - Investigation and Response - Fixed an issue where the ***get-endpoint*** and ***endpoint*** commands failed when the endpoint did not include an IP field. diff --git a/Packs/CortexXDR/ReleaseNotes/6_1_69.md b/Packs/CortexXDR/ReleaseNotes/6_1_69.md new file mode 100644 index 000000000000..76723140685f --- /dev/null +++ b/Packs/CortexXDR/ReleaseNotes/6_1_69.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### Cortex XDR - IOC + +- Updated the Docker image to: *demisto/python3:3.11.9.107902*. diff --git a/Packs/CortexXDR/pack_metadata.json b/Packs/CortexXDR/pack_metadata.json index 06263da338eb..644d6409aaf8 100644 --- a/Packs/CortexXDR/pack_metadata.json +++ b/Packs/CortexXDR/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Cortex XDR by Palo Alto Networks", "description": "Automates Cortex XDR incident response, and includes custom Cortex XDR incident views and layouts to aid analyst investigations.", "support": "xsoar", - "currentVersion": "6.1.68", + "currentVersion": "6.1.69", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.yml b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.yml index f807532ab463..449fde5475af 100644 --- a/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.yml +++ b/Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.yml @@ -159,6 +159,13 @@ configuration: name: es_query type: 0 required: false +- additionalinfo: Incremental feeds pull only new or modified indicators that have been sent from the integration. As the determination if the indicator is new or modified happens on the 3rd-party vendor's side, and only indicators that are new or modified are sent to Cortex XSOAR, all indicators coming from these feeds are labeled new or modified. + defaultvalue: 'true' + display: Incremental Feed + hidden: true + name: feedIncremental + type: 8 + required: false description: Fetches indicators stored in an Elasticsearch database. display: Elasticsearch Feed name: ElasticsearchFeed diff --git a/Packs/FeedElasticsearch/ReleaseNotes/1_1_7.md b/Packs/FeedElasticsearch/ReleaseNotes/1_1_7.md new file mode 100644 index 000000000000..fa8270c5d42e --- /dev/null +++ b/Packs/FeedElasticsearch/ReleaseNotes/1_1_7.md @@ -0,0 +1,6 @@ + +#### Integrations + +##### Elasticsearch Feed + +- Added incremental feed indication to yml file. diff --git a/Packs/FeedElasticsearch/pack_metadata.json b/Packs/FeedElasticsearch/pack_metadata.json index c9f002aa6b5b..1b7d7f674b8a 100644 --- a/Packs/FeedElasticsearch/pack_metadata.json +++ b/Packs/FeedElasticsearch/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Elasticsearch Feed", "description": "Indicators feed from Elasticsearch database", "support": "xsoar", - "currentVersion": "1.1.6", + "currentVersion": "1.1.7", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII.yml b/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII.yml index 0b7c51fca26d..572eb8e135ac 100644 --- a/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII.yml +++ b/Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII.yml @@ -18,6 +18,13 @@ configuration: - Bad type: 18 required: false +- additionalinfo: Incremental feeds pull only new or modified indicators that have been sent from the integration. The determination if the indicator is new or modified happens on the 3rd-party vendor's side, so only indicators that are new or modified are sent to Cortex XSOAR. Therefore, all indicators coming from these feeds are labeled new or modified. + defaultvalue: 'true' + display: Incremental feed + hidden: true + name: feedIncremental + type: 8 + required: false - additionalinfo: Reliability of the source providing the intelligence data display: Source Reliability name: feedReliability @@ -182,7 +189,7 @@ script: - contextPath: TAXII.Indicator.Rawjson description: The indicator rawJSON value. type: Unknown - dockerimage: demisto/taxii:1.0.0.106334 + dockerimage: demisto/taxii:1.0.0.108107 feed: true runonce: false script: '' diff --git a/Packs/FeedTAXII/ReleaseNotes/1_2_18.md b/Packs/FeedTAXII/ReleaseNotes/1_2_18.md new file mode 100644 index 000000000000..de5c01c3065b --- /dev/null +++ b/Packs/FeedTAXII/ReleaseNotes/1_2_18.md @@ -0,0 +1,7 @@ + +#### Integrations + +##### TAXII Feed + +- Added incremental feed indication to yml file. +- Updated the Docker image to: *demisto/taxii:1.0.0.108107*. diff --git a/Packs/FeedTAXII/pack_metadata.json b/Packs/FeedTAXII/pack_metadata.json index 9ede6e34a779..1fef22e3b78a 100644 --- a/Packs/FeedTAXII/pack_metadata.json +++ b/Packs/FeedTAXII/pack_metadata.json @@ -2,7 +2,7 @@ "name": "TAXII Feed", "description": "Ingest indicator feeds from TAXII 1 and TAXII 2 servers.", "support": "xsoar", - "currentVersion": "1.2.17", + "currentVersion": "1.2.18", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",