Fix yml files for incremental feeds (demisto#35937)

* Get updates from pt1

* Replace created file path with a temp file

* Replace created file path with a temp file

* Remove teardown and setup functions for TestCreateFile

* Add RN

* Add RN

Packs/CortexXDR/.pack-ignore

@@ -5,7 +5,7 @@ ignore=PA116
 ignore=IN126,IN139
 
 [file:XDR_iocs.yml]
-ignore=BA108,BA109
+ignore=BA108,BA109,IN122
 
 [file:XDRSyncScript_5_9_9.yml]
 ignore=BA108,BA109,BA110

Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py

@@ -121,7 +121,7 @@ def get_headers(params: dict) -> dict:
     api_key: str = params.get('apikey_creds', {}).get('password', '') or str(params.get('apikey'))
     api_key_id: str = params.get('apikey_id_creds', {}).get('password', '') or str(params.get('apikey_id'))
     nonce: str = "".join([secrets.choice(string.ascii_letters + string.digits) for _ in range(64)])
-    timestamp: str = str(int(datetime.now(timezone.utc).timestamp()) * 1000)
+    timestamp: str = str(int(datetime.now(timezone.utc).timestamp()) * 1000)  # noqa: UP017
     auth_key = f"{api_key}{nonce}{timestamp}"
     auth_key = auth_key.encode("utf-8")
     api_key_hash: str = hashlib.sha256(auth_key).hexdigest()
@@ -214,7 +214,7 @@ def demisto_expiration_to_xdr(expiration) -> int:
         try:
             expiration_date = parse(expiration)
             assert expiration_date is not None, f'could not parse {expiration}'
-            return int(expiration_date.astimezone(timezone.utc).timestamp() * 1000)
+            return int(expiration_date.astimezone(timezone.utc).timestamp() * 1000)  # noqa: UP017
         except (ValueError, AssertionError):
             pass
     return -1
@@ -387,7 +387,7 @@ def sync(client: Client):
     demisto.info("executing sync")
     temp_file_path: str = get_temp_file()
     try:
-        sync_time = datetime.now(timezone.utc)
+        sync_time = datetime.now(timezone.utc)  # noqa: UP017
         create_file_sync(temp_file_path)  # may end up empty
         requests_kwargs: dict = get_requests_kwargs(file_path=temp_file_path)
         path: str = 'sync_tim_iocs'
@@ -614,7 +614,7 @@ def get_changes(client: Client):
 
 
 def module_test(client: Client):
-    ts = int(datetime.now(timezone.utc).timestamp() * 1000) - 1
+    ts = int(datetime.now(timezone.utc).timestamp() * 1000) - 1  # noqa: UP017
     path, requests_kwargs = prepare_get_changes(ts)
     requests_kwargs: dict = get_requests_kwargs(_json=requests_kwargs)
     demisto.debug(f"calling endpoint {path} with {requests_kwargs=}")
@@ -655,7 +655,7 @@ def set_new_iocs_to_keep_time():
     offset = secrets.randbelow(115)
     hour, minute = divmod(offset, 60)
     hour += 1
-    last_ioc_to_keep = datetime.now(timezone.utc)
+    last_ioc_to_keep = datetime.now(timezone.utc)  # noqa: UP017
     last_ioc_to_keep = last_ioc_to_keep.replace(hour=hour, minute=minute) + timedelta(
         days=1
     )
@@ -680,10 +680,10 @@ def is_iocs_to_keep_time():
         set_new_iocs_to_keep_time()
         next_iocs_to_keep_time = get_integration_context().get("next_iocs_to_keep_time")
 
-    time_now = datetime.now(timezone.utc)
+    time_now = datetime.now(timezone.utc)  # noqa: UP017
     if (
         time_now.hour in range(1, 3)
-        and time_now > datetime.strptime(next_iocs_to_keep_time, DEMISTO_TIME_FORMAT).replace(tzinfo=timezone.utc)
+        and time_now > datetime.strptime(next_iocs_to_keep_time, DEMISTO_TIME_FORMAT).replace(tzinfo=timezone.utc)  # noqa: UP017
     ):
         return True
 
@@ -723,7 +723,7 @@ def get_indicator_xdr_score(indicator: str, xdr_server: int):
 def get_sync_file(set_time: bool = False, zip: bool = False) -> None:
     temp_file_path = get_temp_file()
 
-    timestamp = datetime.now(timezone.utc)
+    timestamp = datetime.now(timezone.utc)  # noqa: UP017
     demisto.debug(f"creating sync file with {timestamp=!s}")
     try:
         create_file_sync(temp_file_path)

Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml

@@ -111,7 +111,6 @@ configuration:
   - never
   - interval
   - indicatorType
-  - suddenDeath
   required: false
 - display: ''
   name: feedExpirationInterval
@@ -220,7 +219,7 @@ script:
       required: true
     description: Disables IOCs in the XDR server.
     name: xdr-iocs-disable
-  dockerimage: demisto/python3:3.10.13.89009
+  dockerimage: demisto/python3:3.11.9.107902
   feed: true
   runonce: false
   script: '-'

Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py

@@ -1,3 +1,5 @@
+import tempfile
+
 from XDR_iocs import *
 import pytest
 from freezegun import freeze_time
@@ -180,17 +182,6 @@ class TestCreateFile:
         ('File_iocs', 'File_iocs_to_keep_file')
     ]
 
-    @classmethod
-    def setup_method(cls):
-        # creates the file
-        with open(TestCreateFile.path, 'w') as _file:
-            _file.write('')
-
-    @classmethod
-    def teardown_method(cls):
-        # removes the file when done
-        os.remove(TestCreateFile.path)
-
     @staticmethod
     def get_file(path):
         with open(path) as _file:
@@ -221,8 +212,9 @@ def test_create_file_sync_without_iocs(self, mocker):
                 - Verify sync file data.
         """
         mocker.patch.object(demisto, 'searchIndicators', return_value={"total": 0})
-        create_file_sync(TestCreateFile.path)
-        data = self.get_file(TestCreateFile.path)
+        with tempfile.NamedTemporaryFile(mode='w') as temp_file:
+            create_file_sync(temp_file.name)
+            data = self.get_file(temp_file.name)
         expected_data = ''
         assert data == expected_data, f'create_file_sync with no iocs\n\tcreates: {data}\n\tinstead: {expected_data}'
 
@@ -237,8 +229,9 @@ def test_create_file_sync(self, in_iocs, out_iocs, mocker):
                 - Verify sync file data.
         """
         mocker.patch.object(demisto, 'searchIndicators', return_value=json.loads(self.get_file(f'test_data/{in_iocs}.json')))  # noqa: E501
-        create_file_sync(TestCreateFile.path)
-        data = self.get_file(TestCreateFile.path)
+        with tempfile.NamedTemporaryFile(mode='w') as temp_file:
+            create_file_sync(temp_file.name)
+            data = self.get_file(temp_file.name)
         expected_data = self.get_file(f'test_data/{out_iocs}.txt')
         assert data == expected_data, f'create_file_sync with {in_iocs} iocs\n\tcreates: {data}\n\tinstead: {expected_data}'
 
@@ -253,8 +246,9 @@ def test_create_file_sync_all_types(self, mocker):
         """
         all_iocs, expected_data = self.get_all_iocs(self.data_test_create_file_sync, 'txt')
         mocker.patch.object(demisto, 'searchIndicators', return_value=all_iocs)
-        create_file_sync(TestCreateFile.path)
-        data = self.get_file(TestCreateFile.path)
+        with tempfile.NamedTemporaryFile(mode='w') as temp_file:
+            create_file_sync(temp_file.name)
+            data = self.get_file(temp_file.name)
         assert data == expected_data, f'create_file_sync with all iocs\n\tcreates: {data}\n\tinstead: {expected_data}'
 
     data_test_create_file_with_empty_indicators = [
@@ -277,8 +271,9 @@ def test_create_file_sync_with_empty_indicators(self, defective_indicator, mocke
         all_iocs['iocs'].append(defective_indicator)
         all_iocs['total'] += 1
         mocker.patch.object(demisto, 'searchIndicators', return_value=all_iocs)
-        create_file_sync(TestCreateFile.path)
-        data = self.get_file(TestCreateFile.path)
+        with tempfile.NamedTemporaryFile(mode='w') as temp_file:
+            create_file_sync(temp_file.name)
+            data = self.get_file(temp_file.name)
         assert data == expected_data, f'create_file_sync with all iocs\n\tcreates: {data}\n\tinstead: {expected_data}'
 
     def test_create_file_iocs_to_keep_without_iocs(self, mocker):
@@ -292,8 +287,9 @@ def test_create_file_iocs_to_keep_without_iocs(self, mocker):
         """
 
         mocker.patch.object(demisto, 'searchIndicators', return_value={"total": 0})
-        create_file_iocs_to_keep(TestCreateFile.path)
-        data = self.get_file(TestCreateFile.path)
+        with tempfile.NamedTemporaryFile(mode='w') as temp_file:
+            create_file_iocs_to_keep(temp_file.name)
+            data = self.get_file(temp_file.name)
         expected_data = ' '
         assert data == expected_data, f'create_file_iocs_to_keep with no iocs\n\tcreates: {data}\n\tinstead: {expected_data}'
 
@@ -309,8 +305,9 @@ def test_create_file_iocs_to_keep(self, in_iocs, out_iocs, mocker):
         """
         mocker.patch.object(demisto, 'searchIndicators', return_value=json.loads(
             self.get_file(f'test_data/{in_iocs}.json')))
-        create_file_iocs_to_keep(TestCreateFile.path)
-        data = self.get_file(TestCreateFile.path)
+        with tempfile.NamedTemporaryFile(mode='w') as temp_file:
+            create_file_iocs_to_keep(temp_file.name)
+            data = self.get_file(temp_file.name)
         expected_data = self.get_file(f'test_data/{out_iocs}.txt')
         assert data == expected_data, f'create_file_iocs_to_keep with {in_iocs} iocs\n\tcreates: {data}\n\tinstead: {expected_data}'    # noqa: E501
 
@@ -325,8 +322,9 @@ def test_create_file_iocs_to_keep_all_types(self, mocker):
         """
         all_iocs, expected_data = self.get_all_iocs(self.data_test_create_file_iocs_to_keep, 'txt')
         mocker.patch.object(demisto, 'searchIndicators', return_value=all_iocs)
-        create_file_iocs_to_keep(TestCreateFile.path)
-        data = self.get_file(TestCreateFile.path)
+        with tempfile.NamedTemporaryFile('w') as temp_file:
+            create_file_iocs_to_keep(temp_file.name)
+            data = self.get_file(temp_file.name)
         assert data == expected_data, f'create_file_iocs_to_keep with all iocs\n\tcreates: {data}\n\tinstead: {expected_data}'
 
 

Packs/CortexXDR/ReleaseNotes/6_1_65.md

@@ -1,6 +1,11 @@
 
 #### Integrations
 
+##### Cortex XDR - IOC
+
+- Removed *When removed from the feed* option from *Indicator Expiration Method*
+- Updated the Docker image to: *demisto/python3:3.11.9.107902*.
+- 
 ##### Palo Alto Networks Cortex XDR - Investigation and Response
 
 - Fixed an issue where the ***get-endpoint*** and ***endpoint*** commands failed when the endpoint did not include an IP field.

Packs/CortexXDR/ReleaseNotes/6_1_69.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### Cortex XDR - IOC
+
+- Updated the Docker image to: *demisto/python3:3.11.9.107902*.

Packs/CortexXDR/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Cortex XDR by Palo Alto Networks",
     "description": "Automates Cortex XDR incident response, and includes custom Cortex XDR incident views and layouts to aid analyst investigations.",
     "support": "xsoar",
-    "currentVersion": "6.1.68",
+    "currentVersion": "6.1.69",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/FeedElasticsearch/Integrations/FeedElasticsearch/FeedElasticsearch.yml

@@ -159,6 +159,13 @@ configuration:
   name: es_query
   type: 0
   required: false
+- additionalinfo: Incremental feeds pull only new or modified indicators that have been sent from the integration. As the determination if the indicator is new or modified happens on the 3rd-party vendor's side, and only indicators that are new or modified are sent to Cortex XSOAR, all indicators coming from these feeds are labeled new or modified.
+  defaultvalue: 'true'
+  display: Incremental Feed
+  hidden: true
+  name: feedIncremental
+  type: 8
+  required: false
 description: Fetches indicators stored in an Elasticsearch database.
 display: Elasticsearch Feed
 name: ElasticsearchFeed

Packs/FeedElasticsearch/ReleaseNotes/1_1_7.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### Elasticsearch Feed
+
+- Added incremental feed indication to yml file.

Packs/FeedElasticsearch/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Elasticsearch Feed",
     "description": "Indicators feed from Elasticsearch database",
     "support": "xsoar",
-    "currentVersion": "1.1.6",
+    "currentVersion": "1.1.7",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/FeedTAXII/Integrations/FeedTAXII/FeedTAXII.yml

@@ -18,6 +18,13 @@ configuration:
   - Bad
   type: 18
   required: false
+- additionalinfo: Incremental feeds pull only new or modified indicators that have been sent from the integration. The determination if the indicator is new or modified happens on the 3rd-party vendor's side, so only indicators that are new or modified are sent to Cortex XSOAR. Therefore, all indicators coming from these feeds are labeled new or modified.
+  defaultvalue: 'true'
+  display: Incremental feed
+  hidden: true
+  name: feedIncremental
+  type: 8
+  required: false
 - additionalinfo: Reliability of the source providing the intelligence data
   display: Source Reliability
   name: feedReliability
@@ -182,7 +189,7 @@ script:
     - contextPath: TAXII.Indicator.Rawjson
       description: The indicator rawJSON value.
       type: Unknown
-  dockerimage: demisto/taxii:1.0.0.106334
+  dockerimage: demisto/taxii:1.0.0.108107
   feed: true
   runonce: false
   script: ''

Packs/FeedTAXII/ReleaseNotes/1_2_18.md

@@ -0,0 +1,7 @@
+
+#### Integrations
+
+##### TAXII Feed
+
+- Added incremental feed indication to yml file.
+- Updated the Docker image to: *demisto/taxii:1.0.0.108107*.

Packs/FeedTAXII/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "TAXII Feed",
     "description": "Ingest indicator feeds from TAXII 1 and TAXII 2 servers.",
     "support": "xsoar",
-    "currentVersion": "1.2.17",
+    "currentVersion": "1.2.18",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",