secrets: Fix regex whitelist (#351)

* Fixed finding secrets as a substring

* Fixed finding secrets as a substring

* added integration test

* add integration test

* rename functions

* fix integration test

* maybe in circle itll work

* changed the docstring

* added global test to secrets

* test

* change working env for test

* update description and function name

demisto_sdk/commands/secrets/secrets.py

@@ -56,6 +56,8 @@
 DATES_REGEX = r'((\d{4}[/.-]\d{2}[/.-]\d{2})[T\s](\d{2}:?\d{2}:?\d{2}:?(\.\d{5,10})?([+-]\d{2}:?\d{2})?Z?)?)'
 # false positives
 UUID_REGEX = r'([\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{8,12})'
+# find any substring
+WHILEIST_REGEX = r'\S*{}\S*'
 # disable-secrets-detection-end
 
 
@@ -211,18 +213,18 @@ def search_potential_secrets(self, secrets_file_paths: list, ignore_entropy: boo
         return secrets_found
 
     @staticmethod
-    def remove_whitelisted_items_from_file(file_content: str, secrets_white_list: list) -> str:
+    def remove_whitelisted_items_from_file(file_content: str, secrets_white_list: set) -> str:
         """Removes whitelisted items from file content
 
         Arguments:
             file_content (str): The content of the file to remove the whitelisted item from
-            secrets_white_list (list): List of whitelist items to remove from the file content.
+            secrets_white_list (set): List of whitelist items to remove from the file content.
 
         Returns:
             str: The file content with the whitelisted items removed.
         """
         for item in secrets_white_list:
-            file_content = file_content.replace(item, '')
+            file_content = re.sub(WHILEIST_REGEX.format(item), '', file_content)
         return file_content
 
     @staticmethod

demisto_sdk/commands/secrets/tests/secrets_test.py

@@ -1,6 +1,7 @@
 import io
 import json
 import os
+import re
 import shutil
 
 from demisto_sdk.commands.common.git_tools import git_path
@@ -174,6 +175,17 @@ def test_remove_white_list_regex(self):
         file_contents = self.validator.remove_whitelisted_items_from_file(white_list, file_contents)
         assert white_list not in file_contents
 
+    def test_remove_whitelisted_items_from_file_substring(self):
+        white_list = 'url.com'
+        file_contents = '''
+        url.com
+        boop
+        cool@url.com
+        shmoop
+        https://url.com
+        '''
+        assert not re.match(white_list, file_contents)
+
     def test_temp_white_list(self):
         file_contents = self.validator.get_file_contents(self.TEST_YML_FILE, '.yml')
         temp_white_list = self.validator.create_temp_white_list(file_contents)

demisto_sdk/tests/integration_tests/Tests/secrets_white_list.json

@@ -0,0 +1,9 @@
+{
+  "iocs": [],
+  "urls": [],
+  "somethingelse": [],
+  "generic_strings": [
+    "365ForMarketingEmail",
+    "feedBypassExclusionList"
+  ]
+}

demisto_sdk/tests/integration_tests/secrets_integration_test.py

@@ -1,4 +1,7 @@
+import json
+from os import chdir
 from os.path import join
+from pathlib import Path
 
 from click.testing import CliRunner
 from demisto_sdk.__main__ import main
@@ -9,7 +12,7 @@
 SECRETS_WHITELIST = join(DEMISTO_SDK_PATH, "tests/test_files/secrets_white_list.json")
 
 
-def test_integration_secrets_positive(mocker):
+def test_integration_secrets_incident_field_positive(mocker):
     """
     Given
     - Valid `city` incident field.
@@ -37,7 +40,7 @@ def test_integration_secrets_positive(mocker):
     assert result.stderr == ""
 
 
-def test_integration_secrets_negative(mocker):
+def test_integration_secrets_integration_negative(mocker):
     """
     Given
     - FeedAzure integration yml with secrets.
@@ -67,3 +70,66 @@ def test_integration_secrets_negative(mocker):
     assert "Dynamics365ForMarketingEmail" in result.stdout
     assert "Remove or whitelist secrets in order to proceed, then re-commit" in result.stdout
     assert result.stderr == ""
+
+
+def test_integration_secrets_integration_positive(mocker, tmp_path):
+    """
+    Given
+    - FeedAzure integration yml with secrets.
+
+    When
+    - Running secrets validation on it.
+
+    Then
+    - Ensure secrets validation succeed.
+    """
+    integration_with_secrets_path = join(
+        DEMISTO_SDK_PATH, "tests/test_files/content_repo_example/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.yml"
+    )
+    mocker.patch(
+        "demisto_sdk.__main__.SecretsValidator.get_all_diff_text_files",
+        return_value=[integration_with_secrets_path]
+    )
+    whitelist = {
+        "iocs": [],
+        "urls": [],
+        "somethingelse": [],
+        "generic_strings": [
+            "365ForMarketingEmail",
+            "feedBypassExclusionList"
+        ]
+    }
+    whitelist_path = tmp_path / "whitelist.txt"
+    whitelist_path.write_text(json.dumps(whitelist))
+    runner = CliRunner(mix_stderr=False)
+    result = runner.invoke(main, [SECRETS_CMD, '-wl', Path(whitelist_path)], catch_exceptions=False)
+    assert result.exit_code == 0
+    assert not result.stderr
+    assert "no secrets were found" in result.stdout
+
+
+def test_integration_secrets_integration_global_whitelist_positive(mocker):
+    """
+    Given
+    - An integration yml with secrets.
+    - Content Repo with whitelist file in it (Tests/secrets_white_list.json)
+
+    When
+    - Running secrets validation on it.
+
+    Then
+    - Ensure secrets validation succeed.
+    """
+    integration_with_secrets_path = join(
+        DEMISTO_SDK_PATH, "tests/test_files/content_repo_example/Packs/FeedAzure/Integrations/FeedAzure/FeedAzure.yml"
+    )
+    mocker.patch(
+        "demisto_sdk.__main__.SecretsValidator.get_all_diff_text_files",
+        return_value=[integration_with_secrets_path]
+    )
+    chdir(join(DEMISTO_SDK_PATH, "tests", "integration_tests"))
+    runner = CliRunner(mix_stderr=False)
+    result = runner.invoke(main, [SECRETS_CMD], catch_exceptions=False)
+    assert result.exit_code == 0
+    assert not result.stderr
+    assert "no secrets were found" in result.stdout