add docker image flag to pre-commit (#4223)

.changelog/4223.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Added the `--docker_image` and `--image_ref` flags to **pre-commit* to override values from configuration files.
+  type: feature
+pr_number: 4223

demisto_sdk/__main__.py

@@ -3687,6 +3687,16 @@ def pre_commit(
     docker: bool = typer.Option(
         True, "--docker/--no-docker", help="Whether to run docker based hooks or not."
     ),
+    image_ref: Optional[str] = typer.Option(
+        None,
+        "--image-ref",
+        help="The docker image reference to run docker hooks with. Overrides the docker image from YAML or native image config.",
+    ),
+    docker_image: Optional[str] = typer.Option(
+        None,
+        "--docker-image",
+        help="Override the `docker_image` property in the template file. This is a comma separated list of: `from-yml`, `native:dev`, `native:ga`, `native:candidate`.",
+    ),
     run_hook: Optional[str] = typer.Argument(None, help="A specific hook to run"),
     console_log_threshold: str = typer.Option(
         "INFO",
@@ -3733,6 +3743,8 @@ def pre_commit(
         verbose,
         show_diff_on_failure,
         run_docker_hooks=docker,
+        image_ref=image_ref,
+        docker_image=docker_image,
         dry_run=dry_run,
         run_hook=run_hook,
         pre_commit_template_path=pre_commit_template_path,

demisto_sdk/commands/pre_commit/hooks/docker.py

@@ -56,7 +56,8 @@ def get_docker_python_path() -> str:
 
 def with_native_tags(
     tags_to_files: Dict[str, List[Tuple[Path, IntegrationScript]]],
-    docker_image_flag: str,
+    docker_flags: Set[str],
+    docker_image: Optional[str],
 ) -> Dict[str, List[Tuple[Path, IntegrationScript]]]:
     """
     Adds the native image images into the dict with the files that should be run on them
@@ -67,30 +68,33 @@ def with_native_tags(
     Returns: The updated dict with the native images.
 
     """
-    docker_flags = set(docker_image_flag.split(","))
+
     all_tags_to_files = defaultdict(list)
     native_image_config = NativeImageConfig.get_instance()
 
     for image, scripts in tags_to_files.items():
         for file, obj in scripts:
-
             supported_native_images = ScriptIntegrationSupportedNativeImages(
                 _id=obj.object_id,
                 native_image_config=native_image_config,
                 docker_image=image,
-            ).get_supported_native_docker_tags(docker_flags)
+            ).get_supported_native_docker_tags(docker_flags, include_candidate=True)
+
             for native_image in supported_native_images:
-                all_tags_to_files[native_image].append((file, obj))
+                all_tags_to_files[docker_image or native_image].append((file, obj))
             if {
                 DockerImageFlagOption.FROM_YML.value,
                 DockerImageFlagOption.ALL_IMAGES.value,
             } & docker_flags:
-                all_tags_to_files[image].append((file, obj))
+                all_tags_to_files[docker_image or image].append((file, obj))
+
     return all_tags_to_files
 
 
 def docker_tag_to_runfiles(
-    files_to_run: Iterable[Tuple[Path, Optional[IntegrationScript]]], docker_image_flag
+    files_to_run: Iterable[Tuple[Path, Optional[IntegrationScript]]],
+    docker_image_flag: str,
+    docker_image: Optional[str] = None,
 ) -> Dict[str, List[Tuple[Path, IntegrationScript]]]:
     """
     Iterates over all files snf groups the files by the dockerimages
@@ -101,13 +105,14 @@ def docker_tag_to_runfiles(
     Returns: A dict of image to List of files(Tuple[path, obj]) including native images
 
     """
+    docker_flags = set(docker_image_flag.split(","))
     tags_to_files = defaultdict(list)
     for file, obj in files_to_run:
-        if not obj:
-            continue
-        for docker_image in obj.docker_images:
-            tags_to_files[docker_image].append((file, obj))
-    return with_native_tags(tags_to_files, docker_image_flag)
+        if obj:
+            for image in obj.docker_images:
+                tags_to_files[image].append((file, obj))
+
+    return with_native_tags(tags_to_files, docker_flags, docker_image)
 
 
 @functools.lru_cache(maxsize=512)
@@ -266,7 +271,8 @@ def prepare_hook(
         }
         tag_to_files_objs = docker_tag_to_runfiles(
             filtered_files_with_objects,
-            self._get_property("docker_image", "from-yml"),
+            self.context.docker_image or self._get_property("docker_image", "from-yml"),
+            self.context.image_ref,
         )
         end_time = time.time()
         logger.debug(
@@ -286,7 +292,9 @@ def prepare_hook(
         config_arg = self._get_config_file_arg()
         start_time = time.time()
         logger.debug(f"{len(tag_to_files_objs)} images were collected from files")
-        logger.debug(f'collected images: {" ".join(tag_to_files_objs.keys())}')
+        logger.debug(
+            f'collected images: {" ".join(filter(None, tag_to_files_objs.keys()))}'
+        )
         docker_hook_ids = []
         with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
             results = []

demisto_sdk/commands/pre_commit/pre_commit_command.py

@@ -51,7 +51,7 @@
 SKIPPED_HOOKS = {"format", "validate", "secrets"}
 
 INTEGRATION_SCRIPT_REGEX = re.compile(r"^Packs/.*/(?:Integrations|Scripts)/.*.yml$")
-INTEGRATIONS_BATCH = 300
+INTEGRATIONS_BATCH = 100
 
 
 class PreCommitRunner:
@@ -476,6 +476,8 @@ def pre_commit_manager(
     show_diff_on_failure: bool = False,
     dry_run: bool = False,
     run_docker_hooks: bool = True,
+    image_ref: Optional[str] = None,
+    docker_image: Optional[str] = None,
     run_hook: Optional[str] = None,
     pre_commit_template_path: Optional[Path] = None,
 ) -> int:
@@ -494,6 +496,8 @@ def pre_commit_manager(
         show_diff_on_failure (bool, optional): Whether show git diff after pre-commit failure. Defaults to False.
         dry_run (bool, optional): Whether to run the pre-commit hooks in dry-run mode, which will only create the config file.
         run_docker_hooks (bool, optional): Whether to run docker based hooks or not.
+        image_ref: (str, optional): Override the image from YAML / native config file with this image reference.
+        docker_image: (str, optional): Override the `docker_image` property in the template file. This is a comma separated list of: `from-yml`, `native:dev`, `native:ga`, `native:candidate`.
         pre_commit_template_path (Path, optional): Path to the template pre-commit file.
 
     Returns:
@@ -554,6 +558,8 @@ def pre_commit_manager(
         run_hook,
         skipped_hooks,
         run_docker_hooks,
+        image_ref,
+        docker_image,
         pre_commit_template_path=pre_commit_template_path,
     )
     return PreCommitRunner.prepare_and_run(

demisto_sdk/commands/pre_commit/pre_commit_context.py

@@ -49,6 +49,8 @@ class PreCommitContext:
     run_hook: Optional[str] = None
     skipped_hooks: Set[str] = field(default_factory=set)
     run_docker_hooks: bool = True
+    image_ref: Optional[str] = None
+    docker_image: Optional[str] = None
     dry_run: bool = False
     pre_commit_template_path: Path = PRECOMMIT_TEMPLATE_PATH
 

demisto_sdk/commands/pre_commit/tests/generic_docker_test.py

@@ -267,3 +267,65 @@ def test_docker_pass_extra_args(mocker):
     mocker.patch.object(PreCommitContext, "dry_run", True)
     DockerHook(**hook).prepare_hook()
     assert "--rm=false" in hook["repo"]["hooks"][0]["entry"]
+
+
+def test_image_ref_argument(mocker):
+    """
+    Given:
+        - An object to run pre-commit on
+
+    When:
+        - Providing the `image_ref` flag to override the image to a custom image
+
+    Then:
+        - The hook will run on the provided image ref instead of the image of the YAML
+
+    """
+    file_path = Path("SomeFile.py")
+    files = [(file_path, Obj(object_id="id1"))]
+    hook = create_hook({"id": "test"}, image_ref="python3-custom-image")
+    mocker.patch.object(
+        PreCommitContext,
+        "files_to_run_with_objects",
+        files,
+    )
+    mocker.patch(
+        "demisto_sdk.commands.pre_commit.hooks.docker.devtest_image",
+        return_value="devtestimg",
+    )
+    DockerHook(**hook).prepare_hook()
+    assert hook["repo"]["hooks"][0]["id"] == "test-python3-custom-image"
+
+
+def test_docker_image_argument(mocker):
+    """
+    Given:
+        - An object to run pre-commit on
+
+    When:
+        - Providing the `docker_image` to `native:candidate` and `image_ref` flag for a candidate to test a custom candidate
+
+    Then:
+        - The hook will run on the provided image ref instead of the image of the native image config file
+
+    """
+    file_path = Path("SomeFile.py")
+    files = [
+        (file_path, Obj(object_id="id1", docker_image="demisto/python3:3.10.13.89009"))
+    ]
+    hook = create_hook(
+        {"id": "test"},
+        image_ref="python3-candidate-image",
+        docker_image="native:candidate",
+    )
+    mocker.patch.object(
+        PreCommitContext,
+        "files_to_run_with_objects",
+        files,
+    )
+    mocker.patch(
+        "demisto_sdk.commands.pre_commit.hooks.docker.devtest_image",
+        return_value="devtestimg",
+    )
+    DockerHook(**hook).prepare_hook()
+    assert hook["repo"]["hooks"][0]["id"] == "test-python3-candidate-image"

demisto_sdk/commands/pre_commit/tests/pre_commit_test.py

@@ -68,14 +68,23 @@ def create_hook(
     mode: str = "",
     all_files=False,
     input_files: Optional[List[Path]] = None,
+    image_ref: Optional[str] = None,
+    docker_image: Optional[str] = None,
 ):
     """
     This function mocks hook as he returns in _get_hooks() function
     """
 
     repo_and_hook: dict = {
         "repo": {"repo": "repo", "hooks": [hook]},
-        "context": PreCommitContext(input_files, all_files, mode, {}),
+        "context": PreCommitContext(
+            input_files,
+            all_files,
+            mode,
+            {},
+            image_ref=image_ref,
+            docker_image=docker_image,
+        ),
     }
     repo_and_hook["hook"] = repo_and_hook["repo"]["hooks"][0]
     return repo_and_hook