diff --git a/.changelog/4223.yml b/.changelog/4223.yml new file mode 100644 index 00000000000..a450d8bdafb --- /dev/null +++ b/.changelog/4223.yml @@ -0,0 +1,4 @@ +changes: +- description: Added the `--docker_image` and `--image_ref` flags to **pre-commit* to override values from configuration files. + type: feature +pr_number: 4223 diff --git a/demisto_sdk/__main__.py b/demisto_sdk/__main__.py index 9b05f648bc5..f89a53517a3 100644 --- a/demisto_sdk/__main__.py +++ b/demisto_sdk/__main__.py @@ -3687,6 +3687,16 @@ def pre_commit( docker: bool = typer.Option( True, "--docker/--no-docker", help="Whether to run docker based hooks or not." ), + image_ref: Optional[str] = typer.Option( + None, + "--image-ref", + help="The docker image reference to run docker hooks with. Overrides the docker image from YAML or native image config.", + ), + docker_image: Optional[str] = typer.Option( + None, + "--docker-image", + help="Override the `docker_image` property in the template file. This is a comma separated list of: `from-yml`, `native:dev`, `native:ga`, `native:candidate`.", + ), run_hook: Optional[str] = typer.Argument(None, help="A specific hook to run"), console_log_threshold: str = typer.Option( "INFO", @@ -3733,6 +3743,8 @@ def pre_commit( verbose, show_diff_on_failure, run_docker_hooks=docker, + image_ref=image_ref, + docker_image=docker_image, dry_run=dry_run, run_hook=run_hook, pre_commit_template_path=pre_commit_template_path, diff --git a/demisto_sdk/commands/pre_commit/hooks/docker.py b/demisto_sdk/commands/pre_commit/hooks/docker.py index e7e1ac1b4b4..4b1d4d3ede8 100644 --- a/demisto_sdk/commands/pre_commit/hooks/docker.py +++ b/demisto_sdk/commands/pre_commit/hooks/docker.py @@ -56,7 +56,8 @@ def get_docker_python_path() -> str: def with_native_tags( tags_to_files: Dict[str, List[Tuple[Path, IntegrationScript]]], - docker_image_flag: str, + docker_flags: Set[str], + docker_image: Optional[str], ) -> Dict[str, List[Tuple[Path, IntegrationScript]]]: """ Adds the native image images into the dict with the files that should be run on them @@ -67,30 +68,33 @@ def with_native_tags( Returns: The updated dict with the native images. """ - docker_flags = set(docker_image_flag.split(",")) + all_tags_to_files = defaultdict(list) native_image_config = NativeImageConfig.get_instance() for image, scripts in tags_to_files.items(): for file, obj in scripts: - supported_native_images = ScriptIntegrationSupportedNativeImages( _id=obj.object_id, native_image_config=native_image_config, docker_image=image, - ).get_supported_native_docker_tags(docker_flags) + ).get_supported_native_docker_tags(docker_flags, include_candidate=True) + for native_image in supported_native_images: - all_tags_to_files[native_image].append((file, obj)) + all_tags_to_files[docker_image or native_image].append((file, obj)) if { DockerImageFlagOption.FROM_YML.value, DockerImageFlagOption.ALL_IMAGES.value, } & docker_flags: - all_tags_to_files[image].append((file, obj)) + all_tags_to_files[docker_image or image].append((file, obj)) + return all_tags_to_files def docker_tag_to_runfiles( - files_to_run: Iterable[Tuple[Path, Optional[IntegrationScript]]], docker_image_flag + files_to_run: Iterable[Tuple[Path, Optional[IntegrationScript]]], + docker_image_flag: str, + docker_image: Optional[str] = None, ) -> Dict[str, List[Tuple[Path, IntegrationScript]]]: """ Iterates over all files snf groups the files by the dockerimages @@ -101,13 +105,14 @@ def docker_tag_to_runfiles( Returns: A dict of image to List of files(Tuple[path, obj]) including native images """ + docker_flags = set(docker_image_flag.split(",")) tags_to_files = defaultdict(list) for file, obj in files_to_run: - if not obj: - continue - for docker_image in obj.docker_images: - tags_to_files[docker_image].append((file, obj)) - return with_native_tags(tags_to_files, docker_image_flag) + if obj: + for image in obj.docker_images: + tags_to_files[image].append((file, obj)) + + return with_native_tags(tags_to_files, docker_flags, docker_image) @functools.lru_cache(maxsize=512) @@ -266,7 +271,8 @@ def prepare_hook( } tag_to_files_objs = docker_tag_to_runfiles( filtered_files_with_objects, - self._get_property("docker_image", "from-yml"), + self.context.docker_image or self._get_property("docker_image", "from-yml"), + self.context.image_ref, ) end_time = time.time() logger.debug( @@ -286,7 +292,9 @@ def prepare_hook( config_arg = self._get_config_file_arg() start_time = time.time() logger.debug(f"{len(tag_to_files_objs)} images were collected from files") - logger.debug(f'collected images: {" ".join(tag_to_files_objs.keys())}') + logger.debug( + f'collected images: {" ".join(filter(None, tag_to_files_objs.keys()))}' + ) docker_hook_ids = [] with ThreadPoolExecutor(max_workers=cpu_count()) as executor: results = [] diff --git a/demisto_sdk/commands/pre_commit/pre_commit_command.py b/demisto_sdk/commands/pre_commit/pre_commit_command.py index 65e56191145..e40ac0350e6 100644 --- a/demisto_sdk/commands/pre_commit/pre_commit_command.py +++ b/demisto_sdk/commands/pre_commit/pre_commit_command.py @@ -51,7 +51,7 @@ SKIPPED_HOOKS = {"format", "validate", "secrets"} INTEGRATION_SCRIPT_REGEX = re.compile(r"^Packs/.*/(?:Integrations|Scripts)/.*.yml$") -INTEGRATIONS_BATCH = 300 +INTEGRATIONS_BATCH = 100 class PreCommitRunner: @@ -476,6 +476,8 @@ def pre_commit_manager( show_diff_on_failure: bool = False, dry_run: bool = False, run_docker_hooks: bool = True, + image_ref: Optional[str] = None, + docker_image: Optional[str] = None, run_hook: Optional[str] = None, pre_commit_template_path: Optional[Path] = None, ) -> int: @@ -494,6 +496,8 @@ def pre_commit_manager( show_diff_on_failure (bool, optional): Whether show git diff after pre-commit failure. Defaults to False. dry_run (bool, optional): Whether to run the pre-commit hooks in dry-run mode, which will only create the config file. run_docker_hooks (bool, optional): Whether to run docker based hooks or not. + image_ref: (str, optional): Override the image from YAML / native config file with this image reference. + docker_image: (str, optional): Override the `docker_image` property in the template file. This is a comma separated list of: `from-yml`, `native:dev`, `native:ga`, `native:candidate`. pre_commit_template_path (Path, optional): Path to the template pre-commit file. Returns: @@ -554,6 +558,8 @@ def pre_commit_manager( run_hook, skipped_hooks, run_docker_hooks, + image_ref, + docker_image, pre_commit_template_path=pre_commit_template_path, ) return PreCommitRunner.prepare_and_run( diff --git a/demisto_sdk/commands/pre_commit/pre_commit_context.py b/demisto_sdk/commands/pre_commit/pre_commit_context.py index 8a86d730f00..357a62ced2e 100644 --- a/demisto_sdk/commands/pre_commit/pre_commit_context.py +++ b/demisto_sdk/commands/pre_commit/pre_commit_context.py @@ -49,6 +49,8 @@ class PreCommitContext: run_hook: Optional[str] = None skipped_hooks: Set[str] = field(default_factory=set) run_docker_hooks: bool = True + image_ref: Optional[str] = None + docker_image: Optional[str] = None dry_run: bool = False pre_commit_template_path: Path = PRECOMMIT_TEMPLATE_PATH diff --git a/demisto_sdk/commands/pre_commit/tests/generic_docker_test.py b/demisto_sdk/commands/pre_commit/tests/generic_docker_test.py index 45f9fc91ba6..553732eb19f 100644 --- a/demisto_sdk/commands/pre_commit/tests/generic_docker_test.py +++ b/demisto_sdk/commands/pre_commit/tests/generic_docker_test.py @@ -267,3 +267,65 @@ def test_docker_pass_extra_args(mocker): mocker.patch.object(PreCommitContext, "dry_run", True) DockerHook(**hook).prepare_hook() assert "--rm=false" in hook["repo"]["hooks"][0]["entry"] + + +def test_image_ref_argument(mocker): + """ + Given: + - An object to run pre-commit on + + When: + - Providing the `image_ref` flag to override the image to a custom image + + Then: + - The hook will run on the provided image ref instead of the image of the YAML + + """ + file_path = Path("SomeFile.py") + files = [(file_path, Obj(object_id="id1"))] + hook = create_hook({"id": "test"}, image_ref="python3-custom-image") + mocker.patch.object( + PreCommitContext, + "files_to_run_with_objects", + files, + ) + mocker.patch( + "demisto_sdk.commands.pre_commit.hooks.docker.devtest_image", + return_value="devtestimg", + ) + DockerHook(**hook).prepare_hook() + assert hook["repo"]["hooks"][0]["id"] == "test-python3-custom-image" + + +def test_docker_image_argument(mocker): + """ + Given: + - An object to run pre-commit on + + When: + - Providing the `docker_image` to `native:candidate` and `image_ref` flag for a candidate to test a custom candidate + + Then: + - The hook will run on the provided image ref instead of the image of the native image config file + + """ + file_path = Path("SomeFile.py") + files = [ + (file_path, Obj(object_id="id1", docker_image="demisto/python3:3.10.13.89009")) + ] + hook = create_hook( + {"id": "test"}, + image_ref="python3-candidate-image", + docker_image="native:candidate", + ) + mocker.patch.object( + PreCommitContext, + "files_to_run_with_objects", + files, + ) + mocker.patch( + "demisto_sdk.commands.pre_commit.hooks.docker.devtest_image", + return_value="devtestimg", + ) + DockerHook(**hook).prepare_hook() + assert hook["repo"]["hooks"][0]["id"] == "test-python3-candidate-image" diff --git a/demisto_sdk/commands/pre_commit/tests/pre_commit_test.py b/demisto_sdk/commands/pre_commit/tests/pre_commit_test.py index b6d3da4f442..00d0d903712 100644 --- a/demisto_sdk/commands/pre_commit/tests/pre_commit_test.py +++ b/demisto_sdk/commands/pre_commit/tests/pre_commit_test.py @@ -68,6 +68,8 @@ def create_hook( mode: str = "", all_files=False, input_files: Optional[List[Path]] = None, + image_ref: Optional[str] = None, + docker_image: Optional[str] = None, ): """ This function mocks hook as he returns in _get_hooks() function @@ -75,7 +77,14 @@ def create_hook( repo_and_hook: dict = { "repo": {"repo": "repo", "hooks": [hook]}, - "context": PreCommitContext(input_files, all_files, mode, {}), + "context": PreCommitContext( + input_files, + all_files, + mode, + {}, + image_ref=image_ref, + docker_image=docker_image, + ), } repo_and_hook["hook"] = repo_and_hook["repo"]["hooks"][0] return repo_and_hook