format: support for updating docker image (#720)

* format: update docker support

* update readme

* update validate note about option to run update docker

* ignore update_docker for non code formatters

* add changelog

* Update CHANGELOG.md

Co-authored-by: roysagi <50295826+roysagi@users.noreply.github.com>

* Update demisto_sdk/commands/format/README.md

Co-authored-by: roysagi <50295826+roysagi@users.noreply.github.com>

Co-authored-by: roysagi <50295826+roysagi@users.noreply.github.com>

CHANGELOG.md

@@ -13,6 +13,7 @@
 * Fixed an issue where **update-release-notes** would fail to bump new versions if the feature branch was out of sync with the master branch.
 * Fixed an issue where a non-descriptive error would be returned when giving the **update-release-notes** command a pack which can not be found.
 * Added dependencies check for *widgets* in **find-dependencies** command.
+* Added a `update-docker` flag to **format** command.
 * Added a `json-to-outputs` flag to the **run** command.
 * Added a verbose (`-v`) flag to **format** command.
 * Fixed an issue where **download** added the prefix "playbook-" to the name of playbooks.

demisto_sdk/__main__.py

@@ -447,10 +447,12 @@ def lint(input: str, git: bool, all_packs: bool, verbose: int, quiet: bool, para
     "-fv", "--from-version", help="Specify fromversion of the pack")
 @click.option(
     "-nv", "--no-validate", help="Set when validate on file is not wanted", is_flag=True)
+@click.option(
+    "-ud", "--update-docker", help="Set if you want to update the docker image of the integration/script", is_flag=True)
 @click.option(
     "-v", "--verbose", help="Verbose output", is_flag=True)
-def format_yml(input=None, output=None, from_version=None, no_validate=None, verbose=False):
-    return format_manager(input, output, from_version, no_validate, verbose)
+def format_yml(**kwargs):
+    return format_manager(**kwargs)
 
 
 # ====================== upload ====================== #

demisto_sdk/commands/common/errors.py

@@ -454,12 +454,13 @@ def docker_not_formatted_correctly(docker_image):
 
     @staticmethod
     @error_code_decorator
-    def docker_not_on_the_latest_tag(docker_image_tag, docker_image_latest_tag, docker_image_name):
+    def docker_not_on_the_latest_tag(docker_image_tag, docker_image_latest_tag, docker_image_name, file_path):
         return f'The docker image tag is not the latest numeric tag, please update it.\n' \
                f'The docker image tag in the yml file is: {docker_image_tag}\n' \
                f'The latest docker image tag in docker hub is: {docker_image_latest_tag}\n' \
                f'You can check for the most updated version of {docker_image_name} ' \
-               f'here: https://hub.docker.com/r/{docker_image_name}/tags\n'
+               f'here: https://hub.docker.com/r/{docker_image_name}/tags\n' \
+               f'To update the docker image run: demisto-sdk format -ud -i {file_path}\n'
 
     @staticmethod
     @error_code_decorator

demisto_sdk/commands/common/hook_validations/docker.py

@@ -79,7 +79,8 @@ def is_docker_image_latest_tag(self):
             # If docker image tag is not the most updated one that exists in docker-hub
             error_message, error_code = Errors.docker_not_on_the_latest_tag(self.docker_image_tag,
                                                                             self.docker_image_latest_tag,
-                                                                            self.docker_image_name)
+                                                                            self.docker_image_name,
+                                                                            self.file_path)
             if self.handle_error(error_message, error_code, file_path=self.file_path):
                 self.is_latest_tag = False
 

demisto_sdk/commands/format/README.md

@@ -27,6 +27,10 @@ When done formatting, the **validate** command will run, to let you know of thin
 
    When no validation on file is needed.
 
+* **-ud ,--update-docker**
+
+   Set if you want to update the docker image of the integration/script to the newest available tag.
+
 ### Examples
 ```
 demisto-sdk format

demisto_sdk/commands/format/format_module.py

@@ -63,8 +63,8 @@
 VALIDATE_RES_FAILED_CODE = 3
 
 
-def format_manager(input: str = None, output: str = None, from_version: str = '', no_validate: bool = None,
-                   verbose: bool = False):
+def format_manager(input: str = None, output: str = None, from_version: str = '', no_validate: bool = False,
+                   verbose: bool = False, update_docker: bool = False):
     """
     Format_manager is a function that activated format command on different type of files.
     Args:
@@ -73,6 +73,7 @@ def format_manager(input: str = None, output: str = None, from_version: str = ''
         output: (str) The path to save the formatted file to.
         no_validate (flag): Whether the user specifies not to run validate after format.
         verbose (bool): Whether to print verbose logs or not
+        update_docker (flag): Whether to update the docker image.
     Returns:
         int 0 in case of success 1 otherwise
     """
@@ -95,7 +96,7 @@ def format_manager(input: str = None, output: str = None, from_version: str = ''
                 file_type = file_type.value
                 info_res, err_res, skip_res = run_format_on_file(input=file_path, file_type=file_type,
                                                                  from_version=from_version, output=output,
-                                                                 no_validate=no_validate, verbose=verbose)
+                                                                 no_validate=no_validate, verbose=verbose, update_docker=update_docker)
                 if err_res:
                     error_list.append("err_res")
                 if err_res:
@@ -130,6 +131,9 @@ def run_format_on_file(input: str, file_type: str, from_version: str, **kwargs)
     """
     schema_path = os.path.normpath(
         os.path.join(__file__, "..", "..", "common", SCHEMAS_PATH, '{}.yml'.format(file_type)))
+    if file_type not in ('integration', 'script') and 'update_docker' in kwargs:
+        # non code formatters don't support update_docker param. remove it
+        del kwargs['update_docker']
     UpdateObject = FILE_TYPE_AND_LINKED_CLASS[file_type](input=input, path=schema_path,
                                                          from_version=from_version,
                                                          **kwargs)

demisto_sdk/commands/format/tests/test_formatting_yml_test.py

@@ -6,6 +6,8 @@
 import yaml
 from demisto_sdk.commands.common.constants import (FEED_REQUIRED_PARAMS,
                                                    FETCH_REQUIRED_PARAMS)
+from demisto_sdk.commands.common.hook_validations.docker import \
+    DockerImageValidator
 from demisto_sdk.commands.format.format_module import format_manager
 from demisto_sdk.commands.format.update_integration import IntegrationYMLFormat
 from demisto_sdk.commands.format.update_playbook import (PlaybookYMLFormat,
@@ -440,3 +442,35 @@ def test_run_format_on_tpb():
     assert formatter.data.get('fromversion') == '5.0.0'
     os.remove(DESTINATION_FORMAT_TEST_PLAYBOOK)
     os.rmdir(TEST_PLAYBOOK_PATH)
+
+
+def test_update_docker_format(tmpdir, mocker):
+    """Test that script and integration formatter update docker image tag
+    """
+    test_tag = '1.0.0-test-tag'
+    mocker.patch.object(DockerImageValidator, 'get_docker_image_latest_tag_request', return_value=test_tag)
+    schema_dir = f'{GIT_ROOT}/demisto_sdk/commands/common/schemas'
+    test_files_dir = f'{GIT_ROOT}/demisto_sdk/tests/test_files/update-docker'
+    dest = str(tmpdir.join('docker-res.yml'))
+
+    # test example script file with version before 5.0.0
+    src_file = f'{test_files_dir}/SlackAsk.yml'
+    with open(src_file) as f:
+        data = yaml.safe_load(f)
+    org_docker = data['dockerimage']
+    assert data['fromversion'] < '5.0.0'
+    assert not data.get('dockerimage45')  # make sure for the test that dockerimage45 is not set (so we can verify that we set it in format)
+    format_obj = ScriptYMLFormat(src_file, output=dest, path=f'{schema_dir}/script.yml', no_validate=True, update_docker=True)
+    assert format_obj.run_format() == 0
+    with open(dest) as f:
+        data = yaml.safe_load(f)
+    assert data['dockerimage'].endswith(f':{test_tag}')
+    assert data['dockerimage45'] == org_docker
+
+    # test integration file
+    src_file = f'{test_files_dir}/Slack.yml'
+    format_obj = IntegrationYMLFormat(src_file, output=dest, path=f'{schema_dir}/integration.yml', no_validate=True, update_docker=True)
+    assert format_obj.run_format() == 0
+    with open(dest) as f:
+        data = yaml.safe_load(f)
+    assert data['script']['dockerimage'].endswith(f':{test_tag}')

demisto_sdk/commands/format/update_integration.py

@@ -11,6 +11,7 @@
                                                           SKIP_RETURN_CODE,
                                                           SUCCESS_RETURN_CODE)
 from demisto_sdk.commands.format.update_generic_yml import BaseUpdateYML
+from demisto_sdk.commands.format.update_script import ScriptYMLFormat
 
 
 class IntegrationYMLFormat(BaseUpdateYML):
@@ -27,9 +28,9 @@ class IntegrationYMLFormat(BaseUpdateYML):
     }
 
     def __init__(self, input: str = '', output: str = '', path: str = '', from_version: str = '',
-                 no_validate: bool = False, verbose: bool = False):
-        super().__init__(input=input, output=output, path=path, from_version=from_version, no_validate=no_validate,
-                         verbose=verbose)
+                 no_validate: bool = False, verbose: bool = False, update_docker: bool = False):
+        super().__init__(input, output, path, from_version, no_validate, verbose=verbose)
+        self.update_docker = update_docker
         if not from_version and self.data.get("script", {}).get("type") == TYPE_PWSH:
             self.from_version = '5.5.0'
 
@@ -112,6 +113,10 @@ def set_feed_params_in_config(self):
                 if param not in params:
                     self.data['configuration'].append(param)
 
+    def update_docker_image(self):
+        if self.update_docker:
+            ScriptYMLFormat.update_docker_image_in_script(self.data['script'], self.data.get(self.from_version_key))
+
     def run_format(self) -> int:
         try:
             click.secho(f'\n======= Updating file: {self.source_file} =======', fg='white')
@@ -122,6 +127,7 @@ def run_format(self) -> int:
             self.set_reputation_commands_basic_argument_as_needed()
             self.set_fetch_params_in_config()
             self.set_feed_params_in_config()
+            self.update_docker_image()
             self.save_yml_to_destination_file()
             return SUCCESS_RETURN_CODE
         except Exception:

demisto_sdk/commands/format/update_script.py

@@ -1,8 +1,12 @@
-from typing import Tuple
+from typing import Optional, Tuple
 
 import click
-from demisto_sdk.commands.common.constants import TYPE_PWSH
+from demisto_sdk.commands.common.constants import TYPE_JS, TYPE_PWSH
+from demisto_sdk.commands.common.hook_validations.docker import \
+    DockerImageValidator
 from demisto_sdk.commands.common.hook_validations.script import ScriptValidator
+from demisto_sdk.commands.common.tools import (LOG_COLORS, print_color,
+                                               server_version_compare)
 from demisto_sdk.commands.format.format_constants import (ERROR_RETURN_CODE,
                                                           SKIP_RETURN_CODE,
                                                           SUCCESS_RETURN_CODE)
@@ -17,18 +21,52 @@ class ScriptYMLFormat(BaseUpdateYML):
             output (str): the desired file name to save the updated version of the YML to.
     """
 
-    def __init__(self, input: str = '', output: str = '', path: str = '', from_version: str = '',
-                 no_validate: bool = False, verbose: bool = False):
-        super().__init__(input=input, output=output, path=path, from_version=from_version, no_validate=no_validate,
-                         verbose=verbose)
+    def __init__(self, input: str = '', output: str = '', path: str = '', from_version: str = '', no_validate: bool = False,
+                 update_docker: bool = False, verbose: bool = False):
+        super().__init__(input, output, path, from_version, no_validate, verbose=verbose)
+        self.update_docker = update_docker
         if not from_version and self.data.get("type") == TYPE_PWSH:
             self.from_version = '5.5.0'
 
+    @staticmethod
+    def update_docker_image_in_script(script_obj: dict, from_version: Optional[str] = None):
+        """Update the docker image for the passed script object. Will ignore if this is a javascript
+        object or using default image (not set).
+
+        Args:
+            script_obj (dict): script object
+        """
+        if script_obj.get('type') == TYPE_JS:
+            print_color('Skipping docker image update as this is a Javascript automation.', LOG_COLORS.YELLOW)
+            return
+        dockerimage = script_obj.get('dockerimage')
+        if not dockerimage:  # default image -> nothing to do
+            print_color('Skipping docker image update as default docker image is being used.', LOG_COLORS.YELLOW)
+            return
+        image_name = dockerimage.split(':')[0]
+        latest_tag = DockerImageValidator.get_docker_image_latest_tag_request(image_name)
+        full_name = f'{image_name}:{latest_tag}'
+        if full_name != dockerimage:
+            print(f'Updating docker image to: {full_name}')
+            script_obj['dockerimage'] = full_name
+            if (not from_version) or server_version_compare('5.0.0', from_version):
+                # if this is a script that supports 4.5 and earlier. Make sure dockerimage45 is set
+                if not script_obj.get('dockerimage45'):
+                    print(f'Setting dockerimage45 to previous image value: {dockerimage} for 4.5 and earlier support')
+                    script_obj['dockerimage45'] = dockerimage
+        else:
+            print(f'Already using latest docker image: {dockerimage}. Nothing to update.')
+
+    def update_docker_image(self):
+        if self.update_docker:
+            self.update_docker_image_in_script(self.data, self.data.get(self.from_version_key))
+
     def run_format(self) -> int:
         try:
             click.secho(f'\n======= Updating file: {self.source_file} =======', fg='white')
             super().update_yml()
             self.update_tests()
+            self.update_docker_image()
             self.save_yml_to_destination_file()
             return SUCCESS_RETURN_CODE
         except Exception: