adding test playbook validation (#798)

* adding test playbook validation * fixing test script * fixing test script * Update demisto_sdk/commands/common/hook_validations/test_playbook.py Co-authored-by: Bar Katzir <37335599+bakatzir@users.noreply.github.com> * fixing test script * fixing test script Co-authored-by: Bar Katzir <37335599+bakatzir@users.noreply.github.com> Co-authored-by: sberman <sberman@paloaltonetworks.com>
demisto · Oct 4, 2020 · ce8a1a0 · ce8a1a0
1 parent 7993630
commit ce8a1a0
Show file tree

Hide file tree

Showing 5 changed files with 43 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,7 @@
         * `demisto.params()` should be used only inside main function.
         * `demisto.args()` should be used only inside main function.
         * Functions args should have type annotations.
+* Added `fromversion` field validation to test playbooks and scripts in **validate** command.
 
 # 1.2.2
 * Add support for warning msgs in the report and summary to **lint** command.

diff --git a/demisto_sdk/commands/common/hook_validations/test_playbook.py b/demisto_sdk/commands/common/hook_validations/test_playbook.py
@@ -0,0 +1,16 @@
+from demisto_sdk.commands.common.hook_validations.content_entity_validator import \
+    ContentEntityValidator
+
+
+class TestPlaybookValidator(ContentEntityValidator):
+    """TestPlaybookValidator is designed to validate the correctness of the file structure we enter to content repo for
+    both test playbooks and scripts.
+    """
+
+    def is_valid_file(self, validate_rn=True):
+        """Check whether the test playbook or script file is valid or not
+        """
+
+        return all([
+            self.is_valid_fromversion(),
+        ])
diff --git a/demisto_sdk/commands/validate/validate_manager.py b/demisto_sdk/commands/validate/validate_manager.py
@@ -49,6 +49,8 @@
 from demisto_sdk.commands.common.hook_validations.script import ScriptValidator
 from demisto_sdk.commands.common.hook_validations.structure import \
     StructureValidator
+from demisto_sdk.commands.common.hook_validations.test_playbook import \
+    TestPlaybookValidator
 from demisto_sdk.commands.common.hook_validations.widget import WidgetValidator
 from demisto_sdk.commands.common.tools import (filter_packagify_changes,
                                                find_type, get_api_module_ids,
@@ -98,8 +100,6 @@ def __init__(self, is_backward_check=True, prev_ver=None, use_git=False, only_co
         self.new_packs = set()
         self.skipped_file_types = (FileType.CHANGELOG,
                                    FileType.DESCRIPTION,
-                                   FileType.TEST_PLAYBOOK,
-                                   FileType.TEST_SCRIPT,
                                    FileType.DOC_IMAGE)
 
         if is_external_repo:
@@ -300,22 +300,31 @@ def run_validations_on_file(self, file_path, pack_error_ignore_list, is_modified
                                                  print_as_warnings=self.print_ignored_errors, tag=self.prev_ver,
                                                  old_file_path=old_file_path, branch_name=self.branch_name)
 
-        click.secho(f'Validating scheme for {file_path}')
-        if not structure_validator.is_valid_file():
-            return False
+        # schema validation
+        if file_type not in {FileType.TEST_PLAYBOOK, FileType.TEST_SCRIPT}:
+            click.secho(f'Validating scheme for {file_path}')
+            if not structure_validator.is_valid_file():
+                return False
 
-        elif self.check_only_schema:
+        # Passed schema validation
+        # if only schema validation is required - stop check here
+        if self.check_only_schema:
             return True
 
+        # id_set validation
         if self.validate_in_id_set:
             click.echo(f"Validating id set registration for {file_path}")
             if not self.id_set_validator.is_file_valid_in_set(file_path):
                 return False
 
         # Note: these file are not ignored but there are no additional validators for connections
-        if file_type in {FileType.CONNECTION}:
+        if file_type == FileType.CONNECTION:
             return True
 
+        # test playbooks and test scripts are using the same validation.
+        elif file_type in {FileType.TEST_PLAYBOOK, FileType.TEST_SCRIPT}:
+            return self.validate_test_playbook(structure_validator, pack_error_ignore_list)
+
         elif file_type == FileType.RELEASE_NOTES:
             if not self.skip_pack_rn_validation:
                 return self.validate_release_notes(file_path, added_files, modified_files, pack_error_ignore_list,
@@ -333,7 +342,7 @@ def run_validations_on_file(self, file_path, pack_error_ignore_list, is_modified
         elif file_type == FileType.INTEGRATION:
             return self.validate_integration(structure_validator, pack_error_ignore_list, is_modified)
 
-        elif file_type in (FileType.SCRIPT, FileType.TEST_SCRIPT):
+        elif file_type == FileType.SCRIPT:
             return self.validate_script(structure_validator, pack_error_ignore_list, is_modified)
 
         elif file_type == FileType.BETA_INTEGRATION:
@@ -343,7 +352,7 @@ def run_validations_on_file(self, file_path, pack_error_ignore_list, is_modified
         elif file_type == FileType.IMAGE:
             return self.validate_image(file_path, pack_error_ignore_list)
 
-        # incident fields and indicator fields are using the same scheme.
+        # incident fields and indicator fields are using the same validation.
         elif file_type in (FileType.INCIDENT_FIELD, FileType.INDICATOR_FIELD):
             return self.validate_incident_field(structure_validator, pack_error_ignore_list, is_modified)
 
@@ -424,6 +433,12 @@ def validate_readme(self, file_path, pack_error_ignore_list):
                                            print_as_warnings=self.print_ignored_errors)
         return readme_validator.is_valid_file()
 
+    def validate_test_playbook(self, structure_validator, pack_error_ignore_list):
+        test_playbook_validator = TestPlaybookValidator(structure_validator=structure_validator,
+                                                        ignored_errors=pack_error_ignore_list,
+                                                        print_as_warnings=self.print_ignored_errors)
+        return test_playbook_validator.is_valid_file(validate_rn=False)
+
     def validate_release_notes(self, file_path, added_files, modified_files, pack_error_ignore_list, is_modified):
         pack_name = get_pack_name(file_path)
 

diff --git a/...test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/just_a_test_script.yml b/...test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/just_a_test_script.yml
@@ -15,5 +15,6 @@ args:
   description: Amount of seconds to sleep
 scripttarget: 0
 timeout: 1h40m0s
+fromversion: 5.5.0
 tests:
   - No tests
diff --git a/...es/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/script-prefixed_automation.yml b/...es/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/script-prefixed_automation.yml
@@ -15,5 +15,6 @@ args:
   description: Amount of seconds to sleep
 scripttarget: 0
 timeout: 1h40m0s
+fromversion: 5.0.0
 tests:
   - No tests