From ce8a1a03599931325548599f0fc65556fa17d9dd Mon Sep 17 00:00:00 2001 From: roysagi <50295826+roysagi@users.noreply.github.com> Date: Sun, 4 Oct 2020 12:33:07 +0300 Subject: [PATCH] adding test playbook validation (#798) * adding test playbook validation * fixing test script * fixing test script * Update demisto_sdk/commands/common/hook_validations/test_playbook.py Co-authored-by: Bar Katzir <37335599+bakatzir@users.noreply.github.com> * fixing test script * fixing test script Co-authored-by: Bar Katzir <37335599+bakatzir@users.noreply.github.com> Co-authored-by: sberman --- CHANGELOG.md | 1 + .../common/hook_validations/test_playbook.py | 16 +++++++++ .../commands/validate/validate_manager.py | 33 ++++++++++++++----- .../TestPlaybooks/just_a_test_script.yml | 1 + .../script-prefixed_automation.yml | 1 + 5 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 demisto_sdk/commands/common/hook_validations/test_playbook.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eee0249ada..044e1ff3516 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * `demisto.params()` should be used only inside main function. * `demisto.args()` should be used only inside main function. * Functions args should have type annotations. +* Added `fromversion` field validation to test playbooks and scripts in **validate** command. # 1.2.2 * Add support for warning msgs in the report and summary to **lint** command. diff --git a/demisto_sdk/commands/common/hook_validations/test_playbook.py b/demisto_sdk/commands/common/hook_validations/test_playbook.py new file mode 100644 index 00000000000..dbde22cba97 --- /dev/null +++ b/demisto_sdk/commands/common/hook_validations/test_playbook.py @@ -0,0 +1,16 @@ +from demisto_sdk.commands.common.hook_validations.content_entity_validator import \ + ContentEntityValidator + + +class TestPlaybookValidator(ContentEntityValidator): + """TestPlaybookValidator is designed to validate the correctness of the file structure we enter to content repo for + both test playbooks and scripts. + """ + + def is_valid_file(self, validate_rn=True): + """Check whether the test playbook or script file is valid or not + """ + + return all([ + self.is_valid_fromversion(), + ]) diff --git a/demisto_sdk/commands/validate/validate_manager.py b/demisto_sdk/commands/validate/validate_manager.py index 627214e7088..64229a44bf6 100644 --- a/demisto_sdk/commands/validate/validate_manager.py +++ b/demisto_sdk/commands/validate/validate_manager.py @@ -49,6 +49,8 @@ from demisto_sdk.commands.common.hook_validations.script import ScriptValidator from demisto_sdk.commands.common.hook_validations.structure import \ StructureValidator +from demisto_sdk.commands.common.hook_validations.test_playbook import \ + TestPlaybookValidator from demisto_sdk.commands.common.hook_validations.widget import WidgetValidator from demisto_sdk.commands.common.tools import (filter_packagify_changes, find_type, get_api_module_ids, @@ -98,8 +100,6 @@ def __init__(self, is_backward_check=True, prev_ver=None, use_git=False, only_co self.new_packs = set() self.skipped_file_types = (FileType.CHANGELOG, FileType.DESCRIPTION, - FileType.TEST_PLAYBOOK, - FileType.TEST_SCRIPT, FileType.DOC_IMAGE) if is_external_repo: @@ -300,22 +300,31 @@ def run_validations_on_file(self, file_path, pack_error_ignore_list, is_modified print_as_warnings=self.print_ignored_errors, tag=self.prev_ver, old_file_path=old_file_path, branch_name=self.branch_name) - click.secho(f'Validating scheme for {file_path}') - if not structure_validator.is_valid_file(): - return False + # schema validation + if file_type not in {FileType.TEST_PLAYBOOK, FileType.TEST_SCRIPT}: + click.secho(f'Validating scheme for {file_path}') + if not structure_validator.is_valid_file(): + return False - elif self.check_only_schema: + # Passed schema validation + # if only schema validation is required - stop check here + if self.check_only_schema: return True + # id_set validation if self.validate_in_id_set: click.echo(f"Validating id set registration for {file_path}") if not self.id_set_validator.is_file_valid_in_set(file_path): return False # Note: these file are not ignored but there are no additional validators for connections - if file_type in {FileType.CONNECTION}: + if file_type == FileType.CONNECTION: return True + # test playbooks and test scripts are using the same validation. + elif file_type in {FileType.TEST_PLAYBOOK, FileType.TEST_SCRIPT}: + return self.validate_test_playbook(structure_validator, pack_error_ignore_list) + elif file_type == FileType.RELEASE_NOTES: if not self.skip_pack_rn_validation: return self.validate_release_notes(file_path, added_files, modified_files, pack_error_ignore_list, @@ -333,7 +342,7 @@ def run_validations_on_file(self, file_path, pack_error_ignore_list, is_modified elif file_type == FileType.INTEGRATION: return self.validate_integration(structure_validator, pack_error_ignore_list, is_modified) - elif file_type in (FileType.SCRIPT, FileType.TEST_SCRIPT): + elif file_type == FileType.SCRIPT: return self.validate_script(structure_validator, pack_error_ignore_list, is_modified) elif file_type == FileType.BETA_INTEGRATION: @@ -343,7 +352,7 @@ def run_validations_on_file(self, file_path, pack_error_ignore_list, is_modified elif file_type == FileType.IMAGE: return self.validate_image(file_path, pack_error_ignore_list) - # incident fields and indicator fields are using the same scheme. + # incident fields and indicator fields are using the same validation. elif file_type in (FileType.INCIDENT_FIELD, FileType.INDICATOR_FIELD): return self.validate_incident_field(structure_validator, pack_error_ignore_list, is_modified) @@ -424,6 +433,12 @@ def validate_readme(self, file_path, pack_error_ignore_list): print_as_warnings=self.print_ignored_errors) return readme_validator.is_valid_file() + def validate_test_playbook(self, structure_validator, pack_error_ignore_list): + test_playbook_validator = TestPlaybookValidator(structure_validator=structure_validator, + ignored_errors=pack_error_ignore_list, + print_as_warnings=self.print_ignored_errors) + return test_playbook_validator.is_valid_file(validate_rn=False) + def validate_release_notes(self, file_path, added_files, modified_files, pack_error_ignore_list, is_modified): pack_name = get_pack_name(file_path) diff --git a/demisto_sdk/tests/test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/just_a_test_script.yml b/demisto_sdk/tests/test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/just_a_test_script.yml index a5fc3117f7d..e60deb76af5 100644 --- a/demisto_sdk/tests/test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/just_a_test_script.yml +++ b/demisto_sdk/tests/test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/just_a_test_script.yml @@ -15,5 +15,6 @@ args: description: Amount of seconds to sleep scripttarget: 0 timeout: 1h40m0s +fromversion: 5.5.0 tests: - No tests diff --git a/demisto_sdk/tests/test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/script-prefixed_automation.yml b/demisto_sdk/tests/test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/script-prefixed_automation.yml index 4d577caea13..565c93dc475 100644 --- a/demisto_sdk/tests/test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/script-prefixed_automation.yml +++ b/demisto_sdk/tests/test_files/content_repo_example/Packs/FeedAzureValid/TestPlaybooks/script-prefixed_automation.yml @@ -15,5 +15,6 @@ args: description: Amount of seconds to sleep scripttarget: 0 timeout: 1h40m0s +fromversion: 5.0.0 tests: - No tests