diff --git a/demisto_sdk/__main__.py b/demisto_sdk/__main__.py index e6f65aaa931..5c8fbe10042 100644 --- a/demisto_sdk/__main__.py +++ b/demisto_sdk/__main__.py @@ -348,12 +348,12 @@ def secrets(config, **kwargs): @click.option("--no-pwsh-test", is_flag=True, help="Do NOT run powershell test") @click.option("-kc", "--keep-container", is_flag=True, help="Keep the test container") @click.option("--test-xml", help="Path to store pytest xml results", type=click.Path(exists=True, resolve_path=True)) -@click.option("--json-report", help="Path to store json results", type=click.Path(exists=True, resolve_path=True)) +@click.option("--failure-report", help="Path to store failed packs tests report", type=click.Path(exists=True, resolve_path=True)) @click.option("-lp", "--log-path", help="Path to store all levels of logs", type=click.Path(exists=True, resolve_path=True)) def lint(input: str, git: bool, all_packs: bool, verbose: int, quiet: bool, parallel: int, no_flake8: bool, no_bandit: bool, no_mypy: bool, no_vulture: bool, no_pylint: bool, no_test: bool, no_pwsh_analyze: bool, - no_pwsh_test: bool, keep_container: bool, test_xml: str, json_report: str, log_path: str): + no_pwsh_test: bool, keep_container: bool, test_xml: str, failure_report: str, log_path: str): """Lint command will perform:\n 1. Package in host checks - flake8, bandit, mypy, vulture.\n 2. Package in docker image checks - pylint, pytest, powershell - test, powershell - analyze.\n @@ -376,7 +376,7 @@ def lint(input: str, git: bool, all_packs: bool, verbose: int, quiet: bool, para no_pwsh_test=no_pwsh_test, keep_container=keep_container, test_xml=test_xml, - json_report=json_report) + failure_report=failure_report) # ====================== format ====================== # diff --git a/demisto_sdk/commands/lint/README.md b/demisto_sdk/commands/lint/README.md index 76b8fb1a875..a6ee44316be 100644 --- a/demisto_sdk/commands/lint/README.md +++ b/demisto_sdk/commands/lint/README.md @@ -46,8 +46,8 @@ Options: Keep the test container * **--test-xml PATH** Path to store pytest xml results -* **--json-report PATH** - Path to store json results +* **--failure-report** + Path to store failed unit tests report * **-lp, --log-path PATH** Path to store all levels of logs diff --git a/demisto_sdk/commands/lint/lint_manager.py b/demisto_sdk/commands/lint/lint_manager.py index 38061594e81..cf39e810882 100644 --- a/demisto_sdk/commands/lint/lint_manager.py +++ b/demisto_sdk/commands/lint/lint_manager.py @@ -107,7 +107,8 @@ def _gather_facts() -> Dict[str, Any]: facts["test_modules"] = get_test_modules(content_repo=facts["content_repo"]) logger.debug(f"Test mandatory modules successfully collected") except git.GitCommandError as e: - print_error("Unable to get test-modules demisto-mock.py etc - Aborting! corrupt repository of pull from master") + print_error( + "Unable to get test-modules demisto-mock.py etc - Aborting! corrupt repository of pull from master") logger.error(f"demisto-sdk-unable to get mandatory test-modules demisto-mock.py etc {e}") sys.exit(1) except (requests.exceptions.ConnectionError, urllib3.exceptions.NewConnectionError) as e: @@ -203,8 +204,9 @@ def _filter_changed_packages(content_repo: git.Repo, pkgs: List[Path]) -> List[P return list(pkgs_to_check) def run_dev_packages(self, parallel: int, no_flake8: bool, no_bandit: bool, no_mypy: bool, no_pylint: bool, - no_vulture: bool, no_test: bool, no_pwsh_analyze: bool, no_pwsh_test: bool, keep_container: bool, - test_xml: str, json_report: str) -> int: + no_vulture: bool, no_test: bool, no_pwsh_analyze: bool, no_pwsh_test: bool, + keep_container: bool, + test_xml: str, failure_report: str) -> int: """ Runs the Lint command on all given packages. Args: @@ -219,21 +221,21 @@ def run_dev_packages(self, parallel: int, no_flake8: bool, no_bandit: bool, no_m no_pwsh_test(bool): whether to skip powershell tests keep_container(bool): Whether to keep the test container test_xml(str): Path for saving pytest xml results - json_report(str): Path for store json report + failure_report(str): Path for store failed unit tests report Returns: int: exit code by falil exit codes by var EXIT_CODES """ lint_status = { - "fail_packs_flake8": [], - "fail_packs_bandit": [], - "fail_packs_mypy": [], - "fail_packs_vulture": [], - "fail_packs_pylint": [], - "fail_packs_pytest": [], - "fail_packs_pwsh_analyze": [], - "fail_packs_pwsh_test": [], - "fail_packs_image": [], + "fail_packs_flake8": set(), + "fail_packs_bandit": set(), + "fail_packs_mypy": set(), + "fail_packs_vulture": set(), + "fail_packs_pylint": set(), + "fail_packs_pytest": set(), + "fail_packs_pwsh_analyze": set(), + "fail_packs_pwsh_test": set(), + "fail_packs_image": set() } # Python or powershell or both @@ -303,12 +305,12 @@ def run_dev_packages(self, parallel: int, no_flake8: bool, no_bandit: bool, no_m return_exit_code=return_exit_code, skipped_code=skipped_code, pkgs_type=pkgs_type) - self._create_report(pkgs_status=pkgs_status, - path=json_report) + self._create_failed_unit_tests_report(lint_status=lint_status, path=failure_report) return return_exit_code - def _report_results(self, lint_status: dict, pkgs_status: dict, return_exit_code: int, skipped_code: int, pkgs_type: list): + def _report_results(self, lint_status: dict, pkgs_status: dict, return_exit_code: int, skipped_code: int, + pkgs_type: list): """ Log report to console Args: @@ -547,9 +549,26 @@ def report_summary(self, lint_status: dict): print(f"{Colors.Fg.red}{wrapper_fail_pack.fill(fail_pack)}{Colors.reset}") @staticmethod - def _create_report(pkgs_status: dict, path: str): - if path: - json_path = Path(path) / "lint_report.json" - json.dump(fp=json_path.open(mode='w'), - obj=pkgs_status, - indent=4) + def _create_failed_unit_tests_report(lint_status: dict, path: str): + """ + Creates and saves a file containing all failed unit tests + :param lint_status: dict + Dictionary containing type of failures and corresponding failing tests. Looks like this: + lint_status = { + "fail_packs_flake8": [], + "fail_packs_bandit": [], + "fail_packs_mypy": [], + "fail_packs_vulture": [], + "fail_packs_pylint": [], + "fail_packs_pytest": [], + "fail_packs_pwsh_analyze": [], + "fail_packs_pwsh_test": [], + "fail_packs_image": [] + } + :param path: str + The path to save the report. + """ + failed_ut = set.union(*lint_status.values()) + if path and failed_ut: + file_path = Path(path) / "failed_lint_report.txt" + file_path.write_text('\n'.join(failed_ut)) diff --git a/demisto_sdk/commands/lint/tests/linter_manager_test.py b/demisto_sdk/commands/lint/tests/linter_manager_test.py index 66a4ad031d8..3c065007bac 100644 --- a/demisto_sdk/commands/lint/tests/linter_manager_test.py +++ b/demisto_sdk/commands/lint/tests/linter_manager_test.py @@ -1,7 +1,9 @@ +import os from unittest.mock import MagicMock, patch import pytest from demisto_sdk.commands.common.constants import TYPE_PWSH, TYPE_PYTHON +from demisto_sdk.commands.common.git_tools import git_path @patch('builtins.print') @@ -29,3 +31,70 @@ def test_report_failed_image_creation(): pkgs_status=pkgs_status, return_exit_code=EXIT_CODES["image"]) assert not pkgs_status.called + + +def test_create_failed_unit_tests_report_with_failed_tests(): + """ + Given`: + - Lint manager dictionary with two failed packs -Infoblox and HelloWorld + + When: + - Creating failed unit tests report + + Then: + - Ensure report file is created. + - Ensure report file contains exactly two packs. + - Ensure both pack appear in the report. + """ + from demisto_sdk.commands.lint import lint_manager + lint_status = { + "fail_packs_flake8": set(), + "fail_packs_bandit": set(), + "fail_packs_mypy": {'Infoblox'}, + "fail_packs_vulture": set(), + "fail_packs_pylint": {'HelloWorld'}, + "fail_packs_pytest": {'Infoblox'}, + "fail_packs_pwsh_analyze": set(), + "fail_packs_pwsh_test": set(), + "fail_packs_image": set() + } + path = f'{git_path()}/demisto_sdk/commands/lint/tests' + lint_manager.LintManager._create_failed_unit_tests_report(lint_status, path) + file_path = f'{path}/failed_lint_report.txt' + assert os.path.isfile(file_path) + with open(file_path, 'r') as file: + content = file.read() + fail_list = content.split('\n') + assert len(fail_list) == 2 + assert 'HelloWorld' in fail_list + assert 'Infoblox' in fail_list + os.remove(file_path) + + +def test_create_failed_unit_tests_report_no_failed_tests(): + """ + Given: + - Lint manager dictionary with no failed packs. + + When: + - Creating failed unit tests report. + + Then: + - Ensure report file is not created. + """ + from demisto_sdk.commands.lint import lint_manager + lint_status = { + "fail_packs_flake8": set(), + "fail_packs_bandit": set(), + "fail_packs_mypy": set(), + "fail_packs_vulture": set(), + "fail_packs_pylint": set(), + "fail_packs_pytest": set(), + "fail_packs_pwsh_analyze": set(), + "fail_packs_pwsh_test": set(), + "fail_packs_image": set(), + } + path = f'{git_path()}/demisto_sdk/commands/lint/tests' + lint_manager.LintManager._create_failed_unit_tests_report(lint_status, path) + file_path = f'{path}/failed_lint_report.txt' + assert not os.path.isfile(file_path)