Merge pull request #412 from demisto/lint_manager-unitest-report

Fixed failures report corresponding to the needed failing unit test file structure

demisto_sdk/__main__.py

@@ -348,12 +348,12 @@ def secrets(config, **kwargs):
 @click.option("--no-pwsh-test", is_flag=True, help="Do NOT run powershell test")
 @click.option("-kc", "--keep-container", is_flag=True, help="Keep the test container")
 @click.option("--test-xml", help="Path to store pytest xml results", type=click.Path(exists=True, resolve_path=True))
-@click.option("--json-report", help="Path to store json results", type=click.Path(exists=True, resolve_path=True))
+@click.option("--failure-report", help="Path to store failed packs tests report", type=click.Path(exists=True, resolve_path=True))
 @click.option("-lp", "--log-path", help="Path to store all levels of logs",
               type=click.Path(exists=True, resolve_path=True))
 def lint(input: str, git: bool, all_packs: bool, verbose: int, quiet: bool, parallel: int, no_flake8: bool,
          no_bandit: bool, no_mypy: bool, no_vulture: bool, no_pylint: bool, no_test: bool, no_pwsh_analyze: bool,
-         no_pwsh_test: bool, keep_container: bool, test_xml: str, json_report: str, log_path: str):
+         no_pwsh_test: bool, keep_container: bool, test_xml: str, failure_report: str, log_path: str):
     """Lint command will perform:\n
         1. Package in host checks - flake8, bandit, mypy, vulture.\n
         2. Package in docker image checks -  pylint, pytest, powershell - test, powershell - analyze.\n
@@ -376,7 +376,7 @@ def lint(input: str, git: bool, all_packs: bool, verbose: int, quiet: bool, para
                                          no_pwsh_test=no_pwsh_test,
                                          keep_container=keep_container,
                                          test_xml=test_xml,
-                                         json_report=json_report)
+                                         failure_report=failure_report)
 
 
 # ====================== format ====================== #

demisto_sdk/commands/lint/README.md

@@ -46,8 +46,8 @@ Options:
     Keep the test container
 *  **--test-xml PATH**
     Path to store pytest xml results
-*  **--json-report PATH**
-    Path to store json results
+*  **--failure-report**
+    Path to store failed unit tests report
 *  **-lp, --log-path PATH**
     Path to store all levels of logs
 

demisto_sdk/commands/lint/lint_manager.py

@@ -107,7 +107,8 @@ def _gather_facts() -> Dict[str, Any]:
             facts["test_modules"] = get_test_modules(content_repo=facts["content_repo"])
             logger.debug(f"Test mandatory modules successfully collected")
         except git.GitCommandError as e:
-            print_error("Unable to get test-modules demisto-mock.py etc - Aborting! corrupt repository of pull from master")
+            print_error(
+                "Unable to get test-modules demisto-mock.py etc - Aborting! corrupt repository of pull from master")
             logger.error(f"demisto-sdk-unable to get mandatory test-modules demisto-mock.py etc {e}")
             sys.exit(1)
         except (requests.exceptions.ConnectionError, urllib3.exceptions.NewConnectionError) as e:
@@ -203,8 +204,9 @@ def _filter_changed_packages(content_repo: git.Repo, pkgs: List[Path]) -> List[P
         return list(pkgs_to_check)
 
     def run_dev_packages(self, parallel: int, no_flake8: bool, no_bandit: bool, no_mypy: bool, no_pylint: bool,
-                         no_vulture: bool, no_test: bool, no_pwsh_analyze: bool, no_pwsh_test: bool, keep_container: bool,
-                         test_xml: str, json_report: str) -> int:
+                         no_vulture: bool, no_test: bool, no_pwsh_analyze: bool, no_pwsh_test: bool,
+                         keep_container: bool,
+                         test_xml: str, failure_report: str) -> int:
         """ Runs the Lint command on all given packages.
 
         Args:
@@ -219,21 +221,21 @@ def run_dev_packages(self, parallel: int, no_flake8: bool, no_bandit: bool, no_m
             no_pwsh_test(bool): whether to skip powershell tests
             keep_container(bool): Whether to keep the test container
             test_xml(str): Path for saving pytest xml results
-            json_report(str): Path for store json report
+            failure_report(str): Path for store failed unit tests report
 
         Returns:
             int: exit code by falil exit codes by var EXIT_CODES
         """
         lint_status = {
-            "fail_packs_flake8": [],
-            "fail_packs_bandit": [],
-            "fail_packs_mypy": [],
-            "fail_packs_vulture": [],
-            "fail_packs_pylint": [],
-            "fail_packs_pytest": [],
-            "fail_packs_pwsh_analyze": [],
-            "fail_packs_pwsh_test": [],
-            "fail_packs_image": [],
+            "fail_packs_flake8": set(),
+            "fail_packs_bandit": set(),
+            "fail_packs_mypy": set(),
+            "fail_packs_vulture": set(),
+            "fail_packs_pylint": set(),
+            "fail_packs_pytest": set(),
+            "fail_packs_pwsh_analyze": set(),
+            "fail_packs_pwsh_test": set(),
+            "fail_packs_image": set()
         }
 
         # Python or powershell or both
@@ -303,12 +305,12 @@ def run_dev_packages(self, parallel: int, no_flake8: bool, no_bandit: bool, no_m
                              return_exit_code=return_exit_code,
                              skipped_code=skipped_code,
                              pkgs_type=pkgs_type)
-        self._create_report(pkgs_status=pkgs_status,
-                            path=json_report)
+        self._create_failed_unit_tests_report(lint_status=lint_status, path=failure_report)
 
         return return_exit_code
 
-    def _report_results(self, lint_status: dict, pkgs_status: dict, return_exit_code: int, skipped_code: int, pkgs_type: list):
+    def _report_results(self, lint_status: dict, pkgs_status: dict, return_exit_code: int, skipped_code: int,
+                        pkgs_type: list):
         """ Log report to console
 
         Args:
@@ -547,9 +549,26 @@ def report_summary(self, lint_status: dict):
             print(f"{Colors.Fg.red}{wrapper_fail_pack.fill(fail_pack)}{Colors.reset}")
 
     @staticmethod
-    def _create_report(pkgs_status: dict, path: str):
-        if path:
-            json_path = Path(path) / "lint_report.json"
-            json.dump(fp=json_path.open(mode='w'),
-                      obj=pkgs_status,
-                      indent=4)
+    def _create_failed_unit_tests_report(lint_status: dict, path: str):
+        """
+        Creates and saves a file containing all failed unit tests
+        :param lint_status: dict
+            Dictionary containing type of failures and corresponding failing tests. Looks like this:
+             lint_status = {
+            "fail_packs_flake8": [],
+            "fail_packs_bandit": [],
+            "fail_packs_mypy": [],
+            "fail_packs_vulture": [],
+            "fail_packs_pylint": [],
+            "fail_packs_pytest": [],
+            "fail_packs_pwsh_analyze": [],
+            "fail_packs_pwsh_test": [],
+            "fail_packs_image": []
+        }
+        :param path: str
+            The path to save the report.
+        """
+        failed_ut = set.union(*lint_status.values())
+        if path and failed_ut:
+            file_path = Path(path) / "failed_lint_report.txt"
+            file_path.write_text('\n'.join(failed_ut))

demisto_sdk/commands/lint/tests/linter_manager_test.py

@@ -1,7 +1,9 @@
+import os
 from unittest.mock import MagicMock, patch
 
 import pytest
 from demisto_sdk.commands.common.constants import TYPE_PWSH, TYPE_PYTHON
+from demisto_sdk.commands.common.git_tools import git_path
 
 
 @patch('builtins.print')
@@ -29,3 +31,70 @@ def test_report_failed_image_creation():
                                                           pkgs_status=pkgs_status,
                                                           return_exit_code=EXIT_CODES["image"])
     assert not pkgs_status.called
+
+
+def test_create_failed_unit_tests_report_with_failed_tests():
+    """
+    Given`:
+        - Lint manager dictionary with two failed packs -Infoblox and HelloWorld
+
+    When:
+        - Creating failed unit tests report
+
+    Then:
+        - Ensure report file is created.
+        - Ensure report file contains exactly two packs.
+        - Ensure both pack appear in the report.
+    """
+    from demisto_sdk.commands.lint import lint_manager
+    lint_status = {
+        "fail_packs_flake8": set(),
+        "fail_packs_bandit": set(),
+        "fail_packs_mypy": {'Infoblox'},
+        "fail_packs_vulture": set(),
+        "fail_packs_pylint": {'HelloWorld'},
+        "fail_packs_pytest": {'Infoblox'},
+        "fail_packs_pwsh_analyze": set(),
+        "fail_packs_pwsh_test": set(),
+        "fail_packs_image": set()
+    }
+    path = f'{git_path()}/demisto_sdk/commands/lint/tests'
+    lint_manager.LintManager._create_failed_unit_tests_report(lint_status, path)
+    file_path = f'{path}/failed_lint_report.txt'
+    assert os.path.isfile(file_path)
+    with open(file_path, 'r') as file:
+        content = file.read()
+        fail_list = content.split('\n')
+        assert len(fail_list) == 2
+        assert 'HelloWorld' in fail_list
+        assert 'Infoblox' in fail_list
+    os.remove(file_path)
+
+
+def test_create_failed_unit_tests_report_no_failed_tests():
+    """
+    Given:
+        - Lint manager dictionary with no failed packs.
+
+    When:
+        - Creating failed unit tests report.
+
+    Then:
+        - Ensure report file is not created.
+    """
+    from demisto_sdk.commands.lint import lint_manager
+    lint_status = {
+        "fail_packs_flake8": set(),
+        "fail_packs_bandit": set(),
+        "fail_packs_mypy": set(),
+        "fail_packs_vulture": set(),
+        "fail_packs_pylint": set(),
+        "fail_packs_pytest": set(),
+        "fail_packs_pwsh_analyze": set(),
+        "fail_packs_pwsh_test": set(),
+        "fail_packs_image": set(),
+    }
+    path = f'{git_path()}/demisto_sdk/commands/lint/tests'
+    lint_manager.LintManager._create_failed_unit_tests_report(lint_status, path)
+    file_path = f'{path}/failed_lint_report.txt'
+    assert not os.path.isfile(file_path)