Fix test

tests/test_hydra_cli_errors.py

@@ -1,7 +1,6 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-import re
 from pathlib import Path
-from typing import Any
+from typing import Any, List
 
 from pytest import mark, param
 
@@ -15,66 +14,81 @@
 
 
 @mark.parametrize(
-    "override,expected_substring_1,expected_substring_2",
+    "override,expected_substrings",
     [
         param(
             "+key=int(",
-            "Error when parsing index: 1, string: +key=int( out of [",
-            "no viable alternative at input 'int('",
+            [
+                "Error when parsing index: 1, string: +key=int( out of [",
+                "no viable alternative at input 'int('",
+            ],
             id="parse_error_in_function",
         ),
         param(
             "+key=sort()",
-            """Error when parsing index: 1, string: +key=sort() out of [""",
-            """Error parsing override '+key=sort()'
+            [
+                """Error when parsing index: 1, string: +key=sort() out of [""",
+                """Error parsing override '+key=sort()'
 ValueError while evaluating 'sort()': empty sort input""",
+            ],
             id="empty_sort",
         ),
         param(
             "key=sort(interval(1,10))",
-            """Error when parsing index: 1, string: key=sort(interval(1,10)) out of [""",
-            """Error parsing override 'key=sort(interval(1,10))'
+            [
+                """Error when parsing index: 1, string: key=sort(interval(1,10)) out of [""",
+                """Error parsing override 'key=sort(interval(1,10))'
 TypeError while evaluating 'sort(interval(1,10))': mismatch type argument args[0]""",
+            ],
             id="sort_interval",
         ),
         param(
             "+key=choice()",
-            """Error when parsing index: 1, string: +key=choice() out of [""",
-            """Error parsing override '+key=choice()'
+            [
+                """Error when parsing index: 1, string: +key=choice() out of [""",
+                """Error parsing override '+key=choice()'
 ValueError while evaluating 'choice()': empty choice is not legal""",
+            ],
             id="empty choice",
         ),
         param(
             "+key=extend_list(1, 2, 3)",
-            """Error when parsing index: 1, string: +key=extend_list(1, 2, 3) out of [""",
-            """Error parsing override '+key=extend_list(1, 2, 3)'
+            [
+                """Error when parsing index: 1, string: +key=extend_list(1, 2, 3) out of [""",
+                """Error parsing override '+key=extend_list(1, 2, 3)'
 Trying to use override symbols when extending a list""",
+            ],
             id="plus key extend_list",
         ),
         param(
             "key={inner_key=extend_list(1, 2, 3)}",
-            """Error when parsing index: 1, string: key={inner_key=extend_list(1, 2, 3)} out of [""",
-            "no viable alternative at input '{inner_key='",
+            [
+                """Error when parsing index: 1, string: key={inner_key=extend_list(1, 2, 3)} out of [""",
+                "no viable alternative at input '{inner_key='",
+            ],
             id="embedded extend_list",
         ),
         param(
             ["+key=choice(choice(a,b))", "-m"],
-            """Error when parsing index: 1, string: +key=choice(choice(a,b)) out of [""",
-            """Error parsing override '+key=choice(choice(a,b))'
+            [
+                """Error when parsing index: 1, string: +key=choice(choice(a,b)) out of [""",
+                """Error parsing override '+key=choice(choice(a,b))'
 ValueError while evaluating 'choice(choice(a,b))': nesting choices is not supported
 See https://hydra.cc/docs/1.2/advanced/override_grammar/basic for details
 
 Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
 """,
+            ],
             id="empty choice",
         ),
         param(
             "--config-dir=/dir/not/found",
-            f"""Additional config directory '{Path('/dir/not/found').absolute()}' not found
+            [
+                f"""Additional config directory '{Path('/dir/not/found').absolute()}' not found
 
 Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
-""",
-            "",
+"""
+            ],
             id="config_dir_not_found",
         ),
     ],
@@ -83,27 +97,20 @@ def test_cli_error(
     tmpdir: Any,
     monkeypatch: Any,
     override: Any,
-    expected_substring_1: str,
-    expected_substring_2: str,
+    expected_substrings: List[str],
 ) -> None:
     monkeypatch.chdir("tests/test_apps/app_without_config/")
     if isinstance(override, str):
         override = [override]
     cmd = ["my_app.py", "hydra.sweep.dir=" + str(tmpdir)] + override
     ret = normalize_newlines(run_with_error(cmd))
-    assert (
-        expected_substring_1.strip() in ret and expected_substring_2.strip() in ret
-    ), (
+    missing_substrings = [s for s in expected_substrings if s.strip() not in ret]
+    assert not missing_substrings, (
         f"Result:"
         f"\n---"
         f"\n{ret}"
         f"\n---"
-        f"\nDoes not contain expected_substring_1:"
-        f"\n---"
-        f"\n{expected_substring_1}"
-        f"\n---"
-        f"\nand expected_substring_2:"
-        f"\n---"
-        f"\n{expected_substring_2}"
-        f"\n---"
+        f"\nDoes not contain the following expected substrings:"
+        + "\n---\n".join(f"{s}" for s in missing_substrings)
+        + "\n---"
     )