Stop validation after higher severity fail; update tests

nomenclature/processor/data_validator.py

@@ -177,6 +177,7 @@ def apply(self, df: IamDataFrame) -> IamDataFrame:
                             textwrap.indent(str(failed_validation), prefix="    ")
                             + "\n"
                         )
+                        break
             fail_msg = "(file %s):\n" % get_relative_path(self.file)
             if error:
                 fail_msg = (

tests/data/validation/validate_data/validate_warning_joined_asc.yaml

@@ -0,0 +1,12 @@
+ - variable: Primary Energy
+   year: 2010
+   validation:
+    - warning_level: low
+      upper_bound: 2.5
+      lower_bound: 1
+    - upper_bound: 5
+      lower_bound: 1
+ - variable: Primary Energy|Coal
+   year: 2010
+   upper_bound: 5
+   lower_bound: 1

tests/test_cli.py

@@ -291,7 +291,8 @@ def test_cli_validate_data_fails():
     )
 
     assert cli_result.exit_code == 1
-    assert "Collected 5 errors" in str(cli_result.exception)
+    print(str(cli_result.exception))
+    assert "Collected 6 errors" in str(cli_result.exception)
     assert "Asia" in str(cli_result.exception)
     assert "Final Energy|Industry" in str(cli_result.exception)
 

tests/test_validate_data.py

@@ -137,31 +137,41 @@ def test_DataValidator_apply_fails(simple_df, file, item_1, item_2, item_3, capl
 
 @pytest.mark.parametrize(
     "file",
-    [
-        "separate",
-        "joined",
-    ],
+    ["joined", "legacy"],
 )
 def test_DataValidator_validate_with_warning(file, simple_df, caplog):
+    """Checks that failed validation rows are printed in log."""
     data_validator = DataValidator.from_file(
         DATA_VALIDATION_TEST_DIR / f"validate_warning_{file}.yaml"
     )
-    with pytest.raises(ValueError, match="Data validation failed"):
-        data_validator.apply(simple_df)
 
     failed_validation_message = (
         "Data validation with error(s)/warning(s) "
         f"""(file {(DATA_VALIDATION_TEST_DIR / f"validate_warning_{file}.yaml").relative_to(Path.cwd())}):
   Criteria: variable: ['Primary Energy'], year: [2010], upper_bound: 5.0, lower_bound: 1.0
          model scenario region        variable   unit  year  value warning_level
     0  model_a   scen_a  World  Primary Energy  EJ/yr  2010    6.0         error
-    1  model_a   scen_b  World  Primary Energy  EJ/yr  2010    7.0         error
+    1  model_a   scen_b  World  Primary Energy  EJ/yr  2010    7.0         error"""
+    )
+
+    if file == "legacy":
+        # prints all failed warning levels for legacy format
+        failed_validation_message += """
 
   Criteria: variable: ['Primary Energy'], year: [2010], upper_bound: 2.5, lower_bound: 1.0
          model scenario region        variable   unit  year  value warning_level
     0  model_a   scen_a  World  Primary Energy  EJ/yr  2010    6.0           low
     1  model_a   scen_b  World  Primary Energy  EJ/yr  2010    7.0           low"""
-    )
 
-    # only prints two of three criteria in df to be validated
+    with pytest.raises(ValueError, match="Data validation failed"):
+        data_validator.apply(simple_df)
     assert failed_validation_message in caplog.text
+
+
+def test_DataValidator_warning_order_fail():
+    """Raises validation error if warnings for same criteria not in descending order."""
+    match = "Validation criteria for .* not in descending order of severity."
+    with pytest.raises(ValueError, match=match):
+        DataValidator.from_file(
+            DATA_VALIDATION_TEST_DIR / "validate_warning_joined_asc.yaml"
+        )