Refactor multiple warning levels for same data validation filter

dc-almeida · Jan 20, 2025 · b01834b · b01834b
1 parent c62da7f
commit b01834b
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 14 deletions.
diff --git a/nomenclature/processor/data_validator.py b/nomenclature/processor/data_validator.py
@@ -18,10 +18,10 @@
 
 
 class WarningEnum(str, Enum):
+    error = "error"
     high = "high"
     medium = "medium"
     low = "low"
-    error = "error"
 
 
 class DataValidationCriteria(IamcDataFilter):
@@ -106,7 +106,18 @@ def check_criteria(cls, v):
     def from_file(cls, file: Path | str) -> "DataValidator":
         with open(file, "r", encoding="utf-8") as f:
             content = yaml.safe_load(f)
-        return cls(file=file, criteria_items=content)
+        criteria_items = []
+        for item in content:
+            if "validation" in item:
+                filter_args = {k: v for k, v in item.items() if k != "validation"}
+                for validation_args in sorted(
+                    item["validation"],
+                    key=lambda d: d.get("warning_level", WarningEnum.error),
+                ):
+                    criteria_items.append(dict({**filter_args, **validation_args}))
+            else:
+                criteria_items.append(item)
+        return cls(file=file, criteria_items=criteria_items)
 
     def apply(self, df: IamDataFrame) -> IamDataFrame:
         fail_list = []

diff --git a/tests/data/validation/validate_data/validate_warning_joined.yaml b/tests/data/validation/validate_data/validate_warning_joined.yaml
@@ -0,0 +1,12 @@
+ - variable: Primary Energy
+   year: 2010
+   validation:
+    - upper_bound: 5
+      lower_bound: 1
+    - warning_level: low
+      upper_bound: 2.5
+      lower_bound: 1
+ - variable: Primary Energy|Coal
+   year: 2010
+   upper_bound: 5
+   lower_bound: 1
diff --git a/...ation/validate_data/validate_warning.yaml → ...idate_data/validate_warning_separate.yaml b/...ation/validate_data/validate_warning.yaml → ...idate_data/validate_warning_separate.yaml
@@ -1,12 +1,12 @@
  - variable: Primary Energy
    year: 2010
-   upper_bound: 2.5
+   upper_bound: 5
    lower_bound: 1
-   warning_level: low
  - variable: Primary Energy
    year: 2010
-   upper_bound: 5
+   upper_bound: 2.5
    lower_bound: 1
+   warning_level: low
  - variable: Primary Energy|Coal
    year: 2010
    upper_bound: 5

diff --git a/tests/test_validate_data.py b/tests/test_validate_data.py
@@ -129,25 +129,32 @@ def test_DataValidator_apply_fails(simple_df, file, item_1, item_2, item_3, capl
     assert failed_validation_message in caplog.text
 
 
-def test_DataValidator_validate_with_warning(simple_df, caplog):
+@pytest.mark.parametrize(
+    "file",
+    [
+        "separate",
+        "joined",
+    ],
+)
+def test_DataValidator_validate_with_warning(file, simple_df, caplog):
     data_validator = DataValidator.from_file(
-        DATA_VALIDATION_TEST_DIR / "validate_warning.yaml"
+        DATA_VALIDATION_TEST_DIR / f"validate_warning_{file}.yaml"
     )
     with pytest.raises(ValueError, match="Data validation failed"):
         data_validator.apply(simple_df)
 
     failed_validation_message = (
         "Data validation with error(s)/warning(s) "
-        f"""(file {(DATA_VALIDATION_TEST_DIR / "validate_warning.yaml").relative_to(Path.cwd())}):
-  Criteria: variable: ['Primary Energy'], year: [2010], upper_bound: 2.5, lower_bound: 1.0
-         model scenario region        variable   unit  year  value warning_level
-    0  model_a   scen_a  World  Primary Energy  EJ/yr  2010    6.0           low
-    1  model_a   scen_b  World  Primary Energy  EJ/yr  2010    7.0           low
-
+        f"""(file {(DATA_VALIDATION_TEST_DIR / f"validate_warning_{file}.yaml").relative_to(Path.cwd())}):
   Criteria: variable: ['Primary Energy'], year: [2010], upper_bound: 5.0, lower_bound: 1.0
          model scenario region        variable   unit  year  value warning_level
     0  model_a   scen_a  World  Primary Energy  EJ/yr  2010    6.0         error
-    1  model_a   scen_b  World  Primary Energy  EJ/yr  2010    7.0         error"""
+    1  model_a   scen_b  World  Primary Energy  EJ/yr  2010    7.0         error
+
+  Criteria: variable: ['Primary Energy'], year: [2010], upper_bound: 2.5, lower_bound: 1.0
+         model scenario region        variable   unit  year  value warning_level
+    0  model_a   scen_a  World  Primary Energy  EJ/yr  2010    6.0           low
+    1  model_a   scen_b  World  Primary Energy  EJ/yr  2010    7.0           low"""
     )
 
     # only prints two of three criteria in df to be validated