Enable using same analyzers for anomaly and CHECK constraints (#322)

* use distinct analyzers to enable using same analyzers for anomaly and checks #316 * Confirm to check in maven * Confirm to check in maven line limit
awslabs · Jan 4, 2021 · fc3b307 · fc3b307
1 parent 298022f
commit fc3b307
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 1 deletion.
diff --git a/src/main/scala/com/amazon/deequ/VerificationSuite.scala b/src/main/scala/com/amazon/deequ/VerificationSuite.scala
@@ -120,7 +120,7 @@ class VerificationSuite {
 
     val analysisResults = AnalysisRunner.doAnalysisRun(
       data,
-      analyzers,
+      analyzers.distinct,
       aggregateWith,
       saveStatesWith,
       metricsRepositoryOptions = AnalysisRunnerRepositoryOptions(

diff --git a/src/test/scala/com/amazon/deequ/VerificationSuiteTest.scala b/src/test/scala/com/amazon/deequ/VerificationSuiteTest.scala
@@ -302,6 +302,49 @@ class VerificationSuiteTest extends WordSpec with Matchers with SparkContextSpec
       }
     }
 
+    "addAnomalyCheck with duplicate check analyzer should work" in
+      withSparkSession { sparkSession =>
+      evaluateWithRepositoryWithHistory { repository =>
+
+        val df = getDfWithNRows(sparkSession, 11)
+        val saveResultsWithKey = ResultKey(5, Map.empty)
+
+        val analyzers = Completeness("item") :: Nil
+
+        val verificationResultOne = VerificationSuite()
+          .onData(df)
+          .addCheck(Check(CheckLevel.Error, "group-1").hasSize(_ == 11))
+          .useRepository(repository)
+          .addRequiredAnalyzers(analyzers)
+          .saveOrAppendResult(saveResultsWithKey)
+          .addAnomalyCheck(
+            AbsoluteChangeStrategy(Some(-2.0), Some(2.0)),
+            Size(),
+            Some(AnomalyCheckConfig(CheckLevel.Warning, "Anomaly check to fail"))
+          )
+          .run()
+
+        val verificationResultTwo = VerificationSuite()
+          .onData(df)
+          .useRepository(repository)
+          .addRequiredAnalyzers(analyzers)
+          .saveOrAppendResult(saveResultsWithKey)
+          .addAnomalyCheck(
+            AbsoluteChangeStrategy(Some(-7.0), Some(7.0)),
+            Size(),
+            Some(AnomalyCheckConfig(CheckLevel.Error, "Anomaly check to succeed",
+              Map.empty, Some(0), Some(11)))
+          )
+          .run()
+
+        val checkResultsOne = verificationResultOne.checkResults.values.toSeq(1).status
+        val checkResultsTwo = verificationResultTwo.checkResults.head._2.status
+
+        assert(checkResultsOne == CheckStatus.Warning)
+        assert(checkResultsTwo == CheckStatus.Success)
+      }
+    }
+
     "write output files to specified locations" in withSparkSession { sparkSession =>
 
       val df = getDfWithNumericValues(sparkSession)