AP-23003: Add "Is not missing" filter option to Row Filter/Splitter

AP-23003 (Row Filter is missing \)
knime · Aug 2, 2024 · 2965833 · 2965833
1 parent 65e2fae
commit 2965833
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 7 deletions.
diff --git a/..._snapshots/org.knime.base.node.preproc.filter.row3.RowFilterNodeSettingsSnapshotTest.snap b/..._snapshots/org.knime.base.node.preproc.filter.row3.RowFilterNodeSettingsSnapshotTest.snap
@@ -119,9 +119,12 @@
                   }, {
                     "const" : "IS_MISSING",
                     "title" : "Is missing"
+                  }, {
+                    "const" : "IS_NOT_MISSING",
+                    "title" : "Is not missing"
                   } ],
                   "title" : "Operator",
-                  "description" : "The operator defining the filter criterion.\n<ul>\n<li><b>=</b>: Value in column must be <b>equal</b> to specified value</li>\n<li><b>≠</b>: Value in column must be <b>not equal</b> to specified value</li>\n<li><b>&lt;</b>: Value in column must be <b>strictly smaller</b> than specified value</li>\n<li><b>≤</b>: Value in column must be <b>smaller than or equal</b> to specified value</li>\n<li><b>&gt;</b>: Value in column must be <b>strictly larger</b> than specified value</li>\n<li><b>≥</b>: Value in column must be <b>larger than or equal</b> than specified value</li>\n<li><b>First <i>n</i> rows</b>: Matches the specified number of rows at the start of the input</li>\n<li><b>Last <i>n</i> rows</b>: Matches the specified number of rows at the end of the input</li>\n<li><b>Matches regex</b>: Value in column must match the specified regular expression</li>\n<li><b>Matches wildcard</b>: Value in column must match the specified pattern, which may contain wildcards <tt>*</tt> and <tt>?</tt></li>\n<li><b>Is true</b>: Boolean value in column must be <tt>true</tt></li>\n<li><b>Is false</b>: Boolean value in column must be <tt>false</tt></li>\n<li><b>Is missing</b>: Value in column must be <i>missing</i></li>\n</ul>",
+                  "description" : "The operator defining the filter criterion.\n<ul>\n<li><b>=</b>: Value in column must be <b>equal</b> to specified value</li>\n<li><b>≠</b>: Value in column must be <b>not equal</b> to specified value</li>\n<li><b>&lt;</b>: Value in column must be <b>strictly smaller</b> than specified value</li>\n<li><b>≤</b>: Value in column must be <b>smaller than or equal</b> to specified value</li>\n<li><b>&gt;</b>: Value in column must be <b>strictly larger</b> than specified value</li>\n<li><b>≥</b>: Value in column must be <b>larger than or equal</b> than specified value</li>\n<li><b>First <i>n</i> rows</b>: Matches the specified number of rows at the start of the input</li>\n<li><b>Last <i>n</i> rows</b>: Matches the specified number of rows at the end of the input</li>\n<li><b>Matches regex</b>: Value in column must match the specified regular expression</li>\n<li><b>Matches wildcard</b>: Value in column must match the specified pattern, which may contain wildcards <tt>*</tt> and <tt>?</tt></li>\n<li><b>Is true</b>: Boolean value in column must be <tt>true</tt></li>\n<li><b>Is false</b>: Boolean value in column must be <tt>false</tt></li>\n<li><b>Is missing</b>: Value in column must be <i>missing</i></li>\n<li><b>Is not missing</b>: Value in column must <em>not</em> be <i>missing</i></li>\n</ul>",
                   "default" : "EQ"
                 },
                 "predicateValues" : {

diff --git a/...napshots/org.knime.base.node.preproc.filter.row3.RowSplitterNodeSettingsSnapshotTest.snap b/...napshots/org.knime.base.node.preproc.filter.row3.RowSplitterNodeSettingsSnapshotTest.snap
@@ -119,9 +119,12 @@
                   }, {
                     "const" : "IS_MISSING",
                     "title" : "Is missing"
+                  }, {
+                    "const" : "IS_NOT_MISSING",
+                    "title" : "Is not missing"
                   } ],
                   "title" : "Operator",
-                  "description" : "The operator defining the filter criterion.\n<ul>\n<li><b>=</b>: Value in column must be <b>equal</b> to specified value</li>\n<li><b>≠</b>: Value in column must be <b>not equal</b> to specified value</li>\n<li><b>&lt;</b>: Value in column must be <b>strictly smaller</b> than specified value</li>\n<li><b>≤</b>: Value in column must be <b>smaller than or equal</b> to specified value</li>\n<li><b>&gt;</b>: Value in column must be <b>strictly larger</b> than specified value</li>\n<li><b>≥</b>: Value in column must be <b>larger than or equal</b> than specified value</li>\n<li><b>First <i>n</i> rows</b>: Matches the specified number of rows at the start of the input</li>\n<li><b>Last <i>n</i> rows</b>: Matches the specified number of rows at the end of the input</li>\n<li><b>Matches regex</b>: Value in column must match the specified regular expression</li>\n<li><b>Matches wildcard</b>: Value in column must match the specified pattern, which may contain wildcards <tt>*</tt> and <tt>?</tt></li>\n<li><b>Is true</b>: Boolean value in column must be <tt>true</tt></li>\n<li><b>Is false</b>: Boolean value in column must be <tt>false</tt></li>\n<li><b>Is missing</b>: Value in column must be <i>missing</i></li>\n</ul>",
+                  "description" : "The operator defining the filter criterion.\n<ul>\n<li><b>=</b>: Value in column must be <b>equal</b> to specified value</li>\n<li><b>≠</b>: Value in column must be <b>not equal</b> to specified value</li>\n<li><b>&lt;</b>: Value in column must be <b>strictly smaller</b> than specified value</li>\n<li><b>≤</b>: Value in column must be <b>smaller than or equal</b> to specified value</li>\n<li><b>&gt;</b>: Value in column must be <b>strictly larger</b> than specified value</li>\n<li><b>≥</b>: Value in column must be <b>larger than or equal</b> than specified value</li>\n<li><b>First <i>n</i> rows</b>: Matches the specified number of rows at the start of the input</li>\n<li><b>Last <i>n</i> rows</b>: Matches the specified number of rows at the end of the input</li>\n<li><b>Matches regex</b>: Value in column must match the specified regular expression</li>\n<li><b>Matches wildcard</b>: Value in column must match the specified pattern, which may contain wildcards <tt>*</tt> and <tt>?</tt></li>\n<li><b>Is true</b>: Boolean value in column must be <tt>true</tt></li>\n<li><b>Is false</b>: Boolean value in column must be <tt>false</tt></li>\n<li><b>Is missing</b>: Value in column must be <i>missing</i></li>\n<li><b>Is not missing</b>: Value in column must <em>not</em> be <i>missing</i></li>\n</ul>",
                   "default" : "EQ"
                 },
                 "predicateValues" : {

diff --git a/org.knime.base.tests/src/org/knime/base/node/preproc/filter/row3/FilterOperatorTest.java b/org.knime.base.tests/src/org/knime/base/node/preproc/filter/row3/FilterOperatorTest.java
@@ -96,6 +96,7 @@ void testOperatorChoices() {
         assertThat(operatorChoicesFor("String1", StringCell.TYPE))
             .as("The list of operators for a string column is what is expected").containsExactlyInAnyOrder( //
                 FilterOperator.IS_MISSING, //
+                FilterOperator.IS_NOT_MISSING, //
                 FilterOperator.EQ, //
                 FilterOperator.NEQ, //
                 FilterOperator.REGEX, //
@@ -104,6 +105,7 @@ void testOperatorChoices() {
         assertThat(operatorChoicesFor("Int1", IntCell.TYPE))
             .as("The list of operators for an integer column is what is expected").containsExactlyInAnyOrder( //
                 FilterOperator.IS_MISSING, //
+                FilterOperator.IS_NOT_MISSING, //
                 FilterOperator.EQ, //
                 FilterOperator.NEQ, //
                 FilterOperator.GT, //
@@ -120,12 +122,14 @@ void testOperatorChoices() {
         assertThat(operatorChoicesFor("Bool1", BooleanCell.TYPE))
             .as("The list of operators for a boolean column is what is expected").containsExactlyInAnyOrder( //
                 FilterOperator.IS_MISSING, //
+                FilterOperator.IS_NOT_MISSING, //
                 FilterOperator.IS_TRUE, //
                 FilterOperator.IS_FALSE //
             );
         assertThat(operatorChoicesFor("Unknown Column", DataType.getType(DataType.getMissingCell().getClass())))
             .as("The list of operators for an unknown column type is what is expected").containsExactlyInAnyOrder( //
-                FilterOperator.IS_MISSING //
+                FilterOperator.IS_MISSING, //
+                FilterOperator.IS_NOT_MISSING //
             );
     }
 

diff --git a/...knime.base/src/org/knime/base/node/preproc/filter/row3/AbstractRowFilterNodeSettings.java b/...knime.base/src/org/knime/base/node/preproc/filter/row3/AbstractRowFilterNodeSettings.java
@@ -209,7 +209,9 @@ void validate(final DataTableSpec spec) throws InvalidSettingsException {
 
             final var operator = m_operator;
             if (isFilterOnRowKeys()) {
-                CheckUtils.checkSetting(operator != FilterOperator.IS_MISSING, "Cannot filter RowID for presence.");
+                CheckUtils.checkSetting(
+                    !(operator == FilterOperator.IS_MISSING || operator == FilterOperator.IS_NOT_MISSING),
+                    "Cannot filter RowID for presence.");
                 CheckUtils.checkSetting(operator.isApplicableFor(SpecialColumns.ROWID, StringCell.TYPE),
                     "Filter operator \"%s\" cannot be applied to RowID.", operator.label());
                 return;
@@ -455,7 +457,7 @@ public List<FilterOperator> computeState(final DefaultNodeSettingsContext contex
                 () -> context.getDataTableSpec(0).map(dts -> dts.getColumnSpec(column)).map(DataColumnSpec::getType));
             if (dataType.isEmpty()) {
                 // we don't know the column, but we know that columns always can contain missing cells
-                return List.of(FilterOperator.IS_MISSING);
+                return List.of(FilterOperator.IS_MISSING, FilterOperator.IS_NOT_MISSING);
             }
             // filter on top-level type
             return Arrays.stream(FilterOperator.values()) //

diff --git a/org.knime.base/src/org/knime/base/node/preproc/filter/row3/FilterOperator.java b/org.knime.base/src/org/knime/base/node/preproc/filter/row3/FilterOperator.java
@@ -110,7 +110,9 @@ enum FilterOperator {
         IS_FALSE("Is false", new IsTruthy(), null, false),
 
         @Label(value = "Is missing", description = "Value in column must be <i>missing</i>")
-        IS_MISSING("Is missing", new IsMissing(), null, false);
+        IS_MISSING("Is missing", new IsMissing(), null, false),
+        @Label(value = "Is not missing", description = "Value in column must <em>not</em> be <i>missing</i>")
+        IS_NOT_MISSING("Is not missing", new IsMissing(), null, false);
 
     final String m_label;
 

diff --git a/org.knime.base/src/org/knime/base/node/preproc/filter/row3/RowReadPredicate.java b/org.knime.base/src/org/knime/base/node/preproc/filter/row3/RowReadPredicate.java
@@ -99,9 +99,11 @@ private static Predicate<RowRead> createFrom(final FilterCriterion criterion, fi
         final var columnIndex = spec.findColumnIndex(column);
         final var operator = criterion.m_operator;
 
-        // Special case for "IS MISSING" operator
+        // Special case for "IS (NOT) MISSING" operators
         if (operator == FilterOperator.IS_MISSING) {
             return row -> row.isMissing(columnIndex);
+        } else if (operator == FilterOperator.IS_NOT_MISSING) {
+            return row -> !row.isMissing(columnIndex);
         }
 
         final var columnSpec = spec.getColumnSpec(columnIndex);