From a7456c68ba284fdc4308d8702397133575f573c1 Mon Sep 17 00:00:00 2001 From: Geser Dugarov Date: Mon, 27 Jan 2025 09:22:23 +0700 Subject: [PATCH] [MINOR] Removed `RecordLevelIndexSupport::fetchQueryWithAttribute`, which duplicates `filterQueryWithRecordKey` (#12701) --- .../apache/hudi/RecordLevelIndexSupport.scala | 74 ------------------- 1 file changed, 74 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala index ae4d6afef8e8..40bb888770b8 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/RecordLevelIndexSupport.scala @@ -229,80 +229,6 @@ object RecordLevelIndexSupport { } } - def fetchQueryWithAttribute(queryFilter: Expression, recordKeyOpt: Option[String], literalGenerator: Function2[AttributeReference, Literal, String], - attributeFetcher: Function1[Expression, Expression]): (Option[(Expression, List[String])], Boolean) = { - queryFilter match { - case equalToQuery: EqualTo => - val attributeLiteralTuple = getAttributeLiteralTuple(attributeFetcher.apply(equalToQuery.left), attributeFetcher.apply(equalToQuery.right)).orNull - if (attributeLiteralTuple != null) { - val attribute = attributeLiteralTuple._1 - val literal = attributeLiteralTuple._2 - if (attribute != null && attribute.name != null && attributeMatchesRecordKey(attribute.name, recordKeyOpt)) { - val recordKeyLiteral = literalGenerator.apply(attribute, literal) - (Option.apply(EqualTo(attribute, literal), List.apply(recordKeyLiteral)), true) - } else { - (Option.empty, true) - } - } else { - (Option.empty, true) - } - - case inQuery: In => - var validINQuery = true - val attributeOpt = Option.apply( - attributeFetcher.apply(inQuery.value) match { - case attribute: AttributeReference => - if (!attributeMatchesRecordKey(attribute.name, recordKeyOpt)) { - validINQuery = false - null - } else { - attribute - } - case _ => - validINQuery = false - null - }) - var literals: List[String] = List.empty - inQuery.list.foreach { - case literal: Literal if attributeOpt.isDefined => - val recordKeyLiteral = literalGenerator.apply(attributeOpt.get, literal) - literals = literals :+ recordKeyLiteral - case _ => validINQuery = false - } - if (validINQuery) { - (Option.apply(In(attributeOpt.get, inQuery.list), literals), true) - } else { - (Option.empty, true) - } - - // Handle And expression (composite filter) - case andQuery: And => - val leftResult = filterQueryWithRecordKey(andQuery.left, recordKeyOpt, literalGenerator, attributeFetcher) - val rightResult = filterQueryWithRecordKey(andQuery.right, recordKeyOpt, literalGenerator, attributeFetcher) - - val isSupported = leftResult._2 && rightResult._2 - if (!isSupported) { - (Option.empty, false) - } else { - // If both left and right filters are valid, concatenate their results - (leftResult._1, rightResult._1) match { - case (Some((leftExp, leftKeys)), Some((rightExp, rightKeys))) => - // Return concatenated expressions and record keys - (Option.apply(And(leftExp, rightExp), leftKeys ++ rightKeys), true) - case (Some((leftExp, leftKeys)), None) => - // Return concatenated expressions and record keys - (Option.apply(leftExp, leftKeys), true) - case (None, Some((rightExp, rightKeys))) => - // Return concatenated expressions and record keys - (Option.apply(rightExp, rightKeys), true) - case _ => (Option.empty, true) - } - } - - case _ => (Option.empty, false) - } - } - /** * Returns the list of storage paths from the pruned partitions and file slices. *