feat(spark): add support for more types and literals (binary, list, map, intervals, timestamps) #826
pr.yml
on: pull_request
Build and Test Java
1m 29s
cyclonedx-sbom
1m 30s
Check editorconfig
8s
Lint commits for semantic-release
11s
Security validation
3s
Build Examples
1m 50s
Dry-run release
16s
Matrix: Build Isthmus Native Image
Matrix: osv-scanner
Annotations
2 errors and 1 warning
Build and Test Java:
task ':spark:spotlessScalaCheck'#L1
Execution failed for task ':spark:spotlessScalaCheck':
The following files had format violations:
src/main/scala/io/substrait/spark/expression/ToSparkExpression.scala
@@ -18,18 +18,20 @@
import·io.substrait.spark.{DefaultExpressionVisitor,·HasOutputStack,·SparkExtension,·ToSubstraitType}
import·io.substrait.spark.logical.ToLogicalPlan
+
+import·org.apache.spark.sql.catalyst.CatalystTypeConverters
import·org.apache.spark.sql.catalyst.expressions.{CaseWhen,·Cast,·Expression,·In,·Literal,·MakeDecimal,·NamedExpression,·ScalarSubquery}
+import·org.apache.spark.sql.catalyst.util.{ArrayData,·MapData}
import·org.apache.spark.sql.types.Decimal
import·org.apache.spark.substrait.SparkTypeUtil
import·org.apache.spark.unsafe.types.UTF8String
+
import·io.substrait.`type`.{StringTypeVisitor,·Type}
import·io.substrait.{expression·=>·exp}
import·io.substrait.expression.{Expression·=>·SExpression}
import·io.substrait.util.DecimalUtil
import·io.substrait.utils.Util
import·io.substrait.utils.Util.MICROSECOND_PRECISION
-import·org.apache.spark.sql.catalyst.CatalystTypeConverters
-import·org.apache.spark.sql.catalyst.util.{ArrayData,·MapData}
import·scala.collection.JavaConverters.{asScalaBufferConverter,·mapAsScalaMapConverter}
import·scala.math.pow
@@ -99,7 +101,8 @@
··}
··override·def·visit(expr:·SExpression.MapLiteral):·MapData·=·{
-····val·map·=·expr.values().asScala.map·{·case·(key,·value)·=>·(key.accept(this),·value.accept(this))·}
+····val·map·=
+······expr.values().asScala.map·{·case·(key,·value)·=>·(key.accept(this),·value.accept(this))·}
····CatalystTypeConverters.convertToCatalyst(map).asInstanceOf[MapData]
··}
src/main/scala/io/substrait/spark/expression/ToSubstraitLiteral.scala
@@ -38,10 +38,17 @@
········scale:·Int):·SExpression.Literal·=
······decimal(false,·d.toJavaBigDecimal,·precision,·scale)
-····private·def·sparkArray2Substrait(arrayData:·ArrayData,·elementType:·DataType):·SExpression.Literal·=
-······list(false,·JavaConverters.asJavaIterable(arrayData.array.map(any·=>·apply(Literal(any,·elementType)))))
+····private·def·sparkArray2Substrait(
+········arrayData:·ArrayData,
+········elementType:·DataType):·SExpression.Literal·=
+······list(
+········false,
+········JavaConverters.asJavaIterable(arrayData.array.map(any·=>·apply(Literal(any,·elementType)))))
-····private·def·sparkMap2Substrait(mapData:·MapData,·keyType:·DataType,·valueType:·DataType):·SExpression.Literal·=·{
+····private·def·sparkMap2Substrait(
... (32 more lines that didn't fit)
Run './gradlew :spark:spotlessApply' to fix these violations.
at com.diffplug.gradle.spotless.SpotlessCheck.performAction(SpotlessCheck.java:111)
at com.diffplug.gradle.spotless.SpotlessCheck.performAction(SpotlessCheck.java:53)
at org.gradle.internal.reflect.JavaMethod.invoke(JavaMethod.java:125)
at org.gradle.api.internal.project.taskfactory.StandardTaskAction.doExecute(StandardTaskAction.java:58)
at org.gradle.api.internal.project.taskfactory.StandardTaskAction.execute(StandardTaskAction.java:51)
at org.gradle.api.internal.project.taskfactory.StandardTaskAction.execute(StandardTaskAction.java:29)
at org.gradle.api.internal.tasks.execution.TaskExecution$3.run(TaskExecution.java:244)
at org.gradle.internal.operations.DefaultBuildOperationRunner$1.execute(DefaultBuildOperationRunner.java:29)
at org.gradle.internal.operations.DefaultBuildOperationRunner$1.execute(DefaultBuildOperationR
|
Build and Test Java
Process completed with exit code 1.
|
Check editorconfig
The following actions use a deprecated Node.js version and will be forced to run on node20: editorconfig-checker/action-editorconfig-checker@v2. For more info: https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/
|
Artifacts
Produced during runtime
Name | Size | |
---|---|---|
cyclonedx-sbom
Expired
|
66.4 KB |
|