Skip to content

Commit

Permalink
feat: add support for help version image command options
Browse files Browse the repository at this point in the history
  • Loading branch information
@davisusanibar
davisusanibar committed Feb 4, 2024
1 parent 67a2543 commit 0e1d142
Show file tree
Hide file tree
Showing 3 changed files with 258 additions and 20 deletions.
243 changes: 238 additions & 5 deletions isthmus/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

## Overview

Substrait Isthmus is a Java library which enables serializing SQL to [Substrait Protobuf](https://substrait.io/serialization/binary_serialization/) via
Substrait Isthmus is a Java library which enables serializing SQL to [Substrait Protobuf](https://substrait.io/serialization/binary_serialization/) and SQL Expression to [Extended Expression](https://substrait.io/expressions/extended_expression/) via
the Calcite SQL compiler. Optionally, you can leverage the Calcite RelNode to Substrait Plan translator as an IR translation.

## Build
Expand All @@ -15,20 +15,51 @@ Isthmus can be built as a native executable via Graal

## Usage

### Version

```
$ ./isthmus/build/graal/isthmus --version
isthmus 0.1
```

### Help

```
$ ./isthmus/build/graal/isthmus
$ ./isthmus/build/graal/isthmus --help
Usage: isthmus [-m] [-c=<createStatements>]... <sql>
Converts a SQL query to a Substrait Plan
<sql> The sql we should parse.
Usage: isthmus [-hmV] [--crossjoinpolicy=<crossJoinPolicy>]
[-e=<sqlExpression>] [--outputformat=<outputFormat>]
[--sqlconformancemode=<sqlConformanceMode>]
[-c=<createStatements>]... [<sql>]
Substrait Java Native Image for parsing SQL Query and SQL Expressions
[<sql>] The sql we should parse.
-c, --create=<createStatements>
One or multiple create table statements e.g. CREATE
TABLE T1(foo int, bar bigint)
--crossjoinpolicy=<crossJoinPolicy>
One of built-in Calcite SQL compatibility modes:
KEEP_AS_CROSS_JOIN, CONVERT_TO_INNER_JOIN
-e, --expression=<sqlExpression>
The sql expression we should parse.
-h, --help Show this help message and exit.
-m, --multistatement Allow multiple statements terminated with a semicolon
--outputformat=<outputFormat>
Set the output format for the generated plan:
PROTOJSON, PROTOTEXT, BINARY
--sqlconformancemode=<sqlConformanceMode>
One of built-in Calcite SQL compatibility modes:
DEFAULT, LENIENT, BABEL, STRICT_92, STRICT_99,
PRAGMATIC_99, BIG_QUERY, MYSQL_5, ORACLE_10,
ORACLE_12, STRICT_2003, PRAGMATIC_2003, PRESTO,
SQL_SERVER_2008
-V, --version Print version information and exit.
```

## Example

### SQL to Substrait Plan

```
> $ ./isthmus/build/graal/isthmus \
-c "CREATE TABLE Persons ( firstName VARCHAR, lastName VARCHAR, zip INT )" \
Expand Down Expand Up @@ -155,3 +186,205 @@ Converts a SQL query to a Substrait Plan
"expectedTypeUrls": []
}
```

### SQL Expression to Substrait Extended Expression

#### Projection

```
$ ./isthmus/build/graal/isthmus -c "CREATE TABLE NATION (N_NATIONKEY BIGINT NOT NULL, N_NAME CHAR(25), N_REGIONKEY BIGINT NOT NULL, N_COMMENT VARCHAR(152))" \
-e "N_REGIONKEY + 10"
{
"extensionUris": [{
"extensionUriAnchor": 1,
"uri": "/functions_arithmetic.yaml"
}],
"extensions": [{
"extensionFunction": {
"extensionUriReference": 1,
"functionAnchor": 0,
"name": "add:i64_i64"
}
}],
"referredExpr": [{
"expression": {
"scalarFunction": {
"functionReference": 0,
"args": [],
"outputType": {
"i64": {
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
},
"arguments": [{
"value": {
"selection": {
"directReference": {
"structField": {
"field": 2
}
},
"rootReference": {
}
}
}
}, {
"value": {
"cast": {
"type": {
"i64": {
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
},
"input": {
"literal": {
"i32": 10,
"nullable": false,
"typeVariationReference": 0
}
},
"failureBehavior": "FAILURE_BEHAVIOR_UNSPECIFIED"
}
}
}],
"options": []
}
},
"outputNames": ["new-column"]
}],
"baseSchema": {
"names": ["N_NATIONKEY", "N_NAME", "N_REGIONKEY", "N_COMMENT"],
"struct": {
"types": [{
"i64": {
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
}, {
"fixedChar": {
"length": 25,
"typeVariationReference": 0,
"nullability": "NULLABILITY_NULLABLE"
}
}, {
"i64": {
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
}, {
"varchar": {
"length": 152,
"typeVariationReference": 0,
"nullability": "NULLABILITY_NULLABLE"
}
}],
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
},
"expectedTypeUrls": []
}
```

#### Filter

```
$ ./isthmus/build/graal/isthmus -c "CREATE TABLE NATION (N_NATIONKEY BIGINT NOT NULL, N_NAME CHAR(25), N_REGIONKEY BIGINT NOT NULL, N_COMMENT VARCHAR(152))" \
-e "N_REGIONKEY > 10"
{
"extensionUris": [{
"extensionUriAnchor": 1,
"uri": "/functions_comparison.yaml"
}],
"extensions": [{
"extensionFunction": {
"extensionUriReference": 1,
"functionAnchor": 0,
"name": "gt:any_any"
}
}],
"referredExpr": [{
"expression": {
"scalarFunction": {
"functionReference": 0,
"args": [],
"outputType": {
"bool": {
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
},
"arguments": [{
"value": {
"selection": {
"directReference": {
"structField": {
"field": 2
}
},
"rootReference": {
}
}
}
}, {
"value": {
"cast": {
"type": {
"i64": {
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
},
"input": {
"literal": {
"i32": 10,
"nullable": false,
"typeVariationReference": 0
}
},
"failureBehavior": "FAILURE_BEHAVIOR_UNSPECIFIED"
}
}
}],
"options": []
}
},
"outputNames": ["new-column"]
}],
"baseSchema": {
"names": ["N_NATIONKEY", "N_NAME", "N_REGIONKEY", "N_COMMENT"],
"struct": {
"types": [{
"i64": {
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
}, {
"fixedChar": {
"length": 25,
"typeVariationReference": 0,
"nullability": "NULLABILITY_NULLABLE"
}
}, {
"i64": {
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
}, {
"varchar": {
"length": 152,
"typeVariationReference": 0,
"nullability": "NULLABILITY_NULLABLE"
}
}],
"typeVariationReference": 0,
"nullability": "NULLABILITY_REQUIRED"
}
},
"expectedTypeUrls": []
}
```
3 changes: 2 additions & 1 deletion isthmus/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,8 @@ dependencies {
implementation("org.reflections:reflections:0.9.12")
implementation("com.google.guava:guava:29.0-jre")
implementation("org.graalvm.sdk:graal-sdk:22.1.0")
implementation("info.picocli:picocli:4.6.1")
implementation("info.picocli:picocli:4.7.5")
annotationProcessor("info.picocli:picocli-codegen:4.7.5")
implementation("com.fasterxml.jackson.core:jackson-databind:2.13.4")
implementation("com.fasterxml.jackson.core:jackson-annotations:2.13.4")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.4")
Expand Down
32 changes: 18 additions & 14 deletions isthmus/src/main/java/io/substrait/isthmus/IsthmusEntryPoint.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import io.substrait.isthmus.SubstraitRelVisitor.CrossJoinPolicy;
import io.substrait.proto.ExtendedExpression;
import io.substrait.proto.Plan;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
import org.apache.calcite.sql.validate.SqlConformanceEnum;
Expand All @@ -20,10 +19,9 @@
@Command(
name = "isthmus",
version = "isthmus 0.1",
description = "Substrait Java Native Image for parsing SQL Query and SQL Expressions")
description = "Substrait Java Native Image for parsing SQL Query and SQL Expressions",
mixinStandardHelpOptions = true)
public class IsthmusEntryPoint implements Callable<Integer> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(IsthmusEntryPoint.class);

@Parameters(index = "0", arity = "0..1", description = "The sql we should parse.")
private String sql;

Expand Down Expand Up @@ -65,11 +63,23 @@ enum OutputFormat {
description = "One of built-in Calcite SQL compatibility modes: ${COMPLETION-CANDIDATES}")
private CrossJoinPolicy crossJoinPolicy = CrossJoinPolicy.KEEP_AS_CROSS_JOIN;

// this example implements Callable, so parsing, error handling and handling user
// requests for usage help or version help can be done with one line of code.
public static void main(String... args) {
logger.debug(Arrays.toString(args));
int exitCode = new CommandLine(new IsthmusEntryPoint()).execute(args);
CommandLine commandLine = new CommandLine(new IsthmusEntryPoint());
commandLine.setCaseInsensitiveEnumValuesAllowed(true);
CommandLine.ParseResult parseResult = commandLine.parseArgs(args);
if (parseResult.originalArgs().isEmpty()) { // If no arguments print usage help
commandLine.usage(System.out);
System.exit(0);
}
if (commandLine.isUsageHelpRequested()) {
commandLine.usage(System.out);
System.exit(0);
}
if (commandLine.isVersionHelpRequested()) {
commandLine.printVersionHelp(System.out);
System.exit(0);
}
int exitCode = commandLine.execute(args);
System.exit(exitCode);
}

Expand All @@ -78,22 +88,16 @@ public Integer call() throws Exception {
FeatureBoard featureBoard = buildFeatureBoard();
// Isthmus image is parsing SQL Expression if that argument is defined
if (sqlExpression != null) {
logger.debug(sqlExpression);
logger.debug(String.valueOf(createStatements));
SqlExpressionToSubstrait converter =
new SqlExpressionToSubstrait(featureBoard, SimpleExtension.loadDefaults());
ExtendedExpression extendedExpression = converter.convert(sqlExpression, createStatements);
System.out.println(
JsonFormat.printer().includingDefaultValueFields().print(extendedExpression));
switch (outputFormat) {
case PROTOJSON -> System.out.println(
JsonFormat.printer().includingDefaultValueFields().print(extendedExpression));
case PROTOTEXT -> TextFormat.printer().print(extendedExpression, System.out);
case BINARY -> extendedExpression.writeTo(System.out);
}
} else { // by default Isthmus image are parsing SQL Query
logger.debug(sql);
logger.debug(String.valueOf(createStatements));
SqlToSubstrait converter = new SqlToSubstrait(featureBoard);
Plan plan = converter.execute(sql, createStatements);
switch (outputFormat) {
Expand Down

0 comments on commit 0e1d142

Please sign in to comment.