From 55424fc80d68b40ae1c277757f35223ebaa3ccbc Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Thu, 27 Feb 2025 01:14:23 +0800 Subject: [PATCH 01/15] jdbc --- flink-connector/flink/build.gradle.kts | 3 +- .../flink/connector/catalog/BaseCatalog.java | 2 +- .../connector/jdbc/GravitinoJdbcCatalog.java | 28 +++++++ .../jdbc/GravitinoJdbcCatalogFactory.java | 31 ++++++++ .../GravitinoJdbcCatalogFactoryOptions.java | 6 ++ .../jdbc/JdbcPropertiesConstants.java | 54 +++++++++++++ .../jdbc/JdbcPropertiesConverter.java | 25 ++++++ .../test/jdbc/FlinkJdbcMysqlCatalogIT.java | 77 +++++++++++++++++++ .../jdbc/TestJdbcPropertiesConverter.java | 61 +++++++++++++++ 9 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java create mode 100644 flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java create mode 100644 flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java diff --git a/flink-connector/flink/build.gradle.kts b/flink-connector/flink/build.gradle.kts index 6cbfbfa53b4..c7d9c4fe7a7 100644 --- a/flink-connector/flink/build.gradle.kts +++ b/flink-connector/flink/build.gradle.kts @@ -29,7 +29,7 @@ repositories { var paimonVersion: String = libs.versions.paimon.get() val flinkVersion: String = libs.versions.flink.get() val flinkMajorVersion: String = flinkVersion.substringBeforeLast(".") - +val flinkJdbcVersion: String = "3.2.0-1.18" val icebergVersion: String = libs.versions.iceberg.get() // The Flink only support scala 2.12, and all scala api will be removed in a future version. @@ -56,6 +56,7 @@ dependencies { compileOnly("org.apache.flink:flink-table-common:$flinkVersion") compileOnly("org.apache.flink:flink-table-api-java:$flinkVersion") compileOnly("org.apache.paimon:paimon-flink-1.18:$paimonVersion") + compileOnly("org.apache.flink:flink-connector-jdbc:$flinkJdbcVersion") compileOnly(libs.hive2.exec) { artifact { diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java index e9320c786cd..1c875920642 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java @@ -101,7 +101,7 @@ protected BaseCatalog( this.partitionConverter = partitionConverter; } - protected abstract AbstractCatalog realCatalog(); + protected abstract org.apache.flink.table.catalog.Catalog realCatalog(); @Override public void open() throws CatalogException {} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java new file mode 100644 index 00000000000..78fade90f08 --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java @@ -0,0 +1,28 @@ +package org.apache.gravitino.flink.connector.jdbc; + +import org.apache.flink.connector.jdbc.catalog.factory.JdbcCatalogFactory; +import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.table.factories.CatalogFactory; +import org.apache.gravitino.flink.connector.PartitionConverter; +import org.apache.gravitino.flink.connector.PropertiesConverter; +import org.apache.gravitino.flink.connector.catalog.BaseCatalog; + +public class GravitinoJdbcCatalog extends BaseCatalog { + + Catalog jdbcCatalog; + + protected GravitinoJdbcCatalog( + CatalogFactory.Context context, + String defaultDatabase, + PropertiesConverter propertiesConverter, + PartitionConverter partitionConverter) { + super(context.getName(), defaultDatabase, propertiesConverter, partitionConverter); + JdbcCatalogFactory jdbcCatalogFactory = new JdbcCatalogFactory(); + this.jdbcCatalog = jdbcCatalogFactory.createCatalog(context); + } + + @Override + protected Catalog realCatalog() { + return jdbcCatalog; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java new file mode 100644 index 00000000000..ec7bf5bb56d --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java @@ -0,0 +1,31 @@ +package org.apache.gravitino.flink.connector.jdbc; + +import org.apache.gravitino.Catalog; +import org.apache.gravitino.flink.connector.DefaultPartitionConverter; +import org.apache.gravitino.flink.connector.PartitionConverter; +import org.apache.gravitino.flink.connector.PropertiesConverter; +import org.apache.gravitino.flink.connector.catalog.BaseCatalogFactory; +import org.apache.gravitino.flink.connector.paimon.GravitinoPaimonCatalogFactoryOptions; + +public class GravitinoJdbcCatalogFactory implements BaseCatalogFactory { + + @Override + public String gravitinoCatalogProvider() { + return GravitinoPaimonCatalogFactoryOptions.IDENTIFIER; + } + + @Override + public Catalog.Type gravitinoCatalogType() { + return Catalog.Type.RELATIONAL; + } + + @Override + public PropertiesConverter propertiesConverter() { + return JdbcPropertiesConverter.INSTANCE; + } + + @Override + public PartitionConverter partitionConverter() { + return DefaultPartitionConverter.INSTANCE; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java new file mode 100644 index 00000000000..cdb87a07fde --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java @@ -0,0 +1,6 @@ +package org.apache.gravitino.flink.connector.jdbc; + +public class GravitinoJdbcCatalogFactoryOptions { + + public static final String IDENTIFIER = "gravitino-jdbc"; +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java new file mode 100644 index 00000000000..e4e42404f87 --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.jdbc; + +import java.util.HashMap; +import java.util.Map; + +public class JdbcPropertiesConstants { + + private JdbcPropertiesConstants() {} + + public static final String GRAVITINO_JDBC_USER = "jdbc-user"; + public static final String GRAVITINO_JDBC_PASSWORD = "jdbc-password"; + public static final String GRAVITINO_JDBC_URL = "jdbc-url"; + public static final String GRAVITINO_JDBC_DRIVER = "jdbc-driver"; + public static final String GRAVITINO_JDBC_DEFAULT_DATABASE = "default-database"; + + public static final String FLINK_JDBC_URL = "base-url"; + public static final String FLINK_JDBC_USER = "user"; + public static final String FLINK_JDBC_PASSWORD = "password"; + public static final String FLINK_JDBC_DEFAULT_DATABASE = "default-database"; + + public static Map flinkToGravitinoMap = new HashMap<>(); + public static Map gravitinoToFlinkMap = new HashMap<>(); + + static { + flinkToGravitinoMap.put(FLINK_JDBC_URL, GRAVITINO_JDBC_URL); + flinkToGravitinoMap.put(FLINK_JDBC_USER, GRAVITINO_JDBC_USER); + flinkToGravitinoMap.put(FLINK_JDBC_PASSWORD, GRAVITINO_JDBC_PASSWORD); + flinkToGravitinoMap.put(FLINK_JDBC_DEFAULT_DATABASE, GRAVITINO_JDBC_DEFAULT_DATABASE); + + gravitinoToFlinkMap.put(GRAVITINO_JDBC_URL, FLINK_JDBC_URL); + gravitinoToFlinkMap.put(GRAVITINO_JDBC_USER, FLINK_JDBC_USER); + gravitinoToFlinkMap.put(GRAVITINO_JDBC_PASSWORD, FLINK_JDBC_PASSWORD); + gravitinoToFlinkMap.put(GRAVITINO_JDBC_DEFAULT_DATABASE, FLINK_JDBC_DEFAULT_DATABASE); + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java new file mode 100644 index 00000000000..9ae12de561a --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java @@ -0,0 +1,25 @@ +package org.apache.gravitino.flink.connector.jdbc; + +import org.apache.gravitino.flink.connector.PropertiesConverter; + +public class JdbcPropertiesConverter implements PropertiesConverter { + + public static final JdbcPropertiesConverter INSTANCE = new JdbcPropertiesConverter(); + + private JdbcPropertiesConverter() {} + + @Override + public String transformPropertyToGravitinoCatalog(String configKey) { + return JdbcPropertiesConstants.flinkToGravitinoMap.get(configKey); + } + + @Override + public String transformPropertyToFlinkCatalog(String configKey) { + return JdbcPropertiesConstants.gravitinoToFlinkMap.get(configKey); + } + + @Override + public String getFlinkCatalogType() { + return GravitinoJdbcCatalogFactoryOptions.IDENTIFIER; + } +} diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java new file mode 100644 index 00000000000..bef89d83a3c --- /dev/null +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java @@ -0,0 +1,77 @@ +package org.apache.gravitino.flink.connector.integration.test.jdbc; + +import static org.apache.gravitino.integration.test.util.TestDatabaseName.MYSQL_CATALOG_MYSQL_IT; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.flink.connector.integration.test.FlinkCommonIT; +import org.apache.gravitino.flink.connector.jdbc.JdbcPropertiesConstants; +import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; + +@Tag("gravitino-docker-test") +public class FlinkJdbcMysqlCatalogIT extends FlinkCommonIT { + + protected String mysqlUrl; + protected String mysqlUsername; + protected String mysqlPassword; + protected String mysqlDriver; + protected String mysqlDefaultDatabase = MYSQL_CATALOG_MYSQL_IT.name(); + + protected Catalog catalog; + + protected static final String CATALOG_NAME = "test_flink_jdbc_catalog"; + + @Override + protected Catalog currentCatalog() { + return catalog; + } + + @Override + protected String getProvider() { + return "jdbc-mysql"; + } + + @BeforeAll + void setup() { + init(); + } + + @Override + protected boolean supportDropCascade() { + return true; + } + + private void init() { + Preconditions.checkNotNull(metalake); + catalog = + metalake.createCatalog( + CATALOG_NAME, + org.apache.gravitino.Catalog.Type.RELATIONAL, + getProvider(), + null, + ImmutableMap.of( + JdbcPropertiesConstants.GRAVITINO_JDBC_USER, + mysqlUsername, + JdbcPropertiesConstants.GRAVITINO_JDBC_PASSWORD, + mysqlPassword, + JdbcPropertiesConstants.GRAVITINO_JDBC_URL, + mysqlUrl, + JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER, + mysqlDriver, + JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE, + mysqlDefaultDatabase)); + } + + @Override + protected void initCatalogEnv() throws Exception { + ContainerSuite containerSuite = ContainerSuite.getInstance(); + containerSuite.startMySQLContainer(MYSQL_CATALOG_MYSQL_IT); + mysqlUrl = containerSuite.getMySQLContainer().getJdbcUrl(); + mysqlUsername = containerSuite.getMySQLContainer().getUsername(); + mysqlPassword = containerSuite.getMySQLContainer().getPassword(); + mysqlDriver = containerSuite.getMySQLContainer().getDriverClassName(MYSQL_CATALOG_MYSQL_IT); + } +} diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java new file mode 100644 index 00000000000..76452183c87 --- /dev/null +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java @@ -0,0 +1,61 @@ +package org.apache.gravitino.flink.connector.jdbc; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.flink.configuration.Configuration; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Test for {@link JdbcPropertiesConverter} */ +public class TestJdbcPropertiesConverter { + + String username = "testUser"; + String password = "testPassword"; + String url = "testUrl"; + String defaultDatabase = "test"; + + private static final JdbcPropertiesConverter CONVERTER = JdbcPropertiesConverter.INSTANCE; + + @Test + public void testToPaimonFileSystemCatalog() { + Map catalogProperties = + ImmutableMap.of( + JdbcPropertiesConstants.GRAVITINO_JDBC_USER, + username, + JdbcPropertiesConstants.GRAVITINO_JDBC_PASSWORD, + password, + JdbcPropertiesConstants.GRAVITINO_JDBC_URL, + url, + JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE, + defaultDatabase); + Map properties = CONVERTER.toFlinkCatalogProperties(catalogProperties); + Assertions.assertEquals(username, properties.get(JdbcPropertiesConstants.FLINK_JDBC_USER)); + Assertions.assertEquals(password, properties.get(JdbcPropertiesConstants.FLINK_JDBC_PASSWORD)); + Assertions.assertEquals(url, properties.get(JdbcPropertiesConstants.FLINK_JDBC_URL)); + Assertions.assertEquals( + defaultDatabase, properties.get(JdbcPropertiesConstants.FLINK_JDBC_DEFAULT_DATABASE)); + } + + @Test + public void testToGravitinoCatalogProperties() { + Configuration configuration = + Configuration.fromMap( + ImmutableMap.of( + JdbcPropertiesConstants.FLINK_JDBC_USER, + username, + JdbcPropertiesConstants.FLINK_JDBC_PASSWORD, + password, + JdbcPropertiesConstants.FLINK_JDBC_URL, + url, + JdbcPropertiesConstants.FLINK_JDBC_DEFAULT_DATABASE, + defaultDatabase)); + Map properties = CONVERTER.toGravitinoCatalogProperties(configuration); + + Assertions.assertEquals(username, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_USER)); + Assertions.assertEquals( + password, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_PASSWORD)); + Assertions.assertEquals(url, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_URL)); + Assertions.assertEquals( + defaultDatabase, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE)); + } +} From 5a43410ec9bc849cf20a22a36c5985f6a51c6dee Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Thu, 27 Feb 2025 01:14:42 +0800 Subject: [PATCH 02/15] jdbc --- .../flink/connector/integration/test/FlinkEnvIT.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java index 959123f3362..962f7f2f847 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java @@ -66,13 +66,14 @@ public abstract class FlinkEnvIT extends BaseIT { private static String gravitinoUri = "http://127.0.0.1:8090"; @BeforeAll - void startUp() { + void startUp() throws Exception { // Start Gravitino server initGravitinoEnv(); initMetalake(); initHiveEnv(); initHdfsEnv(); initFlinkEnv(); + initCatalogEnv(); LOG.info("Startup Flink env successfully, Gravitino uri: {}.", gravitinoUri); } @@ -83,6 +84,8 @@ static void stop() { LOG.info("Stop Flink env successfully."); } + protected void initCatalogEnv() throws Exception {}; + protected String flinkByPass(String key) { return PropertiesConverter.FLINK_PROPERTY_PREFIX + key; } From 555d014b3374f628882842c24c09a0096cc08005 Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Thu, 27 Feb 2025 01:22:02 +0800 Subject: [PATCH 03/15] license --- .../connector/jdbc/GravitinoJdbcCatalog.java | 19 +++++++++++++++++++ .../jdbc/GravitinoJdbcCatalogFactory.java | 19 +++++++++++++++++++ .../GravitinoJdbcCatalogFactoryOptions.java | 19 +++++++++++++++++++ .../jdbc/JdbcPropertiesConverter.java | 19 +++++++++++++++++++ .../test/jdbc/FlinkJdbcMysqlCatalogIT.java | 19 +++++++++++++++++++ .../jdbc/TestJdbcPropertiesConverter.java | 19 +++++++++++++++++++ 6 files changed, 114 insertions(+) diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java index 78fade90f08..30cb4886dc6 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.gravitino.flink.connector.jdbc; import org.apache.flink.connector.jdbc.catalog.factory.JdbcCatalogFactory; diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java index ec7bf5bb56d..363f45c43bc 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.gravitino.flink.connector.jdbc; import org.apache.gravitino.Catalog; diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java index cdb87a07fde..a38c3c236b1 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.gravitino.flink.connector.jdbc; public class GravitinoJdbcCatalogFactoryOptions { diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java index 9ae12de561a..41a2f75715e 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.gravitino.flink.connector.jdbc; import org.apache.gravitino.flink.connector.PropertiesConverter; diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java index bef89d83a3c..ae2492e6e26 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.gravitino.flink.connector.integration.test.jdbc; import static org.apache.gravitino.integration.test.util.TestDatabaseName.MYSQL_CATALOG_MYSQL_IT; diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java index 76452183c87..1b56c63c224 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.gravitino.flink.connector.jdbc; import com.google.common.collect.ImmutableMap; From f24b924a1d4b1005e22086769f13b4774bbf56c8 Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Fri, 28 Feb 2025 00:29:38 +0800 Subject: [PATCH 04/15] license --- flink-connector/flink/build.gradle.kts | 5 +- .../flink/connector/catalog/BaseCatalog.java | 2 +- .../connector/jdbc/GravitinoJdbcCatalog.java | 47 +++++++++++++++++-- .../jdbc/GravitinoJdbcCatalogFactory.java | 37 ++++++++++++++- .../GravitinoJdbcCatalogFactoryOptions.java | 8 ++++ .../jdbc/JdbcPropertiesConstants.java | 2 +- .../store/GravitinoCatalogStore.java | 10 +++- .../org.apache.flink.table.factories.Factory | 1 + .../integration/test/FlinkCommonIT.java | 18 +++++-- .../integration/test/FlinkEnvIT.java | 2 - .../test/jdbc/FlinkJdbcMysqlCatalogIT.java | 22 +++++++++ gradle/libs.versions.toml | 2 + 12 files changed, 138 insertions(+), 18 deletions(-) diff --git a/flink-connector/flink/build.gradle.kts b/flink-connector/flink/build.gradle.kts index c7d9c4fe7a7..33cf15889a7 100644 --- a/flink-connector/flink/build.gradle.kts +++ b/flink-connector/flink/build.gradle.kts @@ -29,7 +29,6 @@ repositories { var paimonVersion: String = libs.versions.paimon.get() val flinkVersion: String = libs.versions.flink.get() val flinkMajorVersion: String = flinkVersion.substringBeforeLast(".") -val flinkJdbcVersion: String = "3.2.0-1.18" val icebergVersion: String = libs.versions.iceberg.get() // The Flink only support scala 2.12, and all scala api will be removed in a future version. @@ -56,7 +55,7 @@ dependencies { compileOnly("org.apache.flink:flink-table-common:$flinkVersion") compileOnly("org.apache.flink:flink-table-api-java:$flinkVersion") compileOnly("org.apache.paimon:paimon-flink-1.18:$paimonVersion") - compileOnly("org.apache.flink:flink-connector-jdbc:$flinkJdbcVersion") + compileOnly(libs.flinkjdbc) compileOnly(libs.hive2.exec) { artifact { @@ -98,7 +97,7 @@ dependencies { testImplementation(libs.testcontainers.junit.jupiter) testImplementation(libs.testcontainers.mysql) testImplementation(libs.metrics.core) - + testImplementation(libs.flinkjdbc) testImplementation("org.apache.iceberg:iceberg-flink-runtime-$flinkMajorVersion:$icebergVersion") testImplementation("org.apache.flink:flink-connector-hive_$scalaVersion:$flinkVersion") testImplementation("org.apache.flink:flink-table-common:$flinkVersion") diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java index 1c875920642..e9320c786cd 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java @@ -101,7 +101,7 @@ protected BaseCatalog( this.partitionConverter = partitionConverter; } - protected abstract org.apache.flink.table.catalog.Catalog realCatalog(); + protected abstract AbstractCatalog realCatalog(); @Override public void open() throws CatalogException {} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java index 30cb4886dc6..2b56116b49d 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java @@ -19,16 +19,31 @@ package org.apache.gravitino.flink.connector.jdbc; +import java.util.Map; +import java.util.Optional; +import org.apache.flink.connector.jdbc.catalog.JdbcCatalog; import org.apache.flink.connector.jdbc.catalog.factory.JdbcCatalogFactory; -import org.apache.flink.table.catalog.Catalog; +import org.apache.flink.connector.jdbc.table.JdbcDynamicTableFactory; +import org.apache.flink.table.catalog.AbstractCatalog; +import org.apache.flink.table.catalog.CatalogBaseTable; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.flink.table.factories.CatalogFactory; +import org.apache.flink.table.factories.Factory; import org.apache.gravitino.flink.connector.PartitionConverter; import org.apache.gravitino.flink.connector.PropertiesConverter; import org.apache.gravitino.flink.connector.catalog.BaseCatalog; +/** + * The GravitinoJdbcCatalog class is an implementation of the BaseCatalog class that is used to + * proxy the JdbcCatalog class. + */ public class GravitinoJdbcCatalog extends BaseCatalog { - Catalog jdbcCatalog; + private final JdbcCatalog jdbcCatalog; + + private final CatalogFactory.Context context; protected GravitinoJdbcCatalog( CatalogFactory.Context context, @@ -36,12 +51,36 @@ protected GravitinoJdbcCatalog( PropertiesConverter propertiesConverter, PartitionConverter partitionConverter) { super(context.getName(), defaultDatabase, propertiesConverter, partitionConverter); + this.context = context; JdbcCatalogFactory jdbcCatalogFactory = new JdbcCatalogFactory(); - this.jdbcCatalog = jdbcCatalogFactory.createCatalog(context); + this.jdbcCatalog = (JdbcCatalog) jdbcCatalogFactory.createCatalog(context); } @Override - protected Catalog realCatalog() { + protected AbstractCatalog realCatalog() { return jdbcCatalog; } + + @Override + public Optional getFactory() { + return Optional.of(new JdbcDynamicTableFactory()); + } + + @Override + public CatalogBaseTable getTable(ObjectPath tablePath) + throws TableNotExistException, CatalogException { + CatalogBaseTable table = super.getTable(tablePath); + Map contextOptions = context.getOptions(); + Map tableOptions = table.getOptions(); + tableOptions.remove("engine"); + tableOptions.put( + "url", + contextOptions.get(JdbcPropertiesConstants.FLINK_JDBC_URL) + + "/" + + tablePath.getDatabaseName()); + tableOptions.put("table-name", tablePath.getObjectName()); + tableOptions.put("username", contextOptions.get(JdbcPropertiesConstants.FLINK_JDBC_USER)); + tableOptions.put("password", contextOptions.get(JdbcPropertiesConstants.FLINK_JDBC_PASSWORD)); + return table; + } } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java index 363f45c43bc..2cacd14a296 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java @@ -19,18 +19,41 @@ package org.apache.gravitino.flink.connector.jdbc; +import java.util.Collections; +import java.util.Set; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.table.factories.FactoryUtil; import org.apache.gravitino.Catalog; import org.apache.gravitino.flink.connector.DefaultPartitionConverter; import org.apache.gravitino.flink.connector.PartitionConverter; import org.apache.gravitino.flink.connector.PropertiesConverter; import org.apache.gravitino.flink.connector.catalog.BaseCatalogFactory; -import org.apache.gravitino.flink.connector.paimon.GravitinoPaimonCatalogFactoryOptions; +import org.apache.gravitino.flink.connector.utils.FactoryUtils; +/** + * Factory for creating instances of {@link GravitinoJdbcCatalog}. It will be created by SPI + * discovery in Flink. + */ public class GravitinoJdbcCatalogFactory implements BaseCatalogFactory { + @Override + public org.apache.flink.table.catalog.Catalog createCatalog(Context context) { + final FactoryUtil.CatalogFactoryHelper helper = + FactoryUtils.createCatalogFactoryHelper(this, context); + String defaultDatabase = + helper.getOptions().get(GravitinoJdbcCatalogFactoryOptions.DEFAULT_DATABASE); + return new GravitinoJdbcCatalog( + context, defaultDatabase, propertiesConverter(), partitionConverter()); + } + @Override public String gravitinoCatalogProvider() { - return GravitinoPaimonCatalogFactoryOptions.IDENTIFIER; + return "jdbc"; + } + + @Override + public String factoryIdentifier() { + return GravitinoJdbcCatalogFactoryOptions.IDENTIFIER; } @Override @@ -47,4 +70,14 @@ public PropertiesConverter propertiesConverter() { public PartitionConverter partitionConverter() { return DefaultPartitionConverter.INSTANCE; } + + @Override + public Set> requiredOptions() { + return Collections.emptySet(); + } + + @Override + public Set> optionalOptions() { + return Collections.emptySet(); + } } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java index a38c3c236b1..f656805a229 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java @@ -19,7 +19,15 @@ package org.apache.gravitino.flink.connector.jdbc; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.iceberg.flink.FlinkCatalogFactory; + public class GravitinoJdbcCatalogFactoryOptions { + /** Identifier for the {@link GravitinoJdbcCatalog}. */ public static final String IDENTIFIER = "gravitino-jdbc"; + + public static final ConfigOption DEFAULT_DATABASE = + ConfigOptions.key(FlinkCatalogFactory.DEFAULT_DATABASE).stringType().noDefaultValue(); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java index e4e42404f87..03a9b333eda 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java @@ -33,7 +33,7 @@ private JdbcPropertiesConstants() {} public static final String GRAVITINO_JDBC_DEFAULT_DATABASE = "default-database"; public static final String FLINK_JDBC_URL = "base-url"; - public static final String FLINK_JDBC_USER = "user"; + public static final String FLINK_JDBC_USER = "username"; public static final String FLINK_JDBC_PASSWORD = "password"; public static final String FLINK_JDBC_DEFAULT_DATABASE = "default-database"; diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java index 4c29b7fde3b..a60aaf30c89 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java @@ -121,16 +121,24 @@ private BaseCatalogFactory getCatalogFactory(Map configuration) } private BaseCatalogFactory getCatalogFactory(String provider) { + String convertedProvider = getConvertedProvider(provider); return discoverFactories( catalogFactory -> ((BaseCatalogFactory) catalogFactory) .gravitinoCatalogProvider() - .equalsIgnoreCase(provider), + .equalsIgnoreCase(convertedProvider), String.format( "Gravitino catalog provider [%s] matched multiple flink catalog factories, it should only match one.", provider)); } + private String getConvertedProvider(String provider) { + if (provider.startsWith("jdbc")) { + return "jdbc"; + } + return provider; + } + private BaseCatalogFactory discoverFactories(Predicate predicate, String errorMessage) { Iterator serviceLoaderIterator = ServiceLoader.load(Factory.class).iterator(); final List factories = new ArrayList<>(); diff --git a/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory index 45ff2512e73..3bcf7f942a9 100644 --- a/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory +++ b/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -21,3 +21,4 @@ org.apache.gravitino.flink.connector.store.GravitinoCatalogStoreFactory org.apache.gravitino.flink.connector.hive.GravitinoHiveCatalogFactory org.apache.gravitino.flink.connector.paimon.GravitinoPaimonCatalogFactory org.apache.gravitino.flink.connector.iceberg.GravitinoIcebergCatalogFactory +org.apache.gravitino.flink.connector.jdbc.GravitinoJdbcCatalogFactory \ No newline at end of file diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java index 8ff6f8db7a2..18176bf4a74 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java @@ -85,6 +85,14 @@ protected boolean supportsPrimaryKey() { return true; } + protected boolean supportTablePropertiesOperation() { + return true; + } + + protected String defaultDatabaseName() { + return "default"; + } + @Test public void testCreateSchema() { doWithCatalog( @@ -177,7 +185,7 @@ public void testListSchema() { TestUtils.assertTableResult( sql("SHOW DATABASES"), ResultKind.SUCCESS_WITH_CONTENT, - Row.of("default"), + Row.of(defaultDatabaseName()), Row.of(schema), Row.of(schema2), Row.of(schema3)); @@ -185,7 +193,7 @@ public void testListSchema() { String[] schemas = catalog.asSchemas().listSchemas(); Arrays.sort(schemas); Assertions.assertEquals(4, schemas.length); - Assertions.assertEquals("default", schemas[0]); + Assertions.assertEquals(defaultDatabaseName(), schemas[0]); Assertions.assertEquals(schema, schemas[1]); Assertions.assertEquals(schema2, schemas[2]); Assertions.assertEquals(schema3, schemas[3]); @@ -262,7 +270,9 @@ public void testCreateSimpleTable() { catalog.asTableCatalog().loadTable(NameIdentifier.of(databaseName, tableName)); Assertions.assertNotNull(table); Assertions.assertEquals(comment, table.comment()); - Assertions.assertEquals(value, table.properties().get(key)); + if (supportTablePropertiesOperation()) { + Assertions.assertEquals(value, table.properties().get(key)); + } Column[] columns = new Column[] { Column.of("string_type", Types.StringType.get(), "string_type", true, false, null), @@ -721,7 +731,7 @@ public void testRenameTable() { } @Test - @EnabledIf("supportTableOperation") + @EnabledIf("supportTablePropertiesOperation") public void testAlterTableProperties() { String databaseName = "test_alter_table_properties_db"; String tableName = "test_alter_table_properties"; diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java index 962f7f2f847..7e8911d06fd 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java @@ -70,8 +70,6 @@ void startUp() throws Exception { // Start Gravitino server initGravitinoEnv(); initMetalake(); - initHiveEnv(); - initHdfsEnv(); initFlinkEnv(); initCatalogEnv(); LOG.info("Startup Flink env successfully, Gravitino uri: {}.", gravitinoUri); diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java index ae2492e6e26..6aef6f9f857 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java @@ -27,6 +27,7 @@ import org.apache.gravitino.flink.connector.integration.test.FlinkCommonIT; import org.apache.gravitino.flink.connector.jdbc.JdbcPropertiesConstants; import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -43,6 +44,21 @@ public class FlinkJdbcMysqlCatalogIT extends FlinkCommonIT { protected static final String CATALOG_NAME = "test_flink_jdbc_catalog"; + @Override + protected boolean supportTablePropertiesOperation() { + return false; + } + + @Override + protected String defaultDatabaseName() { + return MYSQL_CATALOG_MYSQL_IT.name(); + } + + @Override + protected boolean supportSchemaOperationWithCommentAndOptions() { + return false; + } + @Override protected Catalog currentCatalog() { return catalog; @@ -58,6 +74,12 @@ void setup() { init(); } + @AfterAll + static void stop() { + Preconditions.checkNotNull(metalake); + metalake.dropCatalog(CATALOG_NAME, true); + } + @Override protected boolean supportDropCascade() { return true; diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 736691c4b52..8ba71d51a77 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -89,6 +89,7 @@ awaitility = "4.2.1" servlet = "3.1.0" jodd = "3.5.2" flink = "1.18.0" +flinkjdbc = "3.2.0-1.18" cglib = "2.2" ranger = "2.4.0" javax-jaxb-api = "2.3.1" @@ -277,6 +278,7 @@ google-auth-http = { group = "com.google.auth", name = "google-auth-library-oaut google-auth-credentials = { group = "com.google.auth", name = "google-auth-library-credentials", version.ref = "google-auth" } aliyun-credentials-sdk = { group='com.aliyun', name='credentials-java', version.ref='aliyun-credentials' } +flinkjdbc = {group='org.apache.flink',name='flink-connector-jdbc', version.ref='flinkjdbc'} [bundles] log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core", "log4j-12-api", "log4j-layout-template-json"] From d30d191fcbbccb7ecfc757d07a519a66081c69aa Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Tue, 4 Mar 2025 00:26:36 +0800 Subject: [PATCH 05/15] support jdbc catalog --- .../jdbc/GravitinoJdbcCatalogFactory.java | 18 +----- .../GravitinoJdbcCatalogFactoryOptions.java | 5 +- .../GravitinoMysqlJdbcCatalogFactory.java | 40 ++++++++++++ .../GravitinoPostgresJdbcCatalogFactory.java | 40 ++++++++++++ .../jdbc/JdbcPropertiesConverter.java | 19 +++--- .../jdbc/MysqlPropertiesConverter.java | 37 +++++++++++ .../jdbc/PostgresqlPropertiesConverter.java | 37 +++++++++++ .../store/GravitinoCatalogStore.java | 10 +-- .../org.apache.flink.table.factories.Factory | 3 +- .../integration/test/FlinkCommonIT.java | 62 ++++++++++++++++--- .../integration/test/FlinkEnvIT.java | 7 ++- .../test/hive/FlinkHiveCatalogIT.java | 4 +- .../test/iceberg/FlinkIcebergCatalogIT.java | 4 +- .../test/jdbc/FlinkJdbcMysqlCatalogIT.java | 49 +++++++++++++++ .../integration/test/utils/TestUtils.java | 11 +++- ...a => AbstractJdbcPropertiesConverter.java} | 10 +-- .../jdbc/TestMysqlPropertiesConverter.java | 28 +++++++++ .../TestPostgresqlPropertiesConverter.java | 28 +++++++++ 18 files changed, 359 insertions(+), 53 deletions(-) create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java rename flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/{TestJdbcPropertiesConverter.java => AbstractJdbcPropertiesConverter.java} (87%) create mode 100644 flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestMysqlPropertiesConverter.java create mode 100644 flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestPostgresqlPropertiesConverter.java diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java index 2cacd14a296..396d533be60 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java @@ -26,7 +26,6 @@ import org.apache.gravitino.Catalog; import org.apache.gravitino.flink.connector.DefaultPartitionConverter; import org.apache.gravitino.flink.connector.PartitionConverter; -import org.apache.gravitino.flink.connector.PropertiesConverter; import org.apache.gravitino.flink.connector.catalog.BaseCatalogFactory; import org.apache.gravitino.flink.connector.utils.FactoryUtils; @@ -34,7 +33,7 @@ * Factory for creating instances of {@link GravitinoJdbcCatalog}. It will be created by SPI * discovery in Flink. */ -public class GravitinoJdbcCatalogFactory implements BaseCatalogFactory { +public abstract class GravitinoJdbcCatalogFactory implements BaseCatalogFactory { @Override public org.apache.flink.table.catalog.Catalog createCatalog(Context context) { @@ -46,26 +45,11 @@ public org.apache.flink.table.catalog.Catalog createCatalog(Context context) { context, defaultDatabase, propertiesConverter(), partitionConverter()); } - @Override - public String gravitinoCatalogProvider() { - return "jdbc"; - } - - @Override - public String factoryIdentifier() { - return GravitinoJdbcCatalogFactoryOptions.IDENTIFIER; - } - @Override public Catalog.Type gravitinoCatalogType() { return Catalog.Type.RELATIONAL; } - @Override - public PropertiesConverter propertiesConverter() { - return JdbcPropertiesConverter.INSTANCE; - } - @Override public PartitionConverter partitionConverter() { return DefaultPartitionConverter.INSTANCE; diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java index f656805a229..7a8176cb9d4 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java @@ -26,7 +26,10 @@ public class GravitinoJdbcCatalogFactoryOptions { /** Identifier for the {@link GravitinoJdbcCatalog}. */ - public static final String IDENTIFIER = "gravitino-jdbc"; + public static final String MYSQL_IDENTIFIER = "gravitino-jdbc-mysql"; + + /** Identifier for the {@link GravitinoJdbcCatalog}. */ + public static final String POSTGRESQL_IDENTIFIER = "gravitino-jdbc-postgresql"; public static final ConfigOption DEFAULT_DATABASE = ConfigOptions.key(FlinkCatalogFactory.DEFAULT_DATABASE).stringType().noDefaultValue(); diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java new file mode 100644 index 00000000000..9cd3c48d9a5 --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.jdbc; + +import org.apache.gravitino.flink.connector.PropertiesConverter; + +public class GravitinoMysqlJdbcCatalogFactory extends GravitinoJdbcCatalogFactory { + + @Override + public String gravitinoCatalogProvider() { + return "jdbc-mysql"; + } + + @Override + public PropertiesConverter propertiesConverter() { + return MysqlPropertiesConverter.INSTANCE; + } + + @Override + public String factoryIdentifier() { + return GravitinoJdbcCatalogFactoryOptions.MYSQL_IDENTIFIER; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java new file mode 100644 index 00000000000..2d895a4eaae --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.jdbc; + +import org.apache.gravitino.flink.connector.PropertiesConverter; + +public class GravitinoPostgresJdbcCatalogFactory extends GravitinoJdbcCatalogFactory { + + @Override + public String gravitinoCatalogProvider() { + return "jdbc-postgresql"; + } + + @Override + public PropertiesConverter propertiesConverter() { + return PostgresqlPropertiesConverter.INSTANCE; + } + + @Override + public String factoryIdentifier() { + return GravitinoJdbcCatalogFactoryOptions.POSTGRESQL_IDENTIFIER; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java index 41a2f75715e..0dc43ff0918 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java @@ -19,13 +19,21 @@ package org.apache.gravitino.flink.connector.jdbc; +import java.util.Map; +import org.apache.flink.configuration.Configuration; import org.apache.gravitino.flink.connector.PropertiesConverter; -public class JdbcPropertiesConverter implements PropertiesConverter { +public abstract class JdbcPropertiesConverter implements PropertiesConverter { - public static final JdbcPropertiesConverter INSTANCE = new JdbcPropertiesConverter(); + protected JdbcPropertiesConverter() {} - private JdbcPropertiesConverter() {} + @Override + public Map toGravitinoCatalogProperties(Configuration flinkConf) { + Map gravitinoCatalogProperties = + PropertiesConverter.super.toGravitinoCatalogProperties(flinkConf); + gravitinoCatalogProperties.put(JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER, driverName()); + return gravitinoCatalogProperties; + } @Override public String transformPropertyToGravitinoCatalog(String configKey) { @@ -37,8 +45,5 @@ public String transformPropertyToFlinkCatalog(String configKey) { return JdbcPropertiesConstants.gravitinoToFlinkMap.get(configKey); } - @Override - public String getFlinkCatalogType() { - return GravitinoJdbcCatalogFactoryOptions.IDENTIFIER; - } + protected abstract String driverName(); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java new file mode 100644 index 00000000000..ddbe4d1f5cf --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.jdbc; + +public class MysqlPropertiesConverter extends JdbcPropertiesConverter { + + public static final MysqlPropertiesConverter INSTANCE = new MysqlPropertiesConverter(); + + private MysqlPropertiesConverter() {} + + @Override + public String driverName() { + return "com.mysql.jdbc.Driver"; + } + + @Override + public String getFlinkCatalogType() { + return GravitinoJdbcCatalogFactoryOptions.MYSQL_IDENTIFIER; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java new file mode 100644 index 00000000000..d46bedb008b --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.jdbc; + +public class PostgresqlPropertiesConverter extends JdbcPropertiesConverter { + + public static final PostgresqlPropertiesConverter INSTANCE = new PostgresqlPropertiesConverter(); + + private PostgresqlPropertiesConverter() {} + + @Override + protected String driverName() { + return "org.postgresql.Driver"; + } + + @Override + public String getFlinkCatalogType() { + return GravitinoJdbcCatalogFactoryOptions.POSTGRESQL_IDENTIFIER; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java index a60aaf30c89..4c29b7fde3b 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java @@ -121,24 +121,16 @@ private BaseCatalogFactory getCatalogFactory(Map configuration) } private BaseCatalogFactory getCatalogFactory(String provider) { - String convertedProvider = getConvertedProvider(provider); return discoverFactories( catalogFactory -> ((BaseCatalogFactory) catalogFactory) .gravitinoCatalogProvider() - .equalsIgnoreCase(convertedProvider), + .equalsIgnoreCase(provider), String.format( "Gravitino catalog provider [%s] matched multiple flink catalog factories, it should only match one.", provider)); } - private String getConvertedProvider(String provider) { - if (provider.startsWith("jdbc")) { - return "jdbc"; - } - return provider; - } - private BaseCatalogFactory discoverFactories(Predicate predicate, String errorMessage) { Iterator serviceLoaderIterator = ServiceLoader.load(Factory.class).iterator(); final List factories = new ArrayList<>(); diff --git a/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory index 3bcf7f942a9..4f373e167d1 100644 --- a/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory +++ b/flink-connector/flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -21,4 +21,5 @@ org.apache.gravitino.flink.connector.store.GravitinoCatalogStoreFactory org.apache.gravitino.flink.connector.hive.GravitinoHiveCatalogFactory org.apache.gravitino.flink.connector.paimon.GravitinoPaimonCatalogFactory org.apache.gravitino.flink.connector.iceberg.GravitinoIcebergCatalogFactory -org.apache.gravitino.flink.connector.jdbc.GravitinoJdbcCatalogFactory \ No newline at end of file +org.apache.gravitino.flink.connector.jdbc.GravitinoMysqlJdbcCatalogFactory +org.apache.gravitino.flink.connector.jdbc.GravitinoPostgresJdbcCatalogFactory diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java index 18176bf4a74..b2a63c202b6 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java @@ -89,6 +89,10 @@ protected boolean supportTablePropertiesOperation() { return true; } + protected boolean supportModifyColumnsMultipleByOneSql() { + return true; + } + protected String defaultDatabaseName() { return "default"; } @@ -278,7 +282,7 @@ public void testCreateSimpleTable() { Column.of("string_type", Types.StringType.get(), "string_type", true, false, null), Column.of("double_type", Types.DoubleType.get(), "double_type") }; - assertColumns(columns, table.columns()); + assertColumns(columns, table.columns(), getProvider()); Assertions.assertArrayEquals(EMPTY_TRANSFORM, table.partitioning()); TestUtils.assertTableResult( @@ -521,7 +525,7 @@ public void testRenameColumn() { Column.of("user_id_new", Types.IntegerType.get(), "USER_ID"), Column.of("order_amount", Types.DoubleType.get(), "ORDER_AMOUNT") }; - assertColumns(expected, actual); + assertColumns(expected, actual, getProvider()); }, true, supportDropCascade()); @@ -620,7 +624,7 @@ public void testAlterTableAddColumn() { Column.of("order_amount", Types.IntegerType.get(), "ORDER_AMOUNT"), Column.of("new_column_2", Types.IntegerType.get(), null), }; - assertColumns(expected, actual); + assertColumns(expected, actual, getProvider()); }, true, supportDropCascade()); @@ -652,15 +656,15 @@ public void testAlterTableDropColumn() { .columns(); Column[] expected = new Column[] {Column.of("order_amount", Types.IntegerType.get(), "ORDER_AMOUNT")}; - assertColumns(expected, actual); + assertColumns(expected, actual, getProvider()); }, true, supportDropCascade()); } @Test - @EnabledIf("supportColumnOperation") - public void testAlterColumnTypeAndChangeOrder() { + @EnabledIf("supportModifyColumnsMultipleByOneSql") + public void testAlterColumnTypeAndChangeOrderInOneSql() { String databaseName = "test_alter_table_alter_column_db"; String tableName = "test_alter_table_rename_column"; doWithSchema( @@ -695,7 +699,51 @@ public void testAlterColumnTypeAndChangeOrder() { Column.of("order_amount", Types.LongType.get(), "new comment2"), Column.of("user_id", Types.LongType.get(), "new comment") }; - assertColumns(expected, actual); + assertColumns(expected, actual, getProvider()); + }, + true, + supportDropCascade()); + } + + @Test + @EnabledIf("supportColumnOperation") + public void testAlterColumnTypeAndChangeOrder() { + String databaseName = "test_alter_table_alter_column_db"; + String tableName = "test_alter_table_rename_column"; + doWithSchema( + currentCatalog(), + databaseName, + catalog -> { + TableResult result = + sql( + "CREATE TABLE %s " + + "(user_id BIGINT COMMENT 'USER_ID', " + + " order_amount INT COMMENT 'ORDER_AMOUNT')" + + " COMMENT 'test comment'" + + " WITH (" + + "'%s' = '%s')", + tableName, "test key", "test value"); + TestUtils.assertTableResult(result, ResultKind.SUCCESS); + result = sql("ALTER TABLE %s MODIFY order_amount INT COMMENT 'new comment2'", tableName); + TestUtils.assertTableResult(result, ResultKind.SUCCESS); + result = + sql("ALTER TABLE %s MODIFY order_amount BIGINT COMMENT 'new comment2'", tableName); + TestUtils.assertTableResult(result, ResultKind.SUCCESS); + result = sql("ALTER TABLE %s MODIFY user_id BIGINT COMMENT 'new comment'", tableName); + TestUtils.assertTableResult(result, ResultKind.SUCCESS); + result = sql("ALTER TABLE %s MODIFY user_id BIGINT AFTER order_amount", tableName); + TestUtils.assertTableResult(result, ResultKind.SUCCESS); + Column[] actual = + catalog + .asTableCatalog() + .loadTable(NameIdentifier.of(databaseName, tableName)) + .columns(); + Column[] expected = + new Column[] { + Column.of("order_amount", Types.LongType.get(), "new comment2"), + Column.of("user_id", Types.LongType.get(), "new comment") + }; + assertColumns(expected, actual, getProvider()); }, true, supportDropCascade()); diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java index 7e8911d06fd..b813a80affb 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkEnvIT.java @@ -70,13 +70,16 @@ void startUp() throws Exception { // Start Gravitino server initGravitinoEnv(); initMetalake(); + initHiveEnv(); + initHdfsEnv(); initFlinkEnv(); initCatalogEnv(); LOG.info("Startup Flink env successfully, Gravitino uri: {}.", gravitinoUri); } @AfterAll - static void stop() { + void stop() throws Exception { + stopCatalogEnv(); stopFlinkEnv(); stopHdfsEnv(); LOG.info("Stop Flink env successfully."); @@ -84,6 +87,8 @@ static void stop() { protected void initCatalogEnv() throws Exception {}; + protected void stopCatalogEnv() throws Exception {} + protected String flinkByPass(String key) { return PropertiesConverter.FLINK_PROPERTY_PREFIX + key; } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java index 3add18211f1..372e8c0362c 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java @@ -338,7 +338,7 @@ public void testHivePartitionTable() { Column.of("string_type", Types.StringType.get(), "string_type", true, false, null), Column.of("double_type", Types.DoubleType.get(), "double_type") }; - assertColumns(columns, table.columns()); + assertColumns(columns, table.columns(), getProvider()); Transform[] partitions = new Transform[] { Transforms.identity("string_type"), Transforms.identity("double_type") @@ -461,7 +461,7 @@ public void testCreateHiveTable() { Types.StructType.Field.nullableField("k2", Types.StringType.get())), null) }; - assertColumns(columns, table.columns()); + assertColumns(columns, table.columns(), getProvider()); Assertions.assertArrayEquals(EMPTY_TRANSFORM, table.partitioning()); }, true); diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java index f8a3cdf2e17..4a2d082a4e7 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java @@ -265,7 +265,7 @@ public void testIcebergTableWithPartition() { Column.of("id", Types.LongType.get(), "unique id", true, false, null), Column.of("data", Types.StringType.get(), null, false, false, null) }; - assertColumns(columns, table.columns()); + assertColumns(columns, table.columns(), getProvider()); Transform[] partitions = new Transform[] {Transforms.identity("data")}; Assertions.assertArrayEquals(partitions, table.partitioning()); @@ -364,7 +364,7 @@ public void testCreateIcebergTable() { Types.StructType.Field.nullableField("k2", Types.StringType.get())), null) }; - assertColumns(columns, table.columns()); + assertColumns(columns, table.columns(), getProvider()); Assertions.assertArrayEquals(EMPTY_TRANSFORM, table.partitioning()); }, true, diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java index 6aef6f9f857..59e372e0e28 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java @@ -23,13 +23,17 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import java.util.Map; import org.apache.gravitino.Catalog; import org.apache.gravitino.flink.connector.integration.test.FlinkCommonIT; import org.apache.gravitino.flink.connector.jdbc.JdbcPropertiesConstants; +import org.apache.gravitino.flink.connector.jdbc.MysqlPropertiesConverter; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") public class FlinkJdbcMysqlCatalogIT extends FlinkCommonIT { @@ -59,6 +63,11 @@ protected boolean supportSchemaOperationWithCommentAndOptions() { return false; } + @Override + protected boolean supportModifyColumnsMultipleByOneSql() { + return false; + } + @Override protected Catalog currentCatalog() { return catalog; @@ -115,4 +124,44 @@ protected void initCatalogEnv() throws Exception { mysqlPassword = containerSuite.getMySQLContainer().getPassword(); mysqlDriver = containerSuite.getMySQLContainer().getDriverClassName(MYSQL_CATALOG_MYSQL_IT); } + + @Override + protected void stopCatalogEnv() throws Exception { + if (null != containerSuite) { + containerSuite.close(); + } + } + + @Test + public void testCreateGravitinoJdbcCatalogUsingSQL() { + tableEnv.useCatalog(DEFAULT_CATALOG); + int numCatalogs = tableEnv.listCatalogs().length; + String catalogName = "gravitino_mysql_jdbc_catalog"; + tableEnv.executeSql( + String.format( + "create catalog %s with (" + + "'type'='gravitino-jdbc-mysql', " + + "'base-url'='%s'," + + "'username'='%s'," + + "'password'='%s'," + + "'default-database'='%s'" + + ")", + catalogName, mysqlUrl, mysqlUsername, mysqlPassword, mysqlDefaultDatabase)); + String[] catalogs = tableEnv.listCatalogs(); + Assertions.assertEquals(numCatalogs + 1, catalogs.length, "Should create a new catalog"); + Assertions.assertTrue(metalake.catalogExists(catalogName)); + org.apache.gravitino.Catalog gravitinoCatalog = metalake.loadCatalog(catalogName); + Map properties = gravitinoCatalog.properties(); + Assertions.assertEquals(mysqlUrl, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_URL)); + Assertions.assertEquals( + mysqlUsername, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_USER)); + Assertions.assertEquals( + mysqlPassword, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_USER)); + Assertions.assertEquals( + mysqlDefaultDatabase, + properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE)); + Assertions.assertEquals( + MysqlPropertiesConverter.INSTANCE.driverName(), + properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER)); + } } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java index 02710bcfb35..88faf7df541 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java @@ -26,6 +26,7 @@ import org.apache.flink.table.types.DataType; import org.apache.flink.types.Row; import org.apache.gravitino.rel.Column; +import org.apache.gravitino.rel.expressions.literals.Literals; import org.junit.jupiter.api.Assertions; public class TestUtils { @@ -48,14 +49,20 @@ public static void assertTableResult( } } - public static void assertColumns(Column[] expected, Column[] actual) { + public static void assertColumns(Column[] expected, Column[] actual, String provider) { Assertions.assertEquals(expected.length, actual.length); for (int i = 0; i < expected.length; i++) { Assertions.assertEquals(expected[i].name(), actual[i].name()); Assertions.assertEquals(expected[i].comment(), actual[i].comment()); Assertions.assertEquals( expected[i].dataType().simpleString(), actual[i].dataType().simpleString()); - Assertions.assertEquals(expected[i].defaultValue(), actual[i].defaultValue()); + if (expected[i].defaultValue().equals(Column.DEFAULT_VALUE_NOT_SET) + && expected[i].nullable() + && provider.startsWith("jdbc")) { + Assertions.assertEquals(Literals.NULL, actual[i].defaultValue()); + } else { + Assertions.assertEquals(expected[i].defaultValue(), actual[i].defaultValue()); + } Assertions.assertEquals(expected[i].autoIncrement(), actual[i].autoIncrement()); Assertions.assertEquals(expected[i].nullable(), actual[i].nullable()); } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/AbstractJdbcPropertiesConverter.java similarity index 87% rename from flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java rename to flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/AbstractJdbcPropertiesConverter.java index 1b56c63c224..019ac592609 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestJdbcPropertiesConverter.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/AbstractJdbcPropertiesConverter.java @@ -26,14 +26,14 @@ import org.junit.jupiter.api.Test; /** Test for {@link JdbcPropertiesConverter} */ -public class TestJdbcPropertiesConverter { +public abstract class AbstractJdbcPropertiesConverter { String username = "testUser"; String password = "testPassword"; String url = "testUrl"; String defaultDatabase = "test"; - private static final JdbcPropertiesConverter CONVERTER = JdbcPropertiesConverter.INSTANCE; + protected abstract JdbcPropertiesConverter getConverter(); @Test public void testToPaimonFileSystemCatalog() { @@ -47,7 +47,7 @@ public void testToPaimonFileSystemCatalog() { url, JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE, defaultDatabase); - Map properties = CONVERTER.toFlinkCatalogProperties(catalogProperties); + Map properties = getConverter().toFlinkCatalogProperties(catalogProperties); Assertions.assertEquals(username, properties.get(JdbcPropertiesConstants.FLINK_JDBC_USER)); Assertions.assertEquals(password, properties.get(JdbcPropertiesConstants.FLINK_JDBC_PASSWORD)); Assertions.assertEquals(url, properties.get(JdbcPropertiesConstants.FLINK_JDBC_URL)); @@ -68,7 +68,7 @@ public void testToGravitinoCatalogProperties() { url, JdbcPropertiesConstants.FLINK_JDBC_DEFAULT_DATABASE, defaultDatabase)); - Map properties = CONVERTER.toGravitinoCatalogProperties(configuration); + Map properties = getConverter().toGravitinoCatalogProperties(configuration); Assertions.assertEquals(username, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_USER)); Assertions.assertEquals( @@ -76,5 +76,7 @@ public void testToGravitinoCatalogProperties() { Assertions.assertEquals(url, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_URL)); Assertions.assertEquals( defaultDatabase, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE)); + Assertions.assertEquals( + getConverter().driverName(), properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER)); } } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestMysqlPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestMysqlPropertiesConverter.java new file mode 100644 index 00000000000..b62c39d3360 --- /dev/null +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestMysqlPropertiesConverter.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.jdbc; + +public class TestMysqlPropertiesConverter extends AbstractJdbcPropertiesConverter { + + @Override + protected JdbcPropertiesConverter getConverter() { + return MysqlPropertiesConverter.INSTANCE; + } +} diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestPostgresqlPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestPostgresqlPropertiesConverter.java new file mode 100644 index 00000000000..206a09d1e8e --- /dev/null +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestPostgresqlPropertiesConverter.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector.jdbc; + +public class TestPostgresqlPropertiesConverter extends AbstractJdbcPropertiesConverter { + + @Override + protected JdbcPropertiesConverter getConverter() { + return PostgresqlPropertiesConverter.INSTANCE; + } +} From d8d55d7fcdfc30228ab0d65c30a40589df2f0d14 Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Tue, 4 Mar 2025 21:16:04 +0800 Subject: [PATCH 06/15] check not null --- flink-connector/flink/build.gradle.kts | 1 + .../flink/connector/jdbc/GravitinoJdbcCatalogFactory.java | 2 ++ .../connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java | 3 +-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flink-connector/flink/build.gradle.kts b/flink-connector/flink/build.gradle.kts index 33cf15889a7..5e22c2c554c 100644 --- a/flink-connector/flink/build.gradle.kts +++ b/flink-connector/flink/build.gradle.kts @@ -29,6 +29,7 @@ repositories { var paimonVersion: String = libs.versions.paimon.get() val flinkVersion: String = libs.versions.flink.get() val flinkMajorVersion: String = flinkVersion.substringBeforeLast(".") + val icebergVersion: String = libs.versions.iceberg.get() // The Flink only support scala 2.12, and all scala api will be removed in a future version. diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java index 396d533be60..f85cdfceff9 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java @@ -23,6 +23,7 @@ import java.util.Set; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.util.Preconditions; import org.apache.gravitino.Catalog; import org.apache.gravitino.flink.connector.DefaultPartitionConverter; import org.apache.gravitino.flink.connector.PartitionConverter; @@ -41,6 +42,7 @@ public org.apache.flink.table.catalog.Catalog createCatalog(Context context) { FactoryUtils.createCatalogFactoryHelper(this, context); String defaultDatabase = helper.getOptions().get(GravitinoJdbcCatalogFactoryOptions.DEFAULT_DATABASE); + Preconditions.checkNotNull(defaultDatabase, "default-database should not be null."); return new GravitinoJdbcCatalog( context, defaultDatabase, propertiesConverter(), partitionConverter()); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java index 7a8176cb9d4..b85775bf9db 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactoryOptions.java @@ -21,7 +21,6 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; -import org.apache.iceberg.flink.FlinkCatalogFactory; public class GravitinoJdbcCatalogFactoryOptions { @@ -32,5 +31,5 @@ public class GravitinoJdbcCatalogFactoryOptions { public static final String POSTGRESQL_IDENTIFIER = "gravitino-jdbc-postgresql"; public static final ConfigOption DEFAULT_DATABASE = - ConfigOptions.key(FlinkCatalogFactory.DEFAULT_DATABASE).stringType().noDefaultValue(); + ConfigOptions.key("default-database").stringType().noDefaultValue(); } From f7eee3975929e21800973d1a70322ba709e0ce95 Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Wed, 5 Mar 2025 22:46:22 +0800 Subject: [PATCH 07/15] check not null --- flink-connector/flink/build.gradle.kts | 1 + .../flink/connector/catalog/BaseCatalog.java | 11 +++++++++-- .../flink/connector/jdbc/GravitinoJdbcCatalog.java | 12 ++++++++++++ .../connector/jdbc/GravitinoJdbcCatalogFactory.java | 7 ++++--- .../test/jdbc/FlinkJdbcMysqlCatalogIT.java | 4 ++-- .../test/paimon/FlinkPaimonCatalogIT.java | 2 +- 6 files changed, 29 insertions(+), 8 deletions(-) diff --git a/flink-connector/flink/build.gradle.kts b/flink-connector/flink/build.gradle.kts index 5e22c2c554c..34a1cfa5b0b 100644 --- a/flink-connector/flink/build.gradle.kts +++ b/flink-connector/flink/build.gradle.kts @@ -99,6 +99,7 @@ dependencies { testImplementation(libs.testcontainers.mysql) testImplementation(libs.metrics.core) testImplementation(libs.flinkjdbc) + testImplementation("org.apache.iceberg:iceberg-flink-runtime-$flinkMajorVersion:$icebergVersion") testImplementation("org.apache.flink:flink-connector-hive_$scalaVersion:$flinkVersion") testImplementation("org.apache.flink:flink-table-common:$flinkVersion") diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java index e9320c786cd..13ef4d549d8 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java @@ -23,6 +23,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -280,8 +281,11 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig String comment = table.getComment(); Map properties = propertiesConverter.toGravitinoTableProperties(table.getOptions()); + Transform[] partitions = - partitionConverter.toGravitinoPartitions(((CatalogTable) table).getPartitionKeys()); + partitionConverter != null + ? partitionConverter.toGravitinoPartitions(((CatalogTable) table).getPartitionKeys()) + : new Transform[0]; try { @@ -558,7 +562,10 @@ protected CatalogBaseTable toFlinkTable(Table table) { flinkPrimaryKey.ifPresent(builder::primaryKey); Map flinkTableProperties = propertiesConverter.toFlinkTableProperties(table.properties()); - List partitionKeys = partitionConverter.toFlinkPartitionKeys(table.partitioning()); + List partitionKeys = + partitionConverter != null + ? partitionConverter.toFlinkPartitionKeys(table.partitioning()) + : new ArrayList<>(); return CatalogTable.of(builder.build(), table.comment(), partitionKeys, flinkTableProperties); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java index 2b56116b49d..25ea1c50186 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java @@ -56,6 +56,18 @@ protected GravitinoJdbcCatalog( this.jdbcCatalog = (JdbcCatalog) jdbcCatalogFactory.createCatalog(context); } + @Override + public void open() throws CatalogException { + super.open(); + jdbcCatalog.open(); + } + + @Override + public void close() throws CatalogException { + super.close(); + jdbcCatalog.close(); + } + @Override protected AbstractCatalog realCatalog() { return jdbcCatalog; diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java index f85cdfceff9..3eec9460be7 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java @@ -25,7 +25,6 @@ import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.util.Preconditions; import org.apache.gravitino.Catalog; -import org.apache.gravitino.flink.connector.DefaultPartitionConverter; import org.apache.gravitino.flink.connector.PartitionConverter; import org.apache.gravitino.flink.connector.catalog.BaseCatalogFactory; import org.apache.gravitino.flink.connector.utils.FactoryUtils; @@ -42,7 +41,9 @@ public org.apache.flink.table.catalog.Catalog createCatalog(Context context) { FactoryUtils.createCatalogFactoryHelper(this, context); String defaultDatabase = helper.getOptions().get(GravitinoJdbcCatalogFactoryOptions.DEFAULT_DATABASE); - Preconditions.checkNotNull(defaultDatabase, "default-database should not be null."); + Preconditions.checkNotNull( + defaultDatabase, + GravitinoJdbcCatalogFactoryOptions.DEFAULT_DATABASE.key() + " should not be null."); return new GravitinoJdbcCatalog( context, defaultDatabase, propertiesConverter(), partitionConverter()); } @@ -54,7 +55,7 @@ public Catalog.Type gravitinoCatalogType() { @Override public PartitionConverter partitionConverter() { - return DefaultPartitionConverter.INSTANCE; + return null; } @Override diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java index 59e372e0e28..0e19ac2b544 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java @@ -79,12 +79,12 @@ protected String getProvider() { } @BeforeAll - void setup() { + void jdbcStartup() { init(); } @AfterAll - static void stop() { + void jdbcStop() { Preconditions.checkNotNull(metalake); metalake.dropCatalog(CATALOG_NAME, true); } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java index 66458ba8e74..948e02dd818 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/paimon/FlinkPaimonCatalogIT.java @@ -67,7 +67,7 @@ void setup() { } @AfterAll - static void stop() { + void stop() { Preconditions.checkNotNull(metalake); metalake.dropCatalog(DEFAULT_PAIMON_CATALOG, true); } From 86039e46fcab46b441870d800be78c2d7c94049b Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Mon, 10 Mar 2025 20:57:59 +0800 Subject: [PATCH 08/15] remove open and close --- .../flink/connector/jdbc/GravitinoJdbcCatalog.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java index 25ea1c50186..2b56116b49d 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java @@ -56,18 +56,6 @@ protected GravitinoJdbcCatalog( this.jdbcCatalog = (JdbcCatalog) jdbcCatalogFactory.createCatalog(context); } - @Override - public void open() throws CatalogException { - super.open(); - jdbcCatalog.open(); - } - - @Override - public void close() throws CatalogException { - super.close(); - jdbcCatalog.close(); - } - @Override protected AbstractCatalog realCatalog() { return jdbcCatalog; From d529f07b14f826eb225c47168b4311b343088f50 Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Wed, 12 Mar 2025 00:20:12 +0800 Subject: [PATCH 09/15] fix --- .../UnsupportPartitionConverter.java | 48 +++++++++++++++++++ .../flink/connector/catalog/BaseCatalog.java | 10 +--- .../jdbc/GravitinoJdbcCatalogFactory.java | 3 +- .../integration/test/FlinkCommonIT.java | 37 +++++++++++--- .../test/hive/FlinkHiveCatalogIT.java | 5 +- .../test/iceberg/FlinkIcebergCatalogIT.java | 5 +- .../test/jdbc/FlinkJdbcMysqlCatalogIT.java | 5 ++ .../integration/test/utils/TestUtils.java | 21 -------- 8 files changed, 91 insertions(+), 43 deletions(-) create mode 100644 flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/UnsupportPartitionConverter.java diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/UnsupportPartitionConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/UnsupportPartitionConverter.java new file mode 100644 index 00000000000..e6c4d0a3c13 --- /dev/null +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/UnsupportPartitionConverter.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.flink.connector; + +import com.google.common.base.Preconditions; +import java.util.Collections; +import java.util.List; +import org.apache.gravitino.rel.expressions.transforms.Transform; + +/** Suitable for Catalog types that do not support partition keys. */ +public class UnsupportPartitionConverter implements PartitionConverter { + + private UnsupportPartitionConverter() {} + + public static final UnsupportPartitionConverter INSTANCE = new UnsupportPartitionConverter(); + + @Override + public List toFlinkPartitionKeys(Transform[] partitions) { + Preconditions.checkArgument( + partitions != null && partitions.length > 0, "Partition key conversion is not supported."); + return Collections.emptyList(); + } + + @Override + public Transform[] toGravitinoPartitions(List partitionsKey) { + Preconditions.checkArgument( + partitionsKey != null && !partitionsKey.isEmpty(), + "Partition key conversion is not supported."); + return new Transform[0]; + } +} diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java index b31fd1bdc85..12e91da884a 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java @@ -23,7 +23,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -287,9 +286,7 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig propertiesConverter.toGravitinoTableProperties(table.getOptions()); Transform[] partitions = - partitionConverter != null - ? partitionConverter.toGravitinoPartitions(((CatalogTable) table).getPartitionKeys()) - : new Transform[0]; + partitionConverter.toGravitinoPartitions(((CatalogTable) table).getPartitionKeys()); try { @@ -566,10 +563,7 @@ protected CatalogBaseTable toFlinkTable(Table table) { flinkPrimaryKey.ifPresent(builder::primaryKey); Map flinkTableProperties = propertiesConverter.toFlinkTableProperties(table.properties()); - List partitionKeys = - partitionConverter != null - ? partitionConverter.toFlinkPartitionKeys(table.partitioning()) - : new ArrayList<>(); + List partitionKeys = partitionConverter.toFlinkPartitionKeys(table.partitioning()); return CatalogTable.of(builder.build(), table.comment(), partitionKeys, flinkTableProperties); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java index 3eec9460be7..cd9d9743025 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java @@ -26,6 +26,7 @@ import org.apache.flink.util.Preconditions; import org.apache.gravitino.Catalog; import org.apache.gravitino.flink.connector.PartitionConverter; +import org.apache.gravitino.flink.connector.UnsupportPartitionConverter; import org.apache.gravitino.flink.connector.catalog.BaseCatalogFactory; import org.apache.gravitino.flink.connector.utils.FactoryUtils; @@ -55,7 +56,7 @@ public Catalog.Type gravitinoCatalogType() { @Override public PartitionConverter partitionConverter() { - return null; + return UnsupportPartitionConverter.INSTANCE; } @Override diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java index b2a63c202b6..2a687b5c7fd 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/FlinkCommonIT.java @@ -19,7 +19,6 @@ package org.apache.gravitino.flink.connector.integration.test; -import static org.apache.gravitino.flink.connector.integration.test.utils.TestUtils.assertColumns; import static org.apache.gravitino.flink.connector.integration.test.utils.TestUtils.toFlinkPhysicalColumn; import static org.apache.gravitino.rel.expressions.transforms.Transforms.EMPTY_TRANSFORM; import static org.junit.jupiter.api.Assertions.fail; @@ -51,6 +50,7 @@ import org.apache.gravitino.flink.connector.integration.test.utils.TestUtils; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; +import org.apache.gravitino.rel.expressions.literals.Literals; import org.apache.gravitino.rel.indexes.Index; import org.apache.gravitino.rel.types.Types; import org.junit.jupiter.api.Assertions; @@ -93,6 +93,10 @@ protected boolean supportModifyColumnsMultipleByOneSql() { return true; } + protected boolean defaultValueWithNullLiterals() { + return false; + } + protected String defaultDatabaseName() { return "default"; } @@ -282,7 +286,7 @@ public void testCreateSimpleTable() { Column.of("string_type", Types.StringType.get(), "string_type", true, false, null), Column.of("double_type", Types.DoubleType.get(), "double_type") }; - assertColumns(columns, table.columns(), getProvider()); + assertColumns(columns, table.columns()); Assertions.assertArrayEquals(EMPTY_TRANSFORM, table.partitioning()); TestUtils.assertTableResult( @@ -525,7 +529,7 @@ public void testRenameColumn() { Column.of("user_id_new", Types.IntegerType.get(), "USER_ID"), Column.of("order_amount", Types.DoubleType.get(), "ORDER_AMOUNT") }; - assertColumns(expected, actual, getProvider()); + assertColumns(expected, actual); }, true, supportDropCascade()); @@ -624,7 +628,7 @@ public void testAlterTableAddColumn() { Column.of("order_amount", Types.IntegerType.get(), "ORDER_AMOUNT"), Column.of("new_column_2", Types.IntegerType.get(), null), }; - assertColumns(expected, actual, getProvider()); + assertColumns(expected, actual); }, true, supportDropCascade()); @@ -656,7 +660,7 @@ public void testAlterTableDropColumn() { .columns(); Column[] expected = new Column[] {Column.of("order_amount", Types.IntegerType.get(), "ORDER_AMOUNT")}; - assertColumns(expected, actual, getProvider()); + assertColumns(expected, actual); }, true, supportDropCascade()); @@ -699,7 +703,7 @@ public void testAlterColumnTypeAndChangeOrderInOneSql() { Column.of("order_amount", Types.LongType.get(), "new comment2"), Column.of("user_id", Types.LongType.get(), "new comment") }; - assertColumns(expected, actual, getProvider()); + assertColumns(expected, actual); }, true, supportDropCascade()); @@ -743,7 +747,7 @@ public void testAlterColumnTypeAndChangeOrder() { Column.of("order_amount", Types.LongType.get(), "new comment2"), Column.of("user_id", Types.LongType.get(), "new comment") }; - assertColumns(expected, actual, getProvider()); + assertColumns(expected, actual); }, true, supportDropCascade()); @@ -821,4 +825,23 @@ public void testAlterTableProperties() { true, supportDropCascade()); } + + public void assertColumns(Column[] expected, Column[] actual) { + Assertions.assertEquals(expected.length, actual.length); + for (int i = 0; i < expected.length; i++) { + Assertions.assertEquals(expected[i].name(), actual[i].name()); + Assertions.assertEquals(expected[i].comment(), actual[i].comment()); + Assertions.assertEquals( + expected[i].dataType().simpleString(), actual[i].dataType().simpleString()); + if (expected[i].defaultValue().equals(Column.DEFAULT_VALUE_NOT_SET) + && expected[i].nullable() + && defaultValueWithNullLiterals()) { + Assertions.assertEquals(Literals.NULL, actual[i].defaultValue()); + } else { + Assertions.assertEquals(expected[i].defaultValue(), actual[i].defaultValue()); + } + Assertions.assertEquals(expected[i].autoIncrement(), actual[i].autoIncrement()); + Assertions.assertEquals(expected[i].nullable(), actual[i].nullable()); + } + } } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java index 372e8c0362c..1b9d321370e 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/hive/FlinkHiveCatalogIT.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.flink.connector.integration.test.hive; -import static org.apache.gravitino.flink.connector.integration.test.utils.TestUtils.assertColumns; import static org.apache.gravitino.flink.connector.integration.test.utils.TestUtils.toFlinkPhysicalColumn; import static org.apache.gravitino.rel.expressions.transforms.Transforms.EMPTY_TRANSFORM; @@ -338,7 +337,7 @@ public void testHivePartitionTable() { Column.of("string_type", Types.StringType.get(), "string_type", true, false, null), Column.of("double_type", Types.DoubleType.get(), "double_type") }; - assertColumns(columns, table.columns(), getProvider()); + assertColumns(columns, table.columns()); Transform[] partitions = new Transform[] { Transforms.identity("string_type"), Transforms.identity("double_type") @@ -461,7 +460,7 @@ public void testCreateHiveTable() { Types.StructType.Field.nullableField("k2", Types.StringType.get())), null) }; - assertColumns(columns, table.columns(), getProvider()); + assertColumns(columns, table.columns()); Assertions.assertArrayEquals(EMPTY_TRANSFORM, table.partitioning()); }, true); diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java index 4a2d082a4e7..78537fa080c 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/iceberg/FlinkIcebergCatalogIT.java @@ -19,7 +19,6 @@ package org.apache.gravitino.flink.connector.integration.test.iceberg; -import static org.apache.gravitino.flink.connector.integration.test.utils.TestUtils.assertColumns; import static org.apache.gravitino.flink.connector.integration.test.utils.TestUtils.toFlinkPhysicalColumn; import static org.apache.gravitino.rel.expressions.transforms.Transforms.EMPTY_TRANSFORM; @@ -265,7 +264,7 @@ public void testIcebergTableWithPartition() { Column.of("id", Types.LongType.get(), "unique id", true, false, null), Column.of("data", Types.StringType.get(), null, false, false, null) }; - assertColumns(columns, table.columns(), getProvider()); + assertColumns(columns, table.columns()); Transform[] partitions = new Transform[] {Transforms.identity("data")}; Assertions.assertArrayEquals(partitions, table.partitioning()); @@ -364,7 +363,7 @@ public void testCreateIcebergTable() { Types.StructType.Field.nullableField("k2", Types.StringType.get())), null) }; - assertColumns(columns, table.columns(), getProvider()); + assertColumns(columns, table.columns()); Assertions.assertArrayEquals(EMPTY_TRANSFORM, table.partitioning()); }, true, diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java index 0e19ac2b544..ec3932e1385 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java @@ -94,6 +94,11 @@ protected boolean supportDropCascade() { return true; } + @Override + protected boolean defaultValueWithNullLiterals() { + return true; + } + private void init() { Preconditions.checkNotNull(metalake); catalog = diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java index 88faf7df541..66e59ce41d1 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/utils/TestUtils.java @@ -25,8 +25,6 @@ import org.apache.flink.table.api.TableResult; import org.apache.flink.table.types.DataType; import org.apache.flink.types.Row; -import org.apache.gravitino.rel.Column; -import org.apache.gravitino.rel.expressions.literals.Literals; import org.junit.jupiter.api.Assertions; public class TestUtils { @@ -49,25 +47,6 @@ public static void assertTableResult( } } - public static void assertColumns(Column[] expected, Column[] actual, String provider) { - Assertions.assertEquals(expected.length, actual.length); - for (int i = 0; i < expected.length; i++) { - Assertions.assertEquals(expected[i].name(), actual[i].name()); - Assertions.assertEquals(expected[i].comment(), actual[i].comment()); - Assertions.assertEquals( - expected[i].dataType().simpleString(), actual[i].dataType().simpleString()); - if (expected[i].defaultValue().equals(Column.DEFAULT_VALUE_NOT_SET) - && expected[i].nullable() - && provider.startsWith("jdbc")) { - Assertions.assertEquals(Literals.NULL, actual[i].defaultValue()); - } else { - Assertions.assertEquals(expected[i].defaultValue(), actual[i].defaultValue()); - } - Assertions.assertEquals(expected[i].autoIncrement(), actual[i].autoIncrement()); - Assertions.assertEquals(expected[i].nullable(), actual[i].nullable()); - } - } - public static org.apache.flink.table.catalog.Column[] toFlinkPhysicalColumn( List unresolvedPhysicalColumns) { return unresolvedPhysicalColumns.stream() From 67d731c940fdf3acb1833128c59df816c973ca92 Mon Sep 17 00:00:00 2001 From: "1161623489@qq.com" <1161623489@qq.com> Date: Wed, 12 Mar 2025 09:14:24 +0800 Subject: [PATCH 10/15] fixci --- .../flink/connector/UnsupportPartitionConverter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/UnsupportPartitionConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/UnsupportPartitionConverter.java index e6c4d0a3c13..753404a2f3a 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/UnsupportPartitionConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/UnsupportPartitionConverter.java @@ -34,14 +34,14 @@ private UnsupportPartitionConverter() {} @Override public List toFlinkPartitionKeys(Transform[] partitions) { Preconditions.checkArgument( - partitions != null && partitions.length > 0, "Partition key conversion is not supported."); + partitions == null || partitions.length == 0, "Partition key conversion is not supported."); return Collections.emptyList(); } @Override public Transform[] toGravitinoPartitions(List partitionsKey) { Preconditions.checkArgument( - partitionsKey != null && !partitionsKey.isEmpty(), + partitionsKey == null || partitionsKey.isEmpty(), "Partition key conversion is not supported."); return new Transform[0]; } From 9bb4078d81150375944934d07d8901f82f446a06 Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Wed, 12 Mar 2025 19:27:50 +0800 Subject: [PATCH 11/15] fix --- .../gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java index 03a9b333eda..2b685fccd5f 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConstants.java @@ -30,7 +30,7 @@ private JdbcPropertiesConstants() {} public static final String GRAVITINO_JDBC_PASSWORD = "jdbc-password"; public static final String GRAVITINO_JDBC_URL = "jdbc-url"; public static final String GRAVITINO_JDBC_DRIVER = "jdbc-driver"; - public static final String GRAVITINO_JDBC_DEFAULT_DATABASE = "default-database"; + public static final String GRAVITINO_JDBC_DEFAULT_DATABASE = "flink.bypass.default-database"; public static final String FLINK_JDBC_URL = "base-url"; public static final String FLINK_JDBC_USER = "username"; From deac3d4db89fd36ec567def048f906a4ffc8e274 Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Wed, 12 Mar 2025 20:07:27 +0800 Subject: [PATCH 12/15] Optimize the logic of jdbcCatalog's getTable method. --- .../flink/connector/PropertiesConverter.java | 4 ++- .../flink/connector/catalog/BaseCatalog.java | 6 ++--- .../connector/catalog/BaseCatalogFactory.java | 2 +- .../hive/GravitinoHiveCatalogFactory.java | 4 +-- .../GravitinoIcebergCatalogFactory.java | 4 +-- .../connector/jdbc/GravitinoJdbcCatalog.java | 23 ----------------- .../GravitinoMysqlJdbcCatalogFactory.java | 4 +-- .../GravitinoPostgresJdbcCatalogFactory.java | 4 +-- .../jdbc/JdbcPropertiesConverter.java | 25 ++++++++++++++++++- .../jdbc/MysqlPropertiesConverter.java | 8 +++--- .../jdbc/PostgresqlPropertiesConverter.java | 8 +++--- .../paimon/GravitinoPaimonCatalogFactory.java | 4 +-- 12 files changed, 51 insertions(+), 45 deletions(-) diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java index 15d1a12fa3b..5e135f4f448 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/PropertiesConverter.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.catalog.CommonCatalogOptions; +import org.apache.flink.table.catalog.ObjectPath; /** * PropertiesConverter is used to convert properties between Flink properties and Apache Gravitino @@ -136,7 +137,8 @@ default Map toFlinkDatabaseProperties(Map gravit * @param gravitinoProperties The table properties provided by Gravitino. * @return The table properties for the Flink connector. */ - default Map toFlinkTableProperties(Map gravitinoProperties) { + default Map toFlinkTableProperties( + Map gravitinoProperties, ObjectPath tablePath) { return gravitinoProperties; } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java index 12e91da884a..6501f20b0b0 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java @@ -211,7 +211,7 @@ public CatalogBaseTable getTable(ObjectPath tablePath) catalog() .asTableCatalog() .loadTable(NameIdentifier.of(tablePath.getDatabaseName(), tablePath.getObjectName())); - return toFlinkTable(table); + return toFlinkTable(table, tablePath); } catch (NoSuchTableException e) { throw new TableNotExistException(catalogName(), tablePath, e); } catch (Exception e) { @@ -550,7 +550,7 @@ public void alterPartitionColumnStatistics( throw new UnsupportedOperationException(); } - protected CatalogBaseTable toFlinkTable(Table table) { + protected CatalogBaseTable toFlinkTable(Table table, ObjectPath tablePath) { org.apache.flink.table.api.Schema.Builder builder = org.apache.flink.table.api.Schema.newBuilder(); for (Column column : table.columns()) { @@ -562,7 +562,7 @@ protected CatalogBaseTable toFlinkTable(Table table) { Optional> flinkPrimaryKey = getFlinkPrimaryKey(table); flinkPrimaryKey.ifPresent(builder::primaryKey); Map flinkTableProperties = - propertiesConverter.toFlinkTableProperties(table.properties()); + propertiesConverter.toFlinkTableProperties(table.properties(), tablePath); List partitionKeys = partitionConverter.toFlinkPartitionKeys(table.partitioning()); return CatalogTable.of(builder.build(), table.comment(), partitionKeys, flinkTableProperties); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalogFactory.java index 5086b532571..40b12429d79 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalogFactory.java @@ -45,7 +45,7 @@ public interface BaseCatalogFactory extends CatalogFactory { * * @return The requested property converter. */ - PropertiesConverter propertiesConverter(); + PropertiesConverter propertiesConverter(Context context); /** * Define partition converter. diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java index 23607ebb402..aabe875eb3b 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java @@ -63,7 +63,7 @@ public Catalog createCatalog(Context context) { return new GravitinoHiveCatalog( context.getName(), helper.getOptions().get(HiveCatalogFactoryOptions.DEFAULT_DATABASE), - propertiesConverter(), + propertiesConverter(context), partitionConverter(), hiveConf, helper.getOptions().get(HiveCatalogFactoryOptions.HIVE_VERSION)); @@ -113,7 +113,7 @@ public org.apache.gravitino.Catalog.Type gravitinoCatalogType() { * @return The requested property converter. */ @Override - public PropertiesConverter propertiesConverter() { + public PropertiesConverter propertiesConverter(Context context) { return HivePropertiesConverter.INSTANCE; } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/GravitinoIcebergCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/GravitinoIcebergCatalogFactory.java index ad0363d9867..150f22cd1d0 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/GravitinoIcebergCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/GravitinoIcebergCatalogFactory.java @@ -38,7 +38,7 @@ public Catalog createCatalog(Context context) { return new GravitinoIcebergCatalog( context.getName(), helper.getOptions().get(GravitinoIcebergCatalogFactoryOptions.DEFAULT_DATABASE), - propertiesConverter(), + propertiesConverter(context), partitionConverter(), context.getOptions()); } @@ -84,7 +84,7 @@ public org.apache.gravitino.Catalog.Type gravitinoCatalogType() { * @return */ @Override - public PropertiesConverter propertiesConverter() { + public PropertiesConverter propertiesConverter(Context context) { return IcebergPropertiesConverter.INSTANCE; } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java index 2b56116b49d..53ba8d1c1b0 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java @@ -19,16 +19,11 @@ package org.apache.gravitino.flink.connector.jdbc; -import java.util.Map; import java.util.Optional; import org.apache.flink.connector.jdbc.catalog.JdbcCatalog; import org.apache.flink.connector.jdbc.catalog.factory.JdbcCatalogFactory; import org.apache.flink.connector.jdbc.table.JdbcDynamicTableFactory; import org.apache.flink.table.catalog.AbstractCatalog; -import org.apache.flink.table.catalog.CatalogBaseTable; -import org.apache.flink.table.catalog.ObjectPath; -import org.apache.flink.table.catalog.exceptions.CatalogException; -import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.flink.table.factories.CatalogFactory; import org.apache.flink.table.factories.Factory; import org.apache.gravitino.flink.connector.PartitionConverter; @@ -65,22 +60,4 @@ protected AbstractCatalog realCatalog() { public Optional getFactory() { return Optional.of(new JdbcDynamicTableFactory()); } - - @Override - public CatalogBaseTable getTable(ObjectPath tablePath) - throws TableNotExistException, CatalogException { - CatalogBaseTable table = super.getTable(tablePath); - Map contextOptions = context.getOptions(); - Map tableOptions = table.getOptions(); - tableOptions.remove("engine"); - tableOptions.put( - "url", - contextOptions.get(JdbcPropertiesConstants.FLINK_JDBC_URL) - + "/" - + tablePath.getDatabaseName()); - tableOptions.put("table-name", tablePath.getObjectName()); - tableOptions.put("username", contextOptions.get(JdbcPropertiesConstants.FLINK_JDBC_USER)); - tableOptions.put("password", contextOptions.get(JdbcPropertiesConstants.FLINK_JDBC_PASSWORD)); - return table; - } } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java index 9cd3c48d9a5..ea8ce2ab3dd 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java @@ -29,8 +29,8 @@ public String gravitinoCatalogProvider() { } @Override - public PropertiesConverter propertiesConverter() { - return MysqlPropertiesConverter.INSTANCE; + public PropertiesConverter propertiesConverter(Context context) { + return new MysqlPropertiesConverter(context); } @Override diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java index 2d895a4eaae..52e75857ec5 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java @@ -29,8 +29,8 @@ public String gravitinoCatalogProvider() { } @Override - public PropertiesConverter propertiesConverter() { - return PostgresqlPropertiesConverter.INSTANCE; + public PropertiesConverter propertiesConverter(Context context) { + return new PostgresqlPropertiesConverter(context); } @Override diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java index 0dc43ff0918..7eb1a1a03bf 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java @@ -19,13 +19,20 @@ package org.apache.gravitino.flink.connector.jdbc; +import java.util.HashMap; import java.util.Map; import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.factories.CatalogFactory; import org.apache.gravitino.flink.connector.PropertiesConverter; public abstract class JdbcPropertiesConverter implements PropertiesConverter { - protected JdbcPropertiesConverter() {} + private final CatalogFactory.Context context; + + protected JdbcPropertiesConverter(CatalogFactory.Context context) { + this.context = context; + } @Override public Map toGravitinoCatalogProperties(Configuration flinkConf) { @@ -45,5 +52,21 @@ public String transformPropertyToFlinkCatalog(String configKey) { return JdbcPropertiesConstants.gravitinoToFlinkMap.get(configKey); } + @Override + public Map toFlinkTableProperties( + Map gravitinoProperties, ObjectPath tablePath) { + Map catalogOptions = context.getOptions(); + Map tableOptions = new HashMap<>(); + tableOptions.put( + "url", + catalogOptions.get(JdbcPropertiesConstants.FLINK_JDBC_URL) + + "/" + + tablePath.getDatabaseName()); + tableOptions.put("table-name", tablePath.getObjectName()); + tableOptions.put("username", catalogOptions.get(JdbcPropertiesConstants.FLINK_JDBC_USER)); + tableOptions.put("password", catalogOptions.get(JdbcPropertiesConstants.FLINK_JDBC_PASSWORD)); + return tableOptions; + } + protected abstract String driverName(); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java index ddbe4d1f5cf..c51d8605e25 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java @@ -19,11 +19,13 @@ package org.apache.gravitino.flink.connector.jdbc; -public class MysqlPropertiesConverter extends JdbcPropertiesConverter { +import org.apache.flink.table.factories.CatalogFactory; - public static final MysqlPropertiesConverter INSTANCE = new MysqlPropertiesConverter(); +public class MysqlPropertiesConverter extends JdbcPropertiesConverter { - private MysqlPropertiesConverter() {} + protected MysqlPropertiesConverter(CatalogFactory.Context context) { + super(context); + } @Override public String driverName() { diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java index d46bedb008b..bcf9a45984b 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java @@ -19,11 +19,13 @@ package org.apache.gravitino.flink.connector.jdbc; -public class PostgresqlPropertiesConverter extends JdbcPropertiesConverter { +import org.apache.flink.table.factories.CatalogFactory; - public static final PostgresqlPropertiesConverter INSTANCE = new PostgresqlPropertiesConverter(); +public class PostgresqlPropertiesConverter extends JdbcPropertiesConverter { - private PostgresqlPropertiesConverter() {} + protected PostgresqlPropertiesConverter(CatalogFactory.Context context) { + super(context); + } @Override protected String driverName() { diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java index 8732ade23ed..d0b3c807c42 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java @@ -43,7 +43,7 @@ public Catalog createCatalog(Context context) { String defaultDatabase = helper.getOptions().get(GravitinoPaimonCatalogFactoryOptions.DEFAULT_DATABASE); return new GravitinoPaimonCatalog( - context, defaultDatabase, propertiesConverter(), partitionConverter()); + context, defaultDatabase, propertiesConverter(context), partitionConverter()); } @Override @@ -72,7 +72,7 @@ public org.apache.gravitino.Catalog.Type gravitinoCatalogType() { } @Override - public PropertiesConverter propertiesConverter() { + public PropertiesConverter propertiesConverter(Context context) { return PaimonPropertiesConverter.INSTANCE; } From 0e48086092063519cc44f74dd2128264760f0bc6 Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Wed, 12 Mar 2025 20:25:16 +0800 Subject: [PATCH 13/15] Optimize the logic of jdbcCatalog's getTable method. --- .../connector/catalog/BaseCatalogFactory.java | 3 +- .../hive/GravitinoHiveCatalogFactory.java | 5 +-- .../hive/HivePropertiesConverter.java | 4 ++- .../GravitinoIcebergCatalogFactory.java | 5 +-- .../jdbc/GravitinoJdbcCatalogFactory.java | 2 +- .../GravitinoMysqlJdbcCatalogFactory.java | 5 +-- .../GravitinoPostgresJdbcCatalogFactory.java | 5 +-- .../jdbc/JdbcPropertiesConverter.java | 12 ++++--- .../jdbc/MysqlPropertiesConverter.java | 6 ++-- .../jdbc/PostgresqlPropertiesConverter.java | 6 ++-- .../paimon/GravitinoPaimonCatalogFactory.java | 5 +-- .../jdbc/AbstractJdbcPropertiesConverter.java | 33 +++++++++++-------- .../jdbc/TestMysqlPropertiesConverter.java | 6 ++-- .../TestPostgresqlPropertiesConverter.java | 6 ++-- 14 files changed, 61 insertions(+), 42 deletions(-) diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalogFactory.java index 40b12429d79..db5150811b8 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalogFactory.java @@ -19,6 +19,7 @@ package org.apache.gravitino.flink.connector.catalog; +import java.util.Map; import org.apache.flink.table.factories.CatalogFactory; import org.apache.gravitino.Catalog; import org.apache.gravitino.flink.connector.PartitionConverter; @@ -45,7 +46,7 @@ public interface BaseCatalogFactory extends CatalogFactory { * * @return The requested property converter. */ - PropertiesConverter propertiesConverter(Context context); + PropertiesConverter propertiesConverter(Map catalogOptions); /** * Define partition converter. diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java index aabe875eb3b..3accb1789d2 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/GravitinoHiveCatalogFactory.java @@ -22,6 +22,7 @@ import static org.apache.gravitino.flink.connector.hive.GravitinoHiveCatalogFactoryOptions.IDENTIFIER; import com.google.common.collect.ImmutableSet; +import java.util.Map; import java.util.Set; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.table.catalog.Catalog; @@ -63,7 +64,7 @@ public Catalog createCatalog(Context context) { return new GravitinoHiveCatalog( context.getName(), helper.getOptions().get(HiveCatalogFactoryOptions.DEFAULT_DATABASE), - propertiesConverter(context), + propertiesConverter(context.getOptions()), partitionConverter(), hiveConf, helper.getOptions().get(HiveCatalogFactoryOptions.HIVE_VERSION)); @@ -113,7 +114,7 @@ public org.apache.gravitino.Catalog.Type gravitinoCatalogType() { * @return The requested property converter. */ @Override - public PropertiesConverter propertiesConverter(Context context) { + public PropertiesConverter propertiesConverter(Map catalogOptions) { return HivePropertiesConverter.INSTANCE; } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java index 20a3e8cf62d..70f89025420 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/hive/HivePropertiesConverter.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import java.util.stream.Collectors; +import org.apache.flink.table.catalog.ObjectPath; import org.apache.gravitino.catalog.hive.HiveConstants; import org.apache.gravitino.flink.connector.PropertiesConverter; import org.apache.hadoop.hive.conf.HiveConf; @@ -47,7 +48,8 @@ public String transformPropertyToFlinkCatalog(String configKey) { } @Override - public Map toFlinkTableProperties(Map gravitinoProperties) { + public Map toFlinkTableProperties( + Map gravitinoProperties, ObjectPath tablePath) { Map properties = gravitinoProperties.entrySet().stream() .collect( diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/GravitinoIcebergCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/GravitinoIcebergCatalogFactory.java index 150f22cd1d0..8f024e4c33e 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/GravitinoIcebergCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/iceberg/GravitinoIcebergCatalogFactory.java @@ -19,6 +19,7 @@ package org.apache.gravitino.flink.connector.iceberg; import java.util.Collections; +import java.util.Map; import java.util.Set; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.table.catalog.Catalog; @@ -38,7 +39,7 @@ public Catalog createCatalog(Context context) { return new GravitinoIcebergCatalog( context.getName(), helper.getOptions().get(GravitinoIcebergCatalogFactoryOptions.DEFAULT_DATABASE), - propertiesConverter(context), + propertiesConverter(context.getOptions()), partitionConverter(), context.getOptions()); } @@ -84,7 +85,7 @@ public org.apache.gravitino.Catalog.Type gravitinoCatalogType() { * @return */ @Override - public PropertiesConverter propertiesConverter(Context context) { + public PropertiesConverter propertiesConverter(Map catalogOptions) { return IcebergPropertiesConverter.INSTANCE; } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java index cd9d9743025..4239c900cf8 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalogFactory.java @@ -46,7 +46,7 @@ public org.apache.flink.table.catalog.Catalog createCatalog(Context context) { defaultDatabase, GravitinoJdbcCatalogFactoryOptions.DEFAULT_DATABASE.key() + " should not be null."); return new GravitinoJdbcCatalog( - context, defaultDatabase, propertiesConverter(), partitionConverter()); + context, defaultDatabase, propertiesConverter(context.getOptions()), partitionConverter()); } @Override diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java index ea8ce2ab3dd..dfaba03ade5 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoMysqlJdbcCatalogFactory.java @@ -19,6 +19,7 @@ package org.apache.gravitino.flink.connector.jdbc; +import java.util.Map; import org.apache.gravitino.flink.connector.PropertiesConverter; public class GravitinoMysqlJdbcCatalogFactory extends GravitinoJdbcCatalogFactory { @@ -29,8 +30,8 @@ public String gravitinoCatalogProvider() { } @Override - public PropertiesConverter propertiesConverter(Context context) { - return new MysqlPropertiesConverter(context); + public PropertiesConverter propertiesConverter(Map catalogOptions) { + return new MysqlPropertiesConverter(catalogOptions); } @Override diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java index 52e75857ec5..f85abba0d86 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoPostgresJdbcCatalogFactory.java @@ -19,6 +19,7 @@ package org.apache.gravitino.flink.connector.jdbc; +import java.util.Map; import org.apache.gravitino.flink.connector.PropertiesConverter; public class GravitinoPostgresJdbcCatalogFactory extends GravitinoJdbcCatalogFactory { @@ -29,8 +30,8 @@ public String gravitinoCatalogProvider() { } @Override - public PropertiesConverter propertiesConverter(Context context) { - return new PostgresqlPropertiesConverter(context); + public PropertiesConverter propertiesConverter(Map catalogOptions) { + return new PostgresqlPropertiesConverter(catalogOptions); } @Override diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java index 7eb1a1a03bf..060a84083f7 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/JdbcPropertiesConverter.java @@ -23,15 +23,14 @@ import java.util.Map; import org.apache.flink.configuration.Configuration; import org.apache.flink.table.catalog.ObjectPath; -import org.apache.flink.table.factories.CatalogFactory; import org.apache.gravitino.flink.connector.PropertiesConverter; public abstract class JdbcPropertiesConverter implements PropertiesConverter { - private final CatalogFactory.Context context; + private final Map catalogOptions; - protected JdbcPropertiesConverter(CatalogFactory.Context context) { - this.context = context; + protected JdbcPropertiesConverter(Map catalogOptions) { + this.catalogOptions = catalogOptions; } @Override @@ -55,7 +54,6 @@ public String transformPropertyToFlinkCatalog(String configKey) { @Override public Map toFlinkTableProperties( Map gravitinoProperties, ObjectPath tablePath) { - Map catalogOptions = context.getOptions(); Map tableOptions = new HashMap<>(); tableOptions.put( "url", @@ -69,4 +67,8 @@ public Map toFlinkTableProperties( } protected abstract String driverName(); + + public Map getCatalogOptions() { + return catalogOptions; + } } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java index c51d8605e25..dee5d98f63c 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java @@ -19,12 +19,12 @@ package org.apache.gravitino.flink.connector.jdbc; -import org.apache.flink.table.factories.CatalogFactory; +import java.util.Map; public class MysqlPropertiesConverter extends JdbcPropertiesConverter { - protected MysqlPropertiesConverter(CatalogFactory.Context context) { - super(context); + protected MysqlPropertiesConverter(Map catalogOptions) { + super(catalogOptions); } @Override diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java index bcf9a45984b..00d6d8d81da 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/PostgresqlPropertiesConverter.java @@ -19,12 +19,12 @@ package org.apache.gravitino.flink.connector.jdbc; -import org.apache.flink.table.factories.CatalogFactory; +import java.util.Map; public class PostgresqlPropertiesConverter extends JdbcPropertiesConverter { - protected PostgresqlPropertiesConverter(CatalogFactory.Context context) { - super(context); + protected PostgresqlPropertiesConverter(Map catalogOptions) { + super(catalogOptions); } @Override diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java index d0b3c807c42..659670312c1 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/paimon/GravitinoPaimonCatalogFactory.java @@ -20,6 +20,7 @@ package org.apache.gravitino.flink.connector.paimon; import java.util.Collections; +import java.util.Map; import java.util.Set; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.table.catalog.Catalog; @@ -43,7 +44,7 @@ public Catalog createCatalog(Context context) { String defaultDatabase = helper.getOptions().get(GravitinoPaimonCatalogFactoryOptions.DEFAULT_DATABASE); return new GravitinoPaimonCatalog( - context, defaultDatabase, propertiesConverter(context), partitionConverter()); + context, defaultDatabase, propertiesConverter(context.getOptions()), partitionConverter()); } @Override @@ -72,7 +73,7 @@ public org.apache.gravitino.Catalog.Type gravitinoCatalogType() { } @Override - public PropertiesConverter propertiesConverter(Context context) { + public PropertiesConverter propertiesConverter(Map catalogOptions) { return PaimonPropertiesConverter.INSTANCE; } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/AbstractJdbcPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/AbstractJdbcPropertiesConverter.java index 019ac592609..75f9db7ae46 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/AbstractJdbcPropertiesConverter.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/AbstractJdbcPropertiesConverter.java @@ -33,21 +33,24 @@ public abstract class AbstractJdbcPropertiesConverter { String url = "testUrl"; String defaultDatabase = "test"; - protected abstract JdbcPropertiesConverter getConverter(); + Map catalogProperties = + ImmutableMap.of( + JdbcPropertiesConstants.GRAVITINO_JDBC_USER, + username, + JdbcPropertiesConstants.GRAVITINO_JDBC_PASSWORD, + password, + JdbcPropertiesConstants.GRAVITINO_JDBC_URL, + url, + JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE, + defaultDatabase); + + protected abstract JdbcPropertiesConverter getConverter(Map catalogOptions); @Test public void testToPaimonFileSystemCatalog() { - Map catalogProperties = - ImmutableMap.of( - JdbcPropertiesConstants.GRAVITINO_JDBC_USER, - username, - JdbcPropertiesConstants.GRAVITINO_JDBC_PASSWORD, - password, - JdbcPropertiesConstants.GRAVITINO_JDBC_URL, - url, - JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE, - defaultDatabase); - Map properties = getConverter().toFlinkCatalogProperties(catalogProperties); + + Map properties = + getConverter(catalogProperties).toFlinkCatalogProperties(catalogProperties); Assertions.assertEquals(username, properties.get(JdbcPropertiesConstants.FLINK_JDBC_USER)); Assertions.assertEquals(password, properties.get(JdbcPropertiesConstants.FLINK_JDBC_PASSWORD)); Assertions.assertEquals(url, properties.get(JdbcPropertiesConstants.FLINK_JDBC_URL)); @@ -68,7 +71,8 @@ public void testToGravitinoCatalogProperties() { url, JdbcPropertiesConstants.FLINK_JDBC_DEFAULT_DATABASE, defaultDatabase)); - Map properties = getConverter().toGravitinoCatalogProperties(configuration); + Map properties = + getConverter(catalogProperties).toGravitinoCatalogProperties(configuration); Assertions.assertEquals(username, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_USER)); Assertions.assertEquals( @@ -77,6 +81,7 @@ public void testToGravitinoCatalogProperties() { Assertions.assertEquals( defaultDatabase, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE)); Assertions.assertEquals( - getConverter().driverName(), properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER)); + getConverter(catalogProperties).driverName(), + properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER)); } } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestMysqlPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestMysqlPropertiesConverter.java index b62c39d3360..33a4c171a15 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestMysqlPropertiesConverter.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestMysqlPropertiesConverter.java @@ -19,10 +19,12 @@ package org.apache.gravitino.flink.connector.jdbc; +import java.util.Map; + public class TestMysqlPropertiesConverter extends AbstractJdbcPropertiesConverter { @Override - protected JdbcPropertiesConverter getConverter() { - return MysqlPropertiesConverter.INSTANCE; + protected JdbcPropertiesConverter getConverter(Map catalogOptions) { + return new MysqlPropertiesConverter(catalogOptions); } } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestPostgresqlPropertiesConverter.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestPostgresqlPropertiesConverter.java index 206a09d1e8e..4c4a73cd5b1 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestPostgresqlPropertiesConverter.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/jdbc/TestPostgresqlPropertiesConverter.java @@ -19,10 +19,12 @@ package org.apache.gravitino.flink.connector.jdbc; +import java.util.Map; + public class TestPostgresqlPropertiesConverter extends AbstractJdbcPropertiesConverter { @Override - protected JdbcPropertiesConverter getConverter() { - return PostgresqlPropertiesConverter.INSTANCE; + protected JdbcPropertiesConverter getConverter(Map catalogOptions) { + return new PostgresqlPropertiesConverter(catalogOptions); } } From 16713c1a4079b867ef4659fab87d1f0ae5f96efc Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Wed, 12 Mar 2025 20:29:09 +0800 Subject: [PATCH 14/15] Optimize the logic of jdbcCatalog's getTable method. --- .../apache/gravitino/flink/connector/catalog/BaseCatalog.java | 1 - .../flink/connector/jdbc/MysqlPropertiesConverter.java | 2 ++ .../integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java index 6501f20b0b0..01c05d67fe6 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/catalog/BaseCatalog.java @@ -284,7 +284,6 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig String comment = table.getComment(); Map properties = propertiesConverter.toGravitinoTableProperties(table.getOptions()); - Transform[] partitions = partitionConverter.toGravitinoPartitions(((CatalogTable) table).getPartitionKeys()); diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java index dee5d98f63c..bc98045e3f4 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java @@ -23,6 +23,8 @@ public class MysqlPropertiesConverter extends JdbcPropertiesConverter { + + protected MysqlPropertiesConverter(Map catalogOptions) { super(catalogOptions); } diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java index ec3932e1385..d5d0dc59b6d 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java @@ -166,7 +166,7 @@ public void testCreateGravitinoJdbcCatalogUsingSQL() { mysqlDefaultDatabase, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE)); Assertions.assertEquals( - MysqlPropertiesConverter.INSTANCE.driverName(), + "com.mysql.jdbc.Driver", properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER)); } } From e4fad6c5e2b4fb710fb27dd6bf22e0b69d7fd324 Mon Sep 17 00:00:00 2001 From: yangyang zhong <35210666+hdygxsj@users.noreply.github.com> Date: Wed, 12 Mar 2025 20:50:27 +0800 Subject: [PATCH 15/15] fixci --- .../gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java | 3 --- .../flink/connector/jdbc/MysqlPropertiesConverter.java | 2 -- .../flink/connector/store/GravitinoCatalogStore.java | 5 +++-- .../integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java | 4 +--- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java index 53ba8d1c1b0..39e3c4235d0 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/GravitinoJdbcCatalog.java @@ -38,15 +38,12 @@ public class GravitinoJdbcCatalog extends BaseCatalog { private final JdbcCatalog jdbcCatalog; - private final CatalogFactory.Context context; - protected GravitinoJdbcCatalog( CatalogFactory.Context context, String defaultDatabase, PropertiesConverter propertiesConverter, PartitionConverter partitionConverter) { super(context.getName(), defaultDatabase, propertiesConverter, partitionConverter); - this.context = context; JdbcCatalogFactory jdbcCatalogFactory = new JdbcCatalogFactory(); this.jdbcCatalog = (JdbcCatalog) jdbcCatalogFactory.createCatalog(context); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java index bc98045e3f4..dee5d98f63c 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/jdbc/MysqlPropertiesConverter.java @@ -23,8 +23,6 @@ public class MysqlPropertiesConverter extends JdbcPropertiesConverter { - - protected MysqlPropertiesConverter(Map catalogOptions) { super(catalogOptions); } diff --git a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java index 4c29b7fde3b..986b38be4fd 100644 --- a/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java +++ b/flink-connector/flink/src/main/java/org/apache/gravitino/flink/connector/store/GravitinoCatalogStore.java @@ -57,7 +57,7 @@ public void storeCatalog(String catalogName, CatalogDescriptor descriptor) Map gravitino = configuration.toMap(); BaseCatalogFactory catalogFactory = getCatalogFactory(gravitino); Map gravitinoProperties = - catalogFactory.propertiesConverter().toGravitinoCatalogProperties(configuration); + catalogFactory.propertiesConverter(gravitino).toGravitinoCatalogProperties(configuration); gravitinoCatalogManager.createCatalog( catalogName, catalogFactory.gravitinoCatalogType(), @@ -80,7 +80,8 @@ public Optional getCatalog(String catalogName) throws Catalog try { Catalog catalog = gravitinoCatalogManager.getGravitinoCatalogInfo(catalogName); BaseCatalogFactory catalogFactory = getCatalogFactory(catalog.provider()); - PropertiesConverter propertiesConverter = catalogFactory.propertiesConverter(); + PropertiesConverter propertiesConverter = + catalogFactory.propertiesConverter(catalog.properties()); Map flinkCatalogProperties = propertiesConverter.toFlinkCatalogProperties(catalog.properties()); CatalogDescriptor descriptor = diff --git a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java index d5d0dc59b6d..d75d585b50d 100644 --- a/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java +++ b/flink-connector/flink/src/test/java/org/apache/gravitino/flink/connector/integration/test/jdbc/FlinkJdbcMysqlCatalogIT.java @@ -27,7 +27,6 @@ import org.apache.gravitino.Catalog; import org.apache.gravitino.flink.connector.integration.test.FlinkCommonIT; import org.apache.gravitino.flink.connector.jdbc.JdbcPropertiesConstants; -import org.apache.gravitino.flink.connector.jdbc.MysqlPropertiesConverter; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -166,7 +165,6 @@ public void testCreateGravitinoJdbcCatalogUsingSQL() { mysqlDefaultDatabase, properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DEFAULT_DATABASE)); Assertions.assertEquals( - "com.mysql.jdbc.Driver", - properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER)); + "com.mysql.jdbc.Driver", properties.get(JdbcPropertiesConstants.GRAVITINO_JDBC_DRIVER)); } }