From ed19b7a663f00a0af1fa9c934eab480cb8964930 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 28 Feb 2025 17:15:49 +0800 Subject: [PATCH 1/5] Optimize the entity parent id logic --- .../storage/relational/helper/CatalogIds.java | 37 ++++++++++++++++ .../storage/relational/helper/SchemaIds.java | 43 +++++++++++++++++++ .../relational/mapper/CatalogMetaMapper.java | 7 +++ .../mapper/CatalogMetaSQLProviderFactory.java | 5 +++ .../relational/mapper/SchemaMetaMapper.java | 9 ++++ .../mapper/SchemaMetaSQLProviderFactory.java | 9 ++++ .../base/CatalogMetaBaseSQLProvider.java | 9 ++++ .../base/SchemaMetaBaseSQLProvider.java | 17 ++++++++ .../service/CatalogMetaService.java | 7 +++ .../relational/service/CommonMetaService.java | 42 +++++++++++------- .../relational/service/SchemaMetaService.java | 10 +++++ 11 files changed, 179 insertions(+), 16 deletions(-) create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/helper/CatalogIds.java create mode 100644 core/src/main/java/org/apache/gravitino/storage/relational/helper/SchemaIds.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/helper/CatalogIds.java b/core/src/main/java/org/apache/gravitino/storage/relational/helper/CatalogIds.java new file mode 100644 index 00000000000..bd6654b61a8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/helper/CatalogIds.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.helper; + +public class CatalogIds { + private Long metalakeId; + private Long catalogId; + + public CatalogIds(Long metalakeId, Long catalogId) { + this.metalakeId = metalakeId; + this.catalogId = catalogId; + } + + public Long getMetalakeId() { + return metalakeId; + } + + public Long getCatalogId() { + return catalogId; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/helper/SchemaIds.java b/core/src/main/java/org/apache/gravitino/storage/relational/helper/SchemaIds.java new file mode 100644 index 00000000000..ff6f59b5cf3 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/storage/relational/helper/SchemaIds.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.helper; + +public class SchemaIds { + private Long metalakeId; + private Long catalogId; + private Long schemaId; + + public SchemaIds(Long metalakeId, Long catalogId, Long schemaId) { + this.metalakeId = metalakeId; + this.catalogId = catalogId; + this.schemaId = schemaId; + } + + public Long getMetalakeId() { + return metalakeId; + } + + public Long getCatalogId() { + return catalogId; + } + + public Long getSchemaId() { + return schemaId; + } +} diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java index 28423d75b5c..f74be4275ea 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaMapper.java @@ -20,6 +20,7 @@ package org.apache.gravitino.storage.relational.mapper; import java.util.List; +import org.apache.gravitino.storage.relational.helper.CatalogIds; import org.apache.gravitino.storage.relational.po.CatalogPO; import org.apache.ibatis.annotations.DeleteProvider; import org.apache.ibatis.annotations.InsertProvider; @@ -87,4 +88,10 @@ Integer updateCatalogMeta( method = "deleteCatalogMetasByLegacyTimeline") Integer deleteCatalogMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); + + @SelectProvider( + type = CatalogMetaSQLProviderFactory.class, + method = "selectCatalogIdByMetalakeNameAndCatalogName") + CatalogIds selectCatalogIdByMetalakeNameAndCatalogName( + @Param("metalakeName") String metalakeName, @Param("catalogName") String catalogName); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java index bfde8a034a4..e54a1481b1c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/CatalogMetaSQLProviderFactory.java @@ -71,6 +71,11 @@ public static String selectCatalogMetaByMetalakeIdAndName( return getProvider().selectCatalogMetaByMetalakeIdAndName(metalakeId, name); } + public static String selectCatalogIdByMetalakeNameAndCatalogName( + @Param("metalakeName") String metalakeName, @Param("catalogName") String catalogName) { + return getProvider().selectCatalogIdByMetalakeNameAndCatalogName(metalakeName, catalogName); + } + public static String selectCatalogMetaById(@Param("catalogId") Long catalogId) { return getProvider().selectCatalogMetaById(catalogId); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java index 49598ce727a..e1816a32779 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaMapper.java @@ -20,6 +20,7 @@ package org.apache.gravitino.storage.relational.mapper; import java.util.List; +import org.apache.gravitino.storage.relational.helper.SchemaIds; import org.apache.gravitino.storage.relational.po.SchemaPO; import org.apache.ibatis.annotations.DeleteProvider; import org.apache.ibatis.annotations.InsertProvider; @@ -91,4 +92,12 @@ Integer updateSchemaMeta( method = "deleteSchemaMetasByLegacyTimeline") Integer deleteSchemaMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit); + + @SelectProvider( + type = SchemaMetaSQLProviderFactory.class, + method = "selectSchemaIdByMetalakeNameAndCatalogNameAndSchemaName") + SchemaIds selectSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + @Param("metalakeName") String metalakeName, + @Param("catalogName") String catalogName, + @Param("schemaName") String schemaName); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java index 9f1669e476c..cbab45733cd 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SchemaMetaSQLProviderFactory.java @@ -103,4 +103,13 @@ public static String deleteSchemaMetasByLegacyTimeline( @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) { return getProvider().deleteSchemaMetasByLegacyTimeline(legacyTimeline, limit); } + + public static String selectSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + @Param("metalakeName") String metalakeName, + @Param("catalogName") String catalogName, + @Param("schemaName") String schemaName) { + return getProvider() + .selectSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + metalakeName, catalogName, schemaName); + } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/CatalogMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/CatalogMetaBaseSQLProvider.java index 3b2f603c4bd..6a62044cbf7 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/CatalogMetaBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/CatalogMetaBaseSQLProvider.java @@ -74,6 +74,15 @@ public String selectCatalogMetaByMetalakeIdAndName( + " WHERE metalake_id = #{metalakeId} AND catalog_name = #{catalogName} AND deleted_at = 0"; } + public String selectCatalogIdByMetalakeNameAndCatalogName( + @Param("metalakeName") String metalakeName, @Param("catalogName") String catalogName) { + return "SELECT me.metalake_id as metalakeId, ca.catalog_id as catalogId FROM " + + TABLE_NAME + + " ca INNER JOIN metalake_meta me ON ca.metalake_id = me.metalake_id" + + " WHERE me.metalake_name = #{metalakeName} AND ca.catalog_name = #{catalogName} " + + " AND ca.deleted_at = 0 AND me.deleted_at = 0"; + } + public String selectCatalogMetaById(@Param("catalogId") Long catalogId) { return "SELECT catalog_id as catalogId, catalog_name as catalogName," + " metalake_id as metalakeId, type, provider," diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/SchemaMetaBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/SchemaMetaBaseSQLProvider.java index 84ffcf84086..09d00c58618 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/SchemaMetaBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/SchemaMetaBaseSQLProvider.java @@ -190,4 +190,21 @@ public String deleteSchemaMetasByLegacyTimeline( + TABLE_NAME + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeline} LIMIT #{limit}"; } + + public String selectSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + @Param("metalakeName") String metalakeName, + @Param("catalogName") String catalogName, + @Param("schemaName") String schemaName) { + return "SELECT metalake_meta.metalake_id as metalakeId, catalog_meta.catalog_id as catalogId, " + + " schema_id as schemaId" + + " FROM metalake_meta" + + " JOIN catalog_meta ON metalake_meta.metalake_id = catalog_meta.metalake_id" + + " JOIN schema_meta ON catalog_meta.catalog_id = schema_meta.catalog_id" + + " WHERE metalake_name = #{metalakeName}" + + " AND catalog_name = #{catalogName}" + + " AND schema_name = #{schemaName}" + + " AND schema_meta.deleted_at = 0" + + " AND catalog_meta.deleted_at = 0" + + " AND metalake_meta.deleted_at = 0"; + } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java index 310b8cc08e9..71b700e1b54 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java @@ -33,6 +33,7 @@ import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.meta.SchemaEntity; +import org.apache.gravitino.storage.relational.helper.CatalogIds; import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetVersionMapper; @@ -80,6 +81,12 @@ public CatalogPO getCatalogPOByMetalakeIdAndName(Long metalakeId, String catalog return catalogPO; } + public CatalogIds getCatalogIdByMetalakeAndCatalogName(String metalakeName, String catalogName) { + return SessionUtils.getWithoutCommit( + CatalogMetaMapper.class, + mapper -> mapper.selectCatalogIdByMetalakeNameAndCatalogName(metalakeName, catalogName)); + } + // Catalog may be deleted, so the CatalogPO may be null. @Nullable public CatalogPO getCatalogPOById(Long catalogId) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java index bdab2ad9fe5..5334f47c1b6 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java @@ -21,6 +21,8 @@ import com.google.common.base.Preconditions; import org.apache.gravitino.Namespace; +import org.apache.gravitino.storage.relational.helper.CatalogIds; +import org.apache.gravitino.storage.relational.helper.SchemaIds; /** The service class for common metadata operations. */ public class CommonMetaService { @@ -36,22 +38,27 @@ public Long getParentEntityIdByNamespace(Namespace namespace) { Preconditions.checkArgument( !namespace.isEmpty() && namespace.levels().length <= 3, "Namespace should not be empty and length should be less than or equal to 3."); + Long parentEntityId = null; - if (namespace.levels().length >= 1) { + if (namespace.levels().length == 1) { parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByName(namespace.level(0)); } - if (namespace.levels().length >= 2) { + if (namespace.levels().length == 2) { parentEntityId = CatalogMetaService.getInstance() - .getCatalogIdByMetalakeIdAndName(parentEntityId, namespace.level(1)); + .getCatalogIdByMetalakeAndCatalogName(namespace.level(0), namespace.level(1)) + .getCatalogId(); } - if (namespace.levels().length >= 3) { + if (namespace.levels().length == 3) { parentEntityId = SchemaMetaService.getInstance() - .getSchemaIdByCatalogIdAndName(parentEntityId, namespace.level(2)); + .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + namespace.level(0), namespace.level(1), namespace.level(2)) + .getSchemaId(); } + Preconditions.checkState( parentEntityId != null && parentEntityId > 0, "Parent entity id should not be null and should be greater than 0."); @@ -63,21 +70,24 @@ public Long[] getParentEntityIdsByNamespace(Namespace namespace) { !namespace.isEmpty() && namespace.levels().length <= 3, "Namespace should not be empty and length should be less than or equal to 3."); Long[] parentEntityIds = new Long[namespace.levels().length]; - if (namespace.levels().length >= 1) { + + if (namespace.levels().length == 1) { parentEntityIds[0] = MetalakeMetaService.getInstance().getMetalakeIdByName(namespace.level(0)); - } - - if (namespace.levels().length >= 2) { - parentEntityIds[1] = + } else if (namespace.levels().length == 2) { + CatalogIds catalogIds = CatalogMetaService.getInstance() - .getCatalogIdByMetalakeIdAndName(parentEntityIds[0], namespace.level(1)); - } - - if (namespace.levels().length >= 3) { - parentEntityIds[2] = + .getCatalogIdByMetalakeAndCatalogName(namespace.level(0), namespace.level(1)); + parentEntityIds[0] = catalogIds.getMetalakeId(); + parentEntityIds[1] = catalogIds.getCatalogId(); + } else if (namespace.levels().length == 3) { + SchemaIds schemaIds = SchemaMetaService.getInstance() - .getSchemaIdByCatalogIdAndName(parentEntityIds[1], namespace.level(2)); + .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + namespace.level(0), namespace.level(1), namespace.level(2)); + parentEntityIds[0] = schemaIds.getMetalakeId(); + parentEntityIds[1] = schemaIds.getCatalogId(); + parentEntityIds[2] = schemaIds.getSchemaId(); } return parentEntityIds; diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index f300e70cae3..447f3405c65 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -34,6 +34,7 @@ import org.apache.gravitino.meta.ModelEntity; import org.apache.gravitino.meta.SchemaEntity; import org.apache.gravitino.meta.TableEntity; +import org.apache.gravitino.storage.relational.helper.SchemaIds; import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper; import org.apache.gravitino.storage.relational.mapper.FilesetVersionMapper; import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper; @@ -78,6 +79,15 @@ public SchemaPO getSchemaPOByCatalogIdAndName(Long catalogId, String schemaName) return schemaPO; } + public SchemaIds getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + String metalakeName, String catalogName, String schemaName) { + return SessionUtils.getWithoutCommit( + SchemaMetaMapper.class, + mapper -> + mapper.selectSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + metalakeName, catalogName, schemaName)); + } + // Schema may be deleted, so the SchemaPO may be null. public SchemaPO getSchemaPOById(Long schemaId) { return SessionUtils.getWithoutCommit( From 3aa3a074ed15a1d3fbc272085174491b623b4ba6 Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 28 Feb 2025 19:32:12 +0800 Subject: [PATCH 2/5] Fix --- .../relational/service/CommonMetaService.java | 69 +++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java index 5334f47c1b6..6fb25591af3 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java @@ -20,7 +20,9 @@ package org.apache.gravitino.storage.relational.service; import com.google.common.base.Preconditions; +import org.apache.gravitino.Entity; import org.apache.gravitino.Namespace; +import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.storage.relational.helper.CatalogIds; import org.apache.gravitino.storage.relational.helper.SchemaIds; @@ -42,26 +44,43 @@ public Long getParentEntityIdByNamespace(Namespace namespace) { Long parentEntityId = null; if (namespace.levels().length == 1) { parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByName(namespace.level(0)); + if (parentEntityId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.METALAKE.name().toLowerCase(), + namespace); + } } if (namespace.levels().length == 2) { - parentEntityId = + CatalogIds catalogIds = CatalogMetaService.getInstance() - .getCatalogIdByMetalakeAndCatalogName(namespace.level(0), namespace.level(1)) - .getCatalogId(); + .getCatalogIdByMetalakeAndCatalogName(namespace.level(0), namespace.level(1)); + parentEntityId = catalogIds == null ? null : catalogIds.getCatalogId(); + + if (parentEntityId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace); + } } if (namespace.levels().length == 3) { - parentEntityId = + SchemaIds schemaIds = SchemaMetaService.getInstance() .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( - namespace.level(0), namespace.level(1), namespace.level(2)) - .getSchemaId(); + namespace.level(0), namespace.level(1), namespace.level(2)); + parentEntityId = schemaIds == null ? null : schemaIds.getSchemaId(); + + if (parentEntityId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace); + } } - Preconditions.checkState( - parentEntityId != null && parentEntityId > 0, - "Parent entity id should not be null and should be greater than 0."); return parentEntityId; } @@ -74,20 +93,42 @@ public Long[] getParentEntityIdsByNamespace(Namespace namespace) { if (namespace.levels().length == 1) { parentEntityIds[0] = MetalakeMetaService.getInstance().getMetalakeIdByName(namespace.level(0)); + if (parentEntityIds[0] == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.METALAKE.name().toLowerCase(), + namespace); + } + } else if (namespace.levels().length == 2) { CatalogIds catalogIds = CatalogMetaService.getInstance() .getCatalogIdByMetalakeAndCatalogName(namespace.level(0), namespace.level(1)); - parentEntityIds[0] = catalogIds.getMetalakeId(); - parentEntityIds[1] = catalogIds.getCatalogId(); + parentEntityIds[0] = catalogIds == null ? null : catalogIds.getMetalakeId(); + parentEntityIds[1] = catalogIds == null ? null : catalogIds.getCatalogId(); + + if (parentEntityIds[1] == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace); + } + } else if (namespace.levels().length == 3) { SchemaIds schemaIds = SchemaMetaService.getInstance() .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( namespace.level(0), namespace.level(1), namespace.level(2)); - parentEntityIds[0] = schemaIds.getMetalakeId(); - parentEntityIds[1] = schemaIds.getCatalogId(); - parentEntityIds[2] = schemaIds.getSchemaId(); + parentEntityIds[0] = schemaIds == null ? null : schemaIds.getMetalakeId(); + parentEntityIds[1] = schemaIds == null ? null : schemaIds.getCatalogId(); + parentEntityIds[2] = schemaIds == null ? null : schemaIds.getSchemaId(); + + if (parentEntityIds[2] == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace); + } } return parentEntityIds; From 0293502bd617ef6de8d6e599f6757fc7e145fa23 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 3 Mar 2025 11:53:01 +0800 Subject: [PATCH 3/5] Fix test error. --- .../hadoop/TestHadoopCatalogOperations.java | 58 ++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 1a3e49b5499..0ef99203cd7 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -77,8 +77,11 @@ import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.storage.RandomIdGenerator; import org.apache.gravitino.storage.relational.RelationalEntityStore; +import org.apache.gravitino.storage.relational.helper.CatalogIds; +import org.apache.gravitino.storage.relational.helper.SchemaIds; import org.apache.gravitino.storage.relational.service.CatalogMetaService; import org.apache.gravitino.storage.relational.service.MetalakeMetaService; +import org.apache.gravitino.storage.relational.service.SchemaMetaService; import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -204,10 +207,58 @@ public static void setUp() { .when(spyCatalogMetaService) .getCatalogIdByMetalakeIdAndName(Mockito.anyLong(), Mockito.anyString()); + SchemaMetaService serviceMetaService = SchemaMetaService.getInstance(); + SchemaMetaService spySchemaMetaService = Mockito.spy(serviceMetaService); + + doReturn(new CatalogIds(1L, 1L)) + .when(spyCatalogMetaService) + .getCatalogIdByMetalakeAndCatalogName(Mockito.anyString(), Mockito.anyString()); + + doReturn(new SchemaIds(1L, 1L, 1L)) + .when(spySchemaMetaService) + .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + Mockito.anyString(), Mockito.anyString(), Mockito.eq("schema11")); + + for (int i = 10; i < 30; i++) { + doReturn(new SchemaIds(1L, 1L, (long) i)) + .when(spySchemaMetaService) + .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + Mockito.anyString(), Mockito.anyString(), Mockito.eq("schema" + i)); + } + + Stream argumentsStream = testRenameArguments(); + argumentsStream.forEach( + arguments -> { + String oldName = (String) arguments.get()[0]; + String newName = (String) arguments.get()[1]; + long schemaId = idGenerator.nextId(); + doReturn(new SchemaIds(1L, 1L, schemaId)) + .when(spySchemaMetaService) + .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + Mockito.anyString(), Mockito.anyString(), Mockito.eq("s24_" + oldName)); + doReturn(new SchemaIds(1L, 1L, schemaId)) + .when(spySchemaMetaService) + .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + Mockito.anyString(), Mockito.anyString(), Mockito.eq("s24_" + newName)); + }); + + locationArguments() + .forEach( + arguments -> { + String name = (String) arguments.get()[0]; + long schemaId = idGenerator.nextId(); + doReturn(new SchemaIds(1L, 1L, schemaId)) + .when(spySchemaMetaService) + .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + Mockito.anyString(), Mockito.anyString(), Mockito.eq("s1_" + name)); + }); + MockedStatic metalakeMetaServiceMockedStatic = Mockito.mockStatic(MetalakeMetaService.class); MockedStatic catalogMetaServiceMockedStatic = Mockito.mockStatic(CatalogMetaService.class); + MockedStatic schemaMetaServiceMockedStatic = + Mockito.mockStatic(SchemaMetaService.class); metalakeMetaServiceMockedStatic .when(MetalakeMetaService::getInstance) @@ -215,6 +266,9 @@ public static void setUp() { catalogMetaServiceMockedStatic .when(CatalogMetaService::getInstance) .thenReturn(spyCatalogMetaService); + schemaMetaServiceMockedStatic + .when(SchemaMetaService::getInstance) + .thenReturn(spySchemaMetaService); } @AfterAll @@ -856,8 +910,8 @@ void testTrailSlash() throws IOException { @Test public void testGetFileLocation() throws IOException { - String schemaName = "schema1024"; - String comment = "comment1024"; + String schemaName = "schema29"; + String comment = "schema29"; String schemaPath = TEST_ROOT_PATH + "/" + schemaName; createSchema(schemaName, comment, null, schemaPath); From 3bcfcf53c552ff21e4e1f4ce73ac38d6a5818607 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 3 Mar 2025 12:54:31 +0800 Subject: [PATCH 4/5] Fix test error. --- .../gravitino/catalog/kafka/TestKafkaCatalogOperations.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java index c12c191482e..0a24fa2104f 100644 --- a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java @@ -67,6 +67,7 @@ import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.storage.RandomIdGenerator; +import org.apache.gravitino.storage.relational.helper.CatalogIds; import org.apache.gravitino.storage.relational.service.CatalogMetaService; import org.apache.gravitino.storage.relational.service.MetalakeMetaService; import org.apache.kafka.common.config.TopicConfig; @@ -156,6 +157,9 @@ public static void setUp() { doReturn(1L) .when(spyCatalogMetaService) .getCatalogIdByMetalakeIdAndName(Mockito.anyLong(), Mockito.anyString()); + doReturn(new CatalogIds(1L, 1L)) + .when(spyCatalogMetaService) + .getCatalogIdByMetalakeAndCatalogName(Mockito.anyString(), Mockito.anyString()); MockedStatic metalakeMetaServiceMockedStatic = Mockito.mockStatic(MetalakeMetaService.class); From 28a038af541c0b2f3f55d258bf47f71a27e479fd Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 4 Mar 2025 14:58:41 +0800 Subject: [PATCH 5/5] fix comments --- .../relational/service/CommonMetaService.java | 174 ++++++++++-------- 1 file changed, 93 insertions(+), 81 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java index 6fb25591af3..aa18d4bc568 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/CommonMetaService.java @@ -41,47 +41,51 @@ public Long getParentEntityIdByNamespace(Namespace namespace) { !namespace.isEmpty() && namespace.levels().length <= 3, "Namespace should not be empty and length should be less than or equal to 3."); - Long parentEntityId = null; - if (namespace.levels().length == 1) { - parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByName(namespace.level(0)); - if (parentEntityId == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.METALAKE.name().toLowerCase(), - namespace); - } + int length = namespace.levels().length; + Long parentEntityId; + switch (length) { + case 1: + // Parent is a metalake + parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByName(namespace.level(0)); + if (parentEntityId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.METALAKE.name().toLowerCase(), + namespace); + } + + return parentEntityId; + case 2: + // Parent is a catalog + CatalogIds catalogIds = + CatalogMetaService.getInstance() + .getCatalogIdByMetalakeAndCatalogName(namespace.level(0), namespace.level(1)); + parentEntityId = catalogIds == null ? null : catalogIds.getCatalogId(); + if (parentEntityId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace); + } + + return parentEntityId; + case 3: + // Parent is a schema + SchemaIds schemaIds = + SchemaMetaService.getInstance() + .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + namespace.level(0), namespace.level(1), namespace.level(2)); + parentEntityId = schemaIds == null ? null : schemaIds.getSchemaId(); + if (parentEntityId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace); + } + return parentEntityId; + default: + throw new IllegalArgumentException("Namespace length should be less than or equal to 3."); } - - if (namespace.levels().length == 2) { - CatalogIds catalogIds = - CatalogMetaService.getInstance() - .getCatalogIdByMetalakeAndCatalogName(namespace.level(0), namespace.level(1)); - parentEntityId = catalogIds == null ? null : catalogIds.getCatalogId(); - - if (parentEntityId == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.CATALOG.name().toLowerCase(), - namespace); - } - } - - if (namespace.levels().length == 3) { - SchemaIds schemaIds = - SchemaMetaService.getInstance() - .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( - namespace.level(0), namespace.level(1), namespace.level(2)); - parentEntityId = schemaIds == null ? null : schemaIds.getSchemaId(); - - if (parentEntityId == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.SCHEMA.name().toLowerCase(), - namespace); - } - } - - return parentEntityId; } public Long[] getParentEntityIdsByNamespace(Namespace namespace) { @@ -90,47 +94,55 @@ public Long[] getParentEntityIdsByNamespace(Namespace namespace) { "Namespace should not be empty and length should be less than or equal to 3."); Long[] parentEntityIds = new Long[namespace.levels().length]; - if (namespace.levels().length == 1) { - parentEntityIds[0] = - MetalakeMetaService.getInstance().getMetalakeIdByName(namespace.level(0)); - if (parentEntityIds[0] == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.METALAKE.name().toLowerCase(), - namespace); - } - - } else if (namespace.levels().length == 2) { - CatalogIds catalogIds = - CatalogMetaService.getInstance() - .getCatalogIdByMetalakeAndCatalogName(namespace.level(0), namespace.level(1)); - parentEntityIds[0] = catalogIds == null ? null : catalogIds.getMetalakeId(); - parentEntityIds[1] = catalogIds == null ? null : catalogIds.getCatalogId(); - - if (parentEntityIds[1] == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.CATALOG.name().toLowerCase(), - namespace); - } - - } else if (namespace.levels().length == 3) { - SchemaIds schemaIds = - SchemaMetaService.getInstance() - .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( - namespace.level(0), namespace.level(1), namespace.level(2)); - parentEntityIds[0] = schemaIds == null ? null : schemaIds.getMetalakeId(); - parentEntityIds[1] = schemaIds == null ? null : schemaIds.getCatalogId(); - parentEntityIds[2] = schemaIds == null ? null : schemaIds.getSchemaId(); - - if (parentEntityIds[2] == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.SCHEMA.name().toLowerCase(), - namespace); - } + int length = namespace.levels().length; + switch (length) { + case 1: + // Parent is a metalake + parentEntityIds[0] = + MetalakeMetaService.getInstance().getMetalakeIdByName(namespace.level(0)); + if (parentEntityIds[0] == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.METALAKE.name().toLowerCase(), + namespace); + } + + return parentEntityIds; + case 2: + // Parent is a catalog + CatalogIds catalogIds = + CatalogMetaService.getInstance() + .getCatalogIdByMetalakeAndCatalogName(namespace.level(0), namespace.level(1)); + parentEntityIds[0] = catalogIds == null ? null : catalogIds.getMetalakeId(); + parentEntityIds[1] = catalogIds == null ? null : catalogIds.getCatalogId(); + + if (parentEntityIds[1] == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.CATALOG.name().toLowerCase(), + namespace); + } + return parentEntityIds; + case 3: + // Parent is a schema + SchemaIds schemaIds = + SchemaMetaService.getInstance() + .getSchemaIdByMetalakeNameAndCatalogNameAndSchemaName( + namespace.level(0), namespace.level(1), namespace.level(2)); + parentEntityIds[0] = schemaIds == null ? null : schemaIds.getMetalakeId(); + parentEntityIds[1] = schemaIds == null ? null : schemaIds.getCatalogId(); + parentEntityIds[2] = schemaIds == null ? null : schemaIds.getSchemaId(); + + if (parentEntityIds[2] == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.SCHEMA.name().toLowerCase(), + namespace); + } + + return parentEntityIds; + default: + throw new IllegalArgumentException("Namespace length should be less than or equal to 3."); } - - return parentEntityIds; } }