From 2be36050f89006cc7e49764f10da9932087aebd4 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 27 Jun 2024 19:12:47 +0800 Subject: [PATCH 01/14] Add a cache service to cache the mapping of name identifer to id --- .../storage/relational/JDBCBackend.java | 67 ++++++++++------ .../service/CatalogMetaService.java | 25 ++++-- .../relational/service/CommonMetaService.java | 32 ++++---- .../service/FilesetMetaService.java | 47 ++++++----- .../relational/service/GroupMetaService.java | 16 +++- .../service/IdNameMappingService.java | 79 +++++++++++++++++++ .../service/MetalakeMetaService.java | 46 ++++++++--- .../relational/service/RoleMetaService.java | 21 ++++- .../relational/service/SchemaMetaService.java | 47 +++++++---- .../relational/service/TableMetaService.java | 52 +++++++----- .../relational/service/TopicMetaService.java | 49 +++++++----- .../relational/service/UserMetaService.java | 18 ++++- .../relational/utils/MetadataObjectUtils.java | 32 ++++---- 13 files changed, 370 insertions(+), 161 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/storage/relational/service/IdNameMappingService.java diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java index b36f7e71545..cfe578f4b99 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java @@ -31,6 +31,7 @@ import com.datastrato.gravitino.storage.relational.service.CatalogMetaService; import com.datastrato.gravitino.storage.relational.service.FilesetMetaService; import com.datastrato.gravitino.storage.relational.service.GroupMetaService; +import com.datastrato.gravitino.storage.relational.service.IdNameMappingService; import com.datastrato.gravitino.storage.relational.service.MetalakeMetaService; import com.datastrato.gravitino.storage.relational.service.RoleMetaService; import com.datastrato.gravitino.storage.relational.service.SchemaMetaService; @@ -124,12 +125,18 @@ public void insert(E e, boolean overwritten) throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for insert operation", e.getClass()); } + + IdNameMappingService.getInstance().put(e.nameIdentifier(), e.id()); } @Override public E update( NameIdentifier ident, Entity.EntityType entityType, Function updater) throws IOException, NoSuchEntityException, AlreadyExistsException { + // Remove all the children entities in the cache as we can't guarantee the children entities + // are still valid after the parent entity is updated. + IdNameMappingService.getInstance().invalidateWithPrefix(ident); + switch (entityType) { case METALAKE: return (E) MetalakeMetaService.getInstance().updateMetalake(ident, updater); @@ -183,28 +190,42 @@ public E get( @Override public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { - switch (entityType) { - case METALAKE: - return MetalakeMetaService.getInstance().deleteMetalake(ident, cascade); - case CATALOG: - return CatalogMetaService.getInstance().deleteCatalog(ident, cascade); - case SCHEMA: - return SchemaMetaService.getInstance().deleteSchema(ident, cascade); - case TABLE: - return TableMetaService.getInstance().deleteTable(ident); - case FILESET: - return FilesetMetaService.getInstance().deleteFileset(ident); - case TOPIC: - return TopicMetaService.getInstance().deleteTopic(ident); - case USER: - return UserMetaService.getInstance().deleteUser(ident); - case GROUP: - return GroupMetaService.getInstance().deleteGroup(ident); - case ROLE: - return RoleMetaService.getInstance().deleteRole(ident); - default: - throw new UnsupportedEntityTypeException( - "Unsupported entity type: %s for delete operation", entityType); + // Invalidate the cache first + IdNameMappingService.getInstance().invalidate(ident); + if (cascade) { + // Remove all the children entities in the cache; + IdNameMappingService.getInstance().invalidateWithPrefix(ident); + } + + try { + switch (entityType) { + case METALAKE: + return MetalakeMetaService.getInstance().deleteMetalake(ident, cascade); + case CATALOG: + return CatalogMetaService.getInstance().deleteCatalog(ident, cascade); + case SCHEMA: + return SchemaMetaService.getInstance().deleteSchema(ident, cascade); + case TABLE: + return TableMetaService.getInstance().deleteTable(ident); + case FILESET: + return FilesetMetaService.getInstance().deleteFileset(ident); + case TOPIC: + return TopicMetaService.getInstance().deleteTopic(ident); + case USER: + return UserMetaService.getInstance().deleteUser(ident); + case GROUP: + return GroupMetaService.getInstance().deleteGroup(ident); + case ROLE: + return RoleMetaService.getInstance().deleteRole(ident); + default: + throw new UnsupportedEntityTypeException( + "Unsupported entity type: %s for delete operation", entityType); + } + } finally { + // Remove the entity from the cache again because we may add the cache during the deletion + // process + IdNameMappingService.getInstance().invalidate(ident); + IdNameMappingService.getInstance().invalidateWithPrefix(ident); } } @@ -291,6 +312,8 @@ public int deleteOldVersionData(Entity.EntityType entityType, long versionRetent public void close() throws IOException { SqlSessionFactoryHelper.getInstance().close(); + IdNameMappingService.getInstance().close(); + if (jdbcDatabase != null) { jdbcDatabase.close(); } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java index 94fee06c222..75d9d8606c6 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CatalogMetaService.java @@ -68,7 +68,7 @@ public CatalogPO getCatalogPOById(Long catalogId) { return catalogPO; } - public Long getCatalogIdByMetalakeIdAndName(Long metalakeId, String catalogName) { + private Long getCatalogIdByMetalakeIdAndName(Long metalakeId, String catalogName) { Long catalogId = SessionUtils.getWithoutCommit( CatalogMetaMapper.class, @@ -83,6 +83,20 @@ public Long getCatalogIdByMetalakeIdAndName(Long metalakeId, String catalogName) return catalogId; } + public Long getCatalogIdByNameIdentifier(NameIdentifier identifier) { + NameIdentifierUtil.checkCatalog(identifier); + + return IdNameMappingService.getInstance() + .get( + identifier, + ident -> { + String catalogName = ident.name(); + Long metalakeId = + CommonMetaService.getInstance().getParentEntityIdByNamespace(ident.namespace()); + return getCatalogIdByMetalakeIdAndName(metalakeId, catalogName); + }); + } + public CatalogEntity getCatalogByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkCatalog(identifier); String catalogName = identifier.name(); @@ -176,10 +190,7 @@ public boolean deleteCatalog(NameIdentifier identifier, boolean cascade) { NameIdentifierUtil.checkCatalog(identifier); String catalogName = identifier.name(); - Long metalakeId = - CommonMetaService.getInstance().getParentEntityIdByNamespace(identifier.namespace()); - - Long catalogId = getCatalogIdByMetalakeIdAndName(metalakeId, catalogName); + Long catalogId = getCatalogIdByNameIdentifier(identifier); if (cascade) { SessionUtils.doMultipleWithCommit( @@ -226,8 +237,6 @@ public boolean deleteCatalog(NameIdentifier identifier, boolean cascade) { public int deleteCatalogMetasByLegacyTimeline(Long legacyTimeline, int limit) { return SessionUtils.doWithCommitAndFetchResult( CatalogMetaMapper.class, - mapper -> { - return mapper.deleteCatalogMetasByLegacyTimeline(legacyTimeline, limit); - }); + mapper -> mapper.deleteCatalogMetasByLegacyTimeline(legacyTimeline, limit)); } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CommonMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CommonMetaService.java index c270dda7c6f..90457da30fc 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CommonMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CommonMetaService.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.storage.relational.service; +import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; import com.google.common.base.Preconditions; @@ -22,24 +23,23 @@ public Long getParentEntityIdByNamespace(Namespace namespace) { Preconditions.checkArgument( !namespace.isEmpty() && namespace.levels().length <= 3, "Namespace should not be empty and length should be less than or equal to 3."); + + String[] level = namespace.levels(); + NameIdentifier ident = NameIdentifier.of(level); Long parentEntityId = null; - for (int level = 0; level < namespace.levels().length; level++) { - String name = namespace.level(level); - switch (level) { - case 0: - parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByName(name); - continue; - case 1: - parentEntityId = - CatalogMetaService.getInstance() - .getCatalogIdByMetalakeIdAndName(parentEntityId, name); - continue; - case 2: - parentEntityId = - SchemaMetaService.getInstance().getSchemaIdByCatalogIdAndName(parentEntityId, name); - break; - } + + switch (level.length) { + case 1: + parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByNameIdentifier(ident); + break; + case 2: + parentEntityId = CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(ident); + break; + case 3: + parentEntityId = SchemaMetaService.getInstance().getSchemaIdByNameIdentifier(ident); + break; } + Preconditions.checkState( parentEntityId != null && parentEntityId > 0, "Parent entity id should not be null and should be greater than 0."); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java index 0bc14605fd8..bb907a93bbc 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/FilesetMetaService.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Objects; import java.util.function.Function; +import org.apache.commons.lang3.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,7 +58,7 @@ public FilesetPO getFilesetPOBySchemaIdAndName(Long schemaId, String filesetName return filesetPO; } - // Filset may be deleted, so the FilesetPO may be null. + // Fileset may be deleted, so the FilesetPO may be null. public FilesetPO getFilesetPOById(Long filesetId) { FilesetPO filesetPO = SessionUtils.getWithoutCommit( @@ -65,7 +66,20 @@ public FilesetPO getFilesetPOById(Long filesetId) { return filesetPO; } - public Long getFilesetIdBySchemaIdAndName(Long schemaId, String filesetName) { + public Long getFilesetIdByNameIdentifier(NameIdentifier identifier) { + NameIdentifierUtil.checkFileset(identifier); + + return IdNameMappingService.getInstance() + .get( + identifier, + ident -> { + Long schemaId = + CommonMetaService.getInstance().getParentEntityIdByNamespace(ident.namespace()); + return getFilesetIdBySchemaIdAndName(schemaId, ident.name()); + }); + } + + private Long getFilesetIdBySchemaIdAndName(Long schemaId, String filesetName) { Long filesetId = SessionUtils.getWithoutCommit( FilesetMetaMapper.class, @@ -207,12 +221,7 @@ public FilesetEntity updateFileset( public boolean deleteFileset(NameIdentifier identifier) { NameIdentifierUtil.checkFileset(identifier); - String filesetName = identifier.name(); - - Long schemaId = - CommonMetaService.getInstance().getParentEntityIdByNamespace(identifier.namespace()); - - Long filesetId = getFilesetIdBySchemaIdAndName(schemaId, filesetName); + Long filesetId = getFilesetIdByNameIdentifier(identifier); // We should delete meta and version info SessionUtils.doMultipleWithCommit( @@ -278,24 +287,24 @@ public int deleteFilesetVersionsByRetentionCount(Long versionRetentionCount, int private void fillFilesetPOBuilderParentEntityId(FilesetPO.Builder builder, Namespace namespace) { NamespaceUtil.checkFileset(namespace); - Long parentEntityId = null; + Long entityId; + for (int level = 0; level < namespace.levels().length; level++) { - String name = namespace.level(level); + String[] levels = ArrayUtils.subarray(namespace.levels(), 0, level + 1); + NameIdentifier nameIdentifier = NameIdentifier.of(levels); switch (level) { case 0: - parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByName(name); - builder.withMetalakeId(parentEntityId); + entityId = + MetalakeMetaService.getInstance().getMetalakeIdByNameIdentifier(nameIdentifier); + builder.withMetalakeId(entityId); continue; case 1: - parentEntityId = - CatalogMetaService.getInstance() - .getCatalogIdByMetalakeIdAndName(parentEntityId, name); - builder.withCatalogId(parentEntityId); + entityId = CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(nameIdentifier); + builder.withCatalogId(entityId); continue; case 2: - parentEntityId = - SchemaMetaService.getInstance().getSchemaIdByCatalogIdAndName(parentEntityId, name); - builder.withSchemaId(parentEntityId); + entityId = SchemaMetaService.getInstance().getSchemaIdByNameIdentifier(nameIdentifier); + builder.withSchemaId(entityId); break; } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java index 36a9dd99fa3..3b11b32140b 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java @@ -67,6 +67,18 @@ private Long getGroupIdByMetalakeIdAndName(Long metalakeId, String groupName) { return groupId; } + private Long getGroupIdByNameIdentifier(NameIdentifier identifier) { + return IdNameMappingService.getInstance() + .get( + identifier, + ident -> { + Long metalakeId = + MetalakeMetaService.getInstance() + .getMetalakeIdByName(identifier.namespace().level(0)); + return getGroupIdByMetalakeIdAndName(metalakeId, identifier.name()); + }); + } + public GroupEntity getGroupByIdentifier(NameIdentifier identifier) { AuthorizationUtils.checkGroup(identifier); @@ -124,9 +136,7 @@ public void insertGroup(GroupEntity groupEntity, boolean overwritten) { public boolean deleteGroup(NameIdentifier identifier) { AuthorizationUtils.checkGroup(identifier); - Long metalakeId = - MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); - Long groupId = getGroupIdByMetalakeIdAndName(metalakeId, identifier.name()); + Long groupId = getGroupIdByNameIdentifier(identifier); SessionUtils.doMultipleWithCommit( () -> diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/IdNameMappingService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/IdNameMappingService.java new file mode 100644 index 00000000000..3cf2f1c8f2e --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/IdNameMappingService.java @@ -0,0 +1,79 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.storage.relational.service; + +import com.datastrato.gravitino.NameIdentifier; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Scheduler; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +public class IdNameMappingService implements Closeable { + + private static volatile IdNameMappingService instance; + + private Cache ident2IdCache; + + private IdNameMappingService() { + this.ident2IdCache = + Caffeine.newBuilder() + .expireAfterAccess(24 * 3600 * 1000 /* 1 day */, TimeUnit.MILLISECONDS) + .maximumSize(1000000) + .initialCapacity(1000) + .scheduler( + Scheduler.forScheduledExecutorService( + new ScheduledThreadPoolExecutor( + 1, + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("ident-to-id-cleaner-%d") + .build()))) + .build(); + } + + public static IdNameMappingService getInstance() { + if (instance == null) { + synchronized (IdNameMappingService.class) { + if (instance == null) { + instance = new IdNameMappingService(); + } + } + } + + return instance; + } + + public void put(NameIdentifier key, Long value) { + ident2IdCache.put(key, value); + } + + public Long get(NameIdentifier key, Function mappingFunction) { + return ident2IdCache.get(key, mappingFunction); + } + + public void invalidate(NameIdentifier key) { + ident2IdCache.invalidate(key); + } + + public void invalidateWithPrefix(NameIdentifier nameIdentifier) { + ident2IdCache.asMap().keySet().stream() + .filter(k -> k.toString().startsWith(nameIdentifier.toString())) + .forEach(ident2IdCache::invalidate); + } + + @Override + public void close() throws IOException { + if (ident2IdCache != null) { + ident2IdCache.invalidateAll(); + ident2IdCache.cleanUp(); + } + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java index 90537ccc7ff..b0a0ea55f11 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/MetalakeMetaService.java @@ -56,17 +56,43 @@ public List listMetalakes() { return POConverters.fromMetalakePOs(metalakePOS); } + public Long getMetalakeIdByNameIdentifier(NameIdentifier nameIdentifier) { + NameIdentifierUtil.checkMetalake(nameIdentifier); + return IdNameMappingService.getInstance() + .get( + nameIdentifier, + ident -> { + Long metalakeId = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, + mapper -> mapper.selectMetalakeIdMetaByName(nameIdentifier.name())); + if (metalakeId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.METALAKE.name().toLowerCase(), + nameIdentifier.toString()); + } + return metalakeId; + }); + } + public Long getMetalakeIdByName(String metalakeName) { - Long metalakeId = - SessionUtils.getWithoutCommit( - MetalakeMetaMapper.class, mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); - if (metalakeId == null) { - throw new NoSuchEntityException( - NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, - Entity.EntityType.METALAKE.name().toLowerCase(), - metalakeName); - } - return metalakeId; + return IdNameMappingService.getInstance() + .get( + NameIdentifierUtil.ofMetalake(metalakeName), + ident -> { + Long metalakeId = + SessionUtils.getWithoutCommit( + MetalakeMetaMapper.class, + mapper -> mapper.selectMetalakeIdMetaByName(metalakeName)); + if (metalakeId == null) { + throw new NoSuchEntityException( + NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, + Entity.EntityType.METALAKE.name().toLowerCase(), + metalakeName); + } + return metalakeId; + }); } public BaseMetalake getMetalakeByIdentifier(NameIdentifier ident) { diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java index 238ba26c2ca..0504f1b2713 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java @@ -69,6 +69,19 @@ public Long getRoleIdByMetalakeIdAndName(Long metalakeId, String roleName) { return roleId; } + public Long getRoleIdByNameIdentifier(NameIdentifier identifier) { + AuthorizationUtils.checkRole(identifier); + + return IdNameMappingService.getInstance() + .get( + identifier, + ident -> { + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(ident.namespace().level(0)); + return getRoleIdByMetalakeIdAndName(metalakeId, ident.name()); + }); + } + public List listRolesByUserId(Long userId) { return SessionUtils.getWithoutCommit( RoleMetaMapper.class, mapper -> mapper.listRolesByUserId(userId)); @@ -83,6 +96,7 @@ public void insertRole(RoleEntity roleEntity, boolean overwritten) { try { AuthorizationUtils.checkRole(roleEntity.nameIdentifier()); + String metalakeName = roleEntity.namespace().level(0); Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(roleEntity.namespace().level(0)); RolePO.Builder builder = RolePO.builder().withMetalakeId(metalakeId); @@ -93,7 +107,8 @@ public void insertRole(RoleEntity roleEntity, boolean overwritten) { POConverters.initializeSecurablePOBuilderWithVersion( roleEntity.id(), object, getEntityType(object)); objectBuilder.withEntityId( - MetadataObjectUtils.getMetadataObjectId(metalakeId, object.fullName(), object.type())); + MetadataObjectUtils.getMetadataObjectId( + metalakeName, object.fullName(), object.type())); securableObjectPOs.add(objectBuilder.build()); } @@ -159,9 +174,7 @@ public RoleEntity getRoleByIdentifier(NameIdentifier identifier) { public boolean deleteRole(NameIdentifier identifier) { AuthorizationUtils.checkRole(identifier); - Long metalakeId = - MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); - Long roleId = getRoleIdByMetalakeIdAndName(metalakeId, identifier.name()); + Long roleId = getRoleIdByNameIdentifier(identifier); SessionUtils.doMultipleWithCommit( () -> diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java index b75941b7acf..eea5d2021e1 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/SchemaMetaService.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Objects; import java.util.function.Function; +import org.apache.commons.lang3.ArrayUtils; /** The service class for schema metadata. It provides the basic database operations for schema. */ public class SchemaMetaService { @@ -61,7 +62,7 @@ public SchemaPO getSchemaPOById(Long schemaId) { SchemaMetaMapper.class, mapper -> mapper.selectSchemaMetaById(schemaId)); } - public Long getSchemaIdByCatalogIdAndName(Long catalogId, String schemaName) { + private Long getSchemaIdByCatalogIdAndName(Long catalogId, String schemaName) { Long schemaId = SessionUtils.getWithoutCommit( SchemaMetaMapper.class, @@ -76,6 +77,21 @@ public Long getSchemaIdByCatalogIdAndName(Long catalogId, String schemaName) { return schemaId; } + public Long getSchemaIdByNameIdentifier(NameIdentifier identifier) { + return IdNameMappingService.getInstance() + .get( + identifier, + ident -> { + NameIdentifierUtil.checkSchema(ident); + String schemaName = ident.name(); + + Long catalogId = + CommonMetaService.getInstance().getParentEntityIdByNamespace(ident.namespace()); + + return getSchemaIdByCatalogIdAndName(catalogId, schemaName); + }); + } + public SchemaEntity getSchemaByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkSchema(identifier); String schemaName = identifier.name(); @@ -166,9 +182,9 @@ public boolean deleteSchema(NameIdentifier identifier, boolean cascade) { NameIdentifierUtil.checkSchema(identifier); String schemaName = identifier.name(); - Long catalogId = - CommonMetaService.getInstance().getParentEntityIdByNamespace(identifier.namespace()); - Long schemaId = getSchemaIdByCatalogIdAndName(catalogId, schemaName); + Long schemaId = getSchemaIdByNameIdentifier(identifier); + // Invalidate it in the cache. + IdNameMappingService.getInstance().invalidate(identifier); if (schemaId != null) { if (cascade) { @@ -226,26 +242,25 @@ public boolean deleteSchema(NameIdentifier identifier, boolean cascade) { public int deleteSchemaMetasByLegacyTimeline(Long legacyTimeline, int limit) { return SessionUtils.doWithCommitAndFetchResult( SchemaMetaMapper.class, - mapper -> { - return mapper.deleteSchemaMetasByLegacyTimeline(legacyTimeline, limit); - }); + mapper -> mapper.deleteSchemaMetasByLegacyTimeline(legacyTimeline, limit)); } private void fillSchemaPOBuilderParentEntityId(SchemaPO.Builder builder, Namespace namespace) { NamespaceUtil.checkSchema(namespace); - Long parentEntityId = null; + + Long entityId; for (int level = 0; level < namespace.levels().length; level++) { - String name = namespace.level(level); + String[] levels = ArrayUtils.subarray(namespace.levels(), 0, level + 1); + NameIdentifier nameIdentifier = NameIdentifier.of(levels); switch (level) { case 0: - parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByName(name); - builder.withMetalakeId(parentEntityId); - continue; + entityId = + MetalakeMetaService.getInstance().getMetalakeIdByNameIdentifier(nameIdentifier); + builder.withMetalakeId(entityId); + break; case 1: - parentEntityId = - CatalogMetaService.getInstance() - .getCatalogIdByMetalakeIdAndName(parentEntityId, name); - builder.withCatalogId(parentEntityId); + entityId = CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(nameIdentifier); + builder.withCatalogId(entityId); break; } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java index 97a1d862f81..3cacefb2083 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TableMetaService.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Objects; import java.util.function.Function; +import org.apache.commons.lang3.ArrayUtils; /** The service class for table metadata. It provides the basic database operations for table. */ public class TableMetaService { @@ -56,7 +57,7 @@ public TablePO getTablePOById(Long tableId) { return tablePO; } - public Long getTableIdBySchemaIdAndName(Long schemaId, String tableName) { + private Long getTableIdBySchemaIdAndName(Long schemaId, String tableName) { Long tableId = SessionUtils.getWithoutCommit( TableMetaMapper.class, @@ -71,6 +72,21 @@ public Long getTableIdBySchemaIdAndName(Long schemaId, String tableName) { return tableId; } + public Long getTableByNameIdentifier(NameIdentifier identifier) { + NameIdentifierUtil.checkTable(identifier); + + return IdNameMappingService.getInstance() + .get( + identifier, + ident -> { + Long schemaId = + CommonMetaService.getInstance() + .getParentEntityIdByNamespace(identifier.namespace()); + + return getTableIdBySchemaIdAndName(schemaId, identifier.name()); + }); + } + public TableEntity getTableByIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkTable(identifier); @@ -160,13 +176,7 @@ public TableEntity updateTable( public boolean deleteTable(NameIdentifier identifier) { NameIdentifierUtil.checkTable(identifier); - String tableName = identifier.name(); - - Long schemaId = - CommonMetaService.getInstance().getParentEntityIdByNamespace(identifier.namespace()); - - Long tableId = getTableIdBySchemaIdAndName(schemaId, tableName); - + Long tableId = getTableByNameIdentifier(identifier); SessionUtils.doWithCommit( TableMetaMapper.class, mapper -> mapper.softDeleteTableMetasByTableId(tableId)); @@ -183,24 +193,24 @@ public int deleteTableMetasByLegacyTimeline(Long legacyTimeline, int limit) { private void fillTablePOBuilderParentEntityId(TablePO.Builder builder, Namespace namespace) { NamespaceUtil.checkTable(namespace); - Long parentEntityId = null; + Long entityId; + for (int level = 0; level < namespace.levels().length; level++) { - String name = namespace.level(level); + String[] levels = ArrayUtils.subarray(namespace.levels(), 0, level + 1); + NameIdentifier nameIdentifier = NameIdentifier.of(levels); switch (level) { case 0: - parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByName(name); - builder.withMetalakeId(parentEntityId); - continue; + entityId = + MetalakeMetaService.getInstance().getMetalakeIdByNameIdentifier(nameIdentifier); + builder.withMetalakeId(entityId); + break; case 1: - parentEntityId = - CatalogMetaService.getInstance() - .getCatalogIdByMetalakeIdAndName(parentEntityId, name); - builder.withCatalogId(parentEntityId); - continue; + entityId = CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(nameIdentifier); + builder.withCatalogId(entityId); + break; case 2: - parentEntityId = - SchemaMetaService.getInstance().getSchemaIdByCatalogIdAndName(parentEntityId, name); - builder.withSchemaId(parentEntityId); + entityId = SchemaMetaService.getInstance().getSchemaIdByNameIdentifier(nameIdentifier); + builder.withSchemaId(entityId); break; } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java index d35e0e8934f..45d42646ff3 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/TopicMetaService.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Objects; import java.util.function.Function; +import org.apache.commons.lang3.ArrayUtils; /** * The service class for topic metadata. It provides the basic database operations for topic @@ -136,24 +137,24 @@ public TopicPO getTopicPOById(Long topicId) { private void fillTopicPOBuilderParentEntityId(TopicPO.Builder builder, Namespace namespace) { NamespaceUtil.checkTopic(namespace); - Long parentEntityId = null; + Long entityId; + for (int level = 0; level < namespace.levels().length; level++) { - String name = namespace.level(level); + String[] levels = ArrayUtils.subarray(namespace.levels(), 0, level + 1); + NameIdentifier nameIdentifier = NameIdentifier.of(levels); switch (level) { case 0: - parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByName(name); - builder.withMetalakeId(parentEntityId); - continue; + entityId = + MetalakeMetaService.getInstance().getMetalakeIdByNameIdentifier(nameIdentifier); + builder.withMetalakeId(entityId); + break; case 1: - parentEntityId = - CatalogMetaService.getInstance() - .getCatalogIdByMetalakeIdAndName(parentEntityId, name); - builder.withCatalogId(parentEntityId); - continue; + entityId = CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(nameIdentifier); + builder.withCatalogId(entityId); + break; case 2: - parentEntityId = - SchemaMetaService.getInstance().getSchemaIdByCatalogIdAndName(parentEntityId, name); - builder.withSchemaId(parentEntityId); + entityId = SchemaMetaService.getInstance().getSchemaIdByNameIdentifier(nameIdentifier); + builder.withSchemaId(entityId); break; } } @@ -173,12 +174,7 @@ public TopicEntity getTopicByIdentifier(NameIdentifier identifier) { public boolean deleteTopic(NameIdentifier identifier) { NameIdentifierUtil.checkTopic(identifier); - String topicName = identifier.name(); - - Long schemaId = - CommonMetaService.getInstance().getParentEntityIdByNamespace(identifier.namespace()); - - Long topicId = getTopicIdBySchemaIdAndName(schemaId, topicName); + Long topicId = getTopicIdByNameIdentifier(identifier); SessionUtils.doWithCommit( TopicMetaMapper.class, mapper -> mapper.softDeleteTopicMetasByTopicId(topicId)); @@ -194,7 +190,20 @@ public int deleteTopicMetasByLegacyTimeline(Long legacyTimeline, int limit) { }); } - public Long getTopicIdBySchemaIdAndName(Long schemaId, String topicName) { + public Long getTopicIdByNameIdentifier(NameIdentifier identifier) { + NameIdentifierUtil.checkTopic(identifier); + + return IdNameMappingService.getInstance() + .get( + identifier, + ident -> { + Long schemaId = + CommonMetaService.getInstance().getParentEntityIdByNamespace(ident.namespace()); + return getTopicIdBySchemaIdAndName(schemaId, ident.name()); + }); + } + + private Long getTopicIdBySchemaIdAndName(Long schemaId, String topicName) { Long topicId = SessionUtils.getWithoutCommit( TopicMetaMapper.class, diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java index b0175de4383..99f5c082af7 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java @@ -67,6 +67,19 @@ private Long getUserIdByMetalakeIdAndName(Long metalakeId, String userName) { return userId; } + private Long getUserIdByNameIdentifier(NameIdentifier identifier) { + AuthorizationUtils.checkUser(identifier); + + return IdNameMappingService.getInstance() + .get( + identifier, + ident -> { + Long metalakeId = + MetalakeMetaService.getInstance().getMetalakeIdByName(ident.namespace().level(0)); + return getUserIdByMetalakeIdAndName(metalakeId, ident.name()); + }); + } + public UserEntity getUserByIdentifier(NameIdentifier identifier) { AuthorizationUtils.checkUser(identifier); @@ -123,10 +136,7 @@ public void insertUser(UserEntity userEntity, boolean overwritten) { public boolean deleteUser(NameIdentifier identifier) { AuthorizationUtils.checkUser(identifier); - - Long metalakeId = - MetalakeMetaService.getInstance().getMetalakeIdByName(identifier.namespace().level(0)); - Long userId = getUserIdByMetalakeIdAndName(metalakeId, identifier.name()); + Long userId = getUserIdByNameIdentifier(identifier); SessionUtils.doMultipleWithCommit( () -> diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java index e532bba8c43..12205a4aca4 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/MetadataObjectUtils.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.Entity; import com.datastrato.gravitino.MetadataObject; import com.datastrato.gravitino.MetadataObjects; +import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.storage.relational.po.CatalogPO; import com.datastrato.gravitino.storage.relational.po.FilesetPO; import com.datastrato.gravitino.storage.relational.po.MetalakePO; @@ -21,8 +22,8 @@ import com.datastrato.gravitino.storage.relational.service.TopicMetaService; import com.google.common.base.Joiner; import com.google.common.base.Splitter; -import java.util.List; import javax.annotation.Nullable; +import org.apache.commons.lang3.ArrayUtils; /** * MetadataObjectUtils is used for converting full name to entity id and converting entity id to @@ -37,7 +38,7 @@ public class MetadataObjectUtils { private MetadataObjectUtils() {} public static long getMetadataObjectId( - long metalakeId, String fullName, MetadataObject.Type type) { + String metalakeName, String fullName, MetadataObject.Type type) { if (fullName.equals(MetadataObjects.METADATA_OBJECT_RESERVED_NAME) && type == MetadataObject.Type.METALAKE) { return Entity.ALL_METALAKES_ENTITY_ID; @@ -47,25 +48,20 @@ public static long getMetadataObjectId( return MetalakeMetaService.getInstance().getMetalakeIdByName(fullName); } - List names = DOT_SPLITTER.splitToList(fullName); - long catalogId = - CatalogMetaService.getInstance().getCatalogIdByMetalakeIdAndName(metalakeId, names.get(0)); - if (type == MetadataObject.Type.CATALOG) { - return catalogId; - } + String[] levelsWithoutMetalake = DOT_SPLITTER.splitToList(fullName).toArray(new String[0]); + String[] fullLevels = ArrayUtils.addFirst(levelsWithoutMetalake, metalakeName); + NameIdentifier identifier = NameIdentifier.of(fullLevels); - long schemaId = - SchemaMetaService.getInstance().getSchemaIdByCatalogIdAndName(catalogId, names.get(1)); - if (type == MetadataObject.Type.SCHEMA) { - return schemaId; - } - - if (type == MetadataObject.Type.FILESET) { - return FilesetMetaService.getInstance().getFilesetIdBySchemaIdAndName(schemaId, names.get(2)); + if (type == MetadataObject.Type.CATALOG) { + return CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(identifier); + } else if (type == MetadataObject.Type.SCHEMA) { + return SchemaMetaService.getInstance().getSchemaIdByNameIdentifier(identifier); + } else if (type == MetadataObject.Type.FILESET) { + return FilesetMetaService.getInstance().getFilesetIdByNameIdentifier(identifier); } else if (type == MetadataObject.Type.TOPIC) { - return TopicMetaService.getInstance().getTopicIdBySchemaIdAndName(schemaId, names.get(2)); + return TopicMetaService.getInstance().getTopicIdByNameIdentifier(identifier); } else if (type == MetadataObject.Type.TABLE) { - return TableMetaService.getInstance().getTableIdBySchemaIdAndName(schemaId, names.get(2)); + return TableMetaService.getInstance().getTableByNameIdentifier(identifier); } throw new IllegalArgumentException(String.format("Doesn't support the type %s", type)); From 18c84fc5afc70612bdc9d8ac591564cb796a5222 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 27 Jun 2024 20:51:09 +0800 Subject: [PATCH 02/14] fix --- .../relational/service/CommonMetaService.java | 12 ++++++------ .../relational/service/IdNameMappingService.java | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CommonMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CommonMetaService.java index 90457da30fc..57b531d8b28 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CommonMetaService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/CommonMetaService.java @@ -26,23 +26,23 @@ public Long getParentEntityIdByNamespace(Namespace namespace) { String[] level = namespace.levels(); NameIdentifier ident = NameIdentifier.of(level); - Long parentEntityId = null; + Long entityId = null; switch (level.length) { case 1: - parentEntityId = MetalakeMetaService.getInstance().getMetalakeIdByNameIdentifier(ident); + entityId = MetalakeMetaService.getInstance().getMetalakeIdByNameIdentifier(ident); break; case 2: - parentEntityId = CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(ident); + entityId = CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(ident); break; case 3: - parentEntityId = SchemaMetaService.getInstance().getSchemaIdByNameIdentifier(ident); + entityId = SchemaMetaService.getInstance().getSchemaIdByNameIdentifier(ident); break; } Preconditions.checkState( - parentEntityId != null && parentEntityId > 0, + entityId != null && entityId > 0, "Parent entity id should not be null and should be greater than 0."); - return parentEntityId; + return entityId; } } diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/IdNameMappingService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/IdNameMappingService.java index 3cf2f1c8f2e..31d76abae00 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/IdNameMappingService.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/IdNameMappingService.java @@ -25,7 +25,7 @@ public class IdNameMappingService implements Closeable { private IdNameMappingService() { this.ident2IdCache = Caffeine.newBuilder() - .expireAfterAccess(24 * 3600 * 1000 /* 1 day */, TimeUnit.MILLISECONDS) + .expireAfterAccess(24 * 3600 * 1000L /* 1 day */, TimeUnit.MILLISECONDS) .maximumSize(1000000) .initialCapacity(1000) .scheduler( From 732c3b4698354a4e5add61b5480d4a17cd2a16ec Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 16 Jul 2024 16:55:16 +0800 Subject: [PATCH 03/14] fix --- .../storage/relational/JDBCBackend.java | 16 +++---- .../service/CatalogMetaService.java | 6 ++- .../service/FilesetMetaService.java | 2 +- .../relational/service/GroupMetaService.java | 2 +- .../service/MetalakeMetaService.java | 4 +- ...Service.java => NameIdMappingService.java} | 43 ++++++++++++++--- .../relational/service/RoleMetaService.java | 2 +- .../relational/service/SchemaMetaService.java | 4 +- .../relational/service/TableMetaService.java | 2 +- .../relational/service/TopicMetaService.java | 2 +- .../relational/service/UserMetaService.java | 2 +- .../service/TestIdNameMappingService.java | 47 +++++++++++++++++++ 12 files changed, 106 insertions(+), 26 deletions(-) rename core/src/main/java/org/apache/gravitino/storage/relational/service/{IdNameMappingService.java => NameIdMappingService.java} (69%) create mode 100644 core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 424fd43bac9..d9e322b3961 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -50,7 +50,7 @@ import org.apache.gravitino.storage.relational.service.CatalogMetaService; import org.apache.gravitino.storage.relational.service.FilesetMetaService; import org.apache.gravitino.storage.relational.service.GroupMetaService; -import org.apache.gravitino.storage.relational.service.IdNameMappingService; +import org.apache.gravitino.storage.relational.service.NameIdMappingService; import org.apache.gravitino.storage.relational.service.MetalakeMetaService; import org.apache.gravitino.storage.relational.service.RoleMetaService; import org.apache.gravitino.storage.relational.service.SchemaMetaService; @@ -145,7 +145,7 @@ public void insert(E e, boolean overwritten) "Unsupported entity type: %s for insert operation", e.getClass()); } - IdNameMappingService.getInstance().put(e.nameIdentifier(), e.id()); + NameIdMappingService.getInstance().put(e.nameIdentifier(), e.id()); } @Override @@ -154,7 +154,7 @@ public E update( throws IOException, NoSuchEntityException, EntityAlreadyExistsException { // Remove all the children entities in the cache as we can't guarantee the children entities // are still valid after the parent entity is updated. - IdNameMappingService.getInstance().invalidateWithPrefix(ident); + NameIdMappingService.getInstance().invalidateWithPrefix(ident); switch (entityType) { case METALAKE: return (E) MetalakeMetaService.getInstance().updateMetalake(ident, updater); @@ -214,10 +214,10 @@ public E get( @Override public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { // Invalidate the cache first - IdNameMappingService.getInstance().invalidate(ident); + NameIdMappingService.getInstance().invalidate(ident); if (cascade) { // Remove all the children entities in the cache; - IdNameMappingService.getInstance().invalidateWithPrefix(ident); + NameIdMappingService.getInstance().invalidateWithPrefix(ident); } try { @@ -249,8 +249,8 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea } finally { // Remove the entity from the cache again because we may add the cache during the deletion // process - IdNameMappingService.getInstance().invalidate(ident); - IdNameMappingService.getInstance().invalidateWithPrefix(ident); + NameIdMappingService.getInstance().invalidate(ident); + NameIdMappingService.getInstance().invalidateWithPrefix(ident); } } @@ -342,7 +342,7 @@ public int deleteOldVersionData(Entity.EntityType entityType, long versionRetent public void close() throws IOException { SqlSessionFactoryHelper.getInstance().close(); - IdNameMappingService.getInstance().close(); + NameIdMappingService.getInstance().close(); if (jdbcDatabase != null) { jdbcDatabase.close(); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java index fd6fe27ba25..b385d4410b8 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.storage.relational.service; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.io.IOException; import java.util.List; @@ -82,7 +83,8 @@ public CatalogPO getCatalogPOById(Long catalogId) { return catalogPO; } - private Long getCatalogIdByMetalakeIdAndName(Long metalakeId, String catalogName) { + @VisibleForTesting + public Long getCatalogIdByMetalakeIdAndName(Long metalakeId, String catalogName) { Long catalogId = SessionUtils.getWithoutCommit( CatalogMetaMapper.class, @@ -100,7 +102,7 @@ private Long getCatalogIdByMetalakeIdAndName(Long metalakeId, String catalogName public Long getCatalogIdByNameIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkCatalog(identifier); - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( identifier, ident -> { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FilesetMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FilesetMetaService.java index 2da11c7eb42..fccae990bf5 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FilesetMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FilesetMetaService.java @@ -83,7 +83,7 @@ public FilesetPO getFilesetPOById(Long filesetId) { public Long getFilesetIdByNameIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkFileset(identifier); - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( identifier, ident -> { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java index d3b6711a40a..18c0a4e2be4 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java @@ -83,7 +83,7 @@ private Long getGroupIdByMetalakeIdAndName(Long metalakeId, String groupName) { } private Long getGroupIdByNameIdentifier(NameIdentifier identifier) { - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( identifier, ident -> { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java index a66afb71da2..8bdbcd86a09 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java @@ -74,7 +74,7 @@ public List listMetalakes() { public Long getMetalakeIdByNameIdentifier(NameIdentifier nameIdentifier) { NameIdentifierUtil.checkMetalake(nameIdentifier); - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( nameIdentifier, ident -> { @@ -93,7 +93,7 @@ public Long getMetalakeIdByNameIdentifier(NameIdentifier nameIdentifier) { } public Long getMetalakeIdByName(String metalakeName) { - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( NameIdentifierUtil.ofMetalake(metalakeName), ident -> { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/IdNameMappingService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java similarity index 69% rename from core/src/main/java/org/apache/gravitino/storage/relational/service/IdNameMappingService.java rename to core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java index 571be72e6f5..6b236074182 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/IdNameMappingService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java @@ -21,6 +21,8 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Scheduler; +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.Closeable; import java.io.IOException; @@ -29,13 +31,13 @@ import java.util.function.Function; import org.apache.gravitino.NameIdentifier; -public class IdNameMappingService implements Closeable { +public class NameIdMappingService implements Closeable { - private static volatile IdNameMappingService instance; + private static volatile NameIdMappingService instance; private Cache ident2IdCache; - private IdNameMappingService() { + private NameIdMappingService() { this.ident2IdCache = Caffeine.newBuilder() .expireAfterAccess(24 * 3600 * 1000L /* 1 day */, TimeUnit.MILLISECONDS) @@ -52,11 +54,11 @@ private IdNameMappingService() { .build(); } - public static IdNameMappingService getInstance() { + public static NameIdMappingService getInstance() { if (instance == null) { - synchronized (IdNameMappingService.class) { + synchronized (NameIdMappingService.class) { if (instance == null) { - instance = new IdNameMappingService(); + instance = new NameIdMappingService(); } } } @@ -72,6 +74,35 @@ public Long get(NameIdentifier key, Function mappingFuncti return ident2IdCache.get(key, mappingFunction); } + public Long get(NameIdentifier key) { + return ident2IdCache.getIfPresent(key); + } + + public NameIdentifier getById(Long value, Function mappingFunction) { + synchronized (this) { + BiMap map = HashBiMap.create(ident2IdCache.asMap()); + if (map.containsValue(value)) { + return map.inverse().get(value); + } else { + NameIdentifier nameIdentifier = mappingFunction.apply(value); + if (nameIdentifier != null) { + ident2IdCache.put(nameIdentifier, value); + } + return nameIdentifier; + } + } + } + + public NameIdentifier getById(Long value) { + synchronized (this) { + BiMap map = HashBiMap.create(ident2IdCache.asMap()); + if (map.containsValue(value)) { + return map.inverse().get(value); + } + return null; + } + } + public void invalidate(NameIdentifier key) { ident2IdCache.invalidate(key); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 473fc015922..8d2ffd01994 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -87,7 +87,7 @@ public Long getRoleIdByMetalakeIdAndName(Long metalakeId, String roleName) { public Long getRoleIdByNameIdentifier(NameIdentifier identifier) { AuthorizationUtils.checkRole(identifier); - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( identifier, ident -> { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index b08d4552c8d..421f309397b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -92,7 +92,7 @@ private Long getSchemaIdByCatalogIdAndName(Long catalogId, String schemaName) { } public Long getSchemaIdByNameIdentifier(NameIdentifier identifier) { - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( identifier, ident -> { @@ -198,7 +198,7 @@ public boolean deleteSchema(NameIdentifier identifier, boolean cascade) { String schemaName = identifier.name(); Long schemaId = getSchemaIdByNameIdentifier(identifier); // Invalidate it in the cache. - IdNameMappingService.getInstance().invalidate(identifier); + NameIdMappingService.getInstance().invalidate(identifier); if (schemaId != null) { if (cascade) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 32e3f456d21..b56de03cf40 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -89,7 +89,7 @@ private Long getTableIdBySchemaIdAndName(Long schemaId, String tableName) { public Long getTableByNameIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkTable(identifier); - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( identifier, ident -> { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TopicMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TopicMetaService.java index fefca517118..844bc3e5687 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TopicMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TopicMetaService.java @@ -207,7 +207,7 @@ public int deleteTopicMetasByLegacyTimeline(Long legacyTimeline, int limit) { public Long getTopicIdByNameIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkTopic(identifier); - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( identifier, ident -> { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java index 46045132aff..e32515b2a97 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java @@ -85,7 +85,7 @@ private Long getUserIdByMetalakeIdAndName(Long metalakeId, String userName) { private Long getUserIdByNameIdentifier(NameIdentifier identifier) { AuthorizationUtils.checkUser(identifier); - return IdNameMappingService.getInstance() + return NameIdMappingService.getInstance() .get( identifier, ident -> { diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java new file mode 100644 index 00000000000..36476e7cf3e --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.storage.relational.service; + +import java.io.IOException; +import org.apache.gravitino.NameIdentifier; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestIdNameMappingService { + + @Test + public void testGetInstance() throws IOException { + NameIdMappingService instance = NameIdMappingService.getInstance(); + Assertions.assertNull(instance.get(NameIdentifier.of("m1"))); + Assertions.assertEquals(1L, instance.get(NameIdentifier.of("m1"), (NameIdentifier key) -> 1L)); + + instance.put(NameIdentifier.of("m2"), 2L); + Assertions.assertEquals(2L, instance.get(NameIdentifier.of("m2"))); + + instance.invalidate(NameIdentifier.of("m2")); + Assertions.assertNull(instance.get(NameIdentifier.of("m2"))); + + Assertions.assertEquals(NameIdentifier.of("m1"), instance.getById(1L)); + + Assertions.assertEquals( + NameIdentifier.of("m2"), instance.getById(2L, (Long value) -> NameIdentifier.of("m2"))); + + instance.close(); + } +} From ee46442b58123ff86fe4ef3dbc055fab3a76d470 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 16 Jul 2024 17:01:12 +0800 Subject: [PATCH 04/14] fix --- .../org/apache/gravitino/storage/relational/JDBCBackend.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index d9e322b3961..67ec99b7c35 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -50,8 +50,8 @@ import org.apache.gravitino.storage.relational.service.CatalogMetaService; import org.apache.gravitino.storage.relational.service.FilesetMetaService; import org.apache.gravitino.storage.relational.service.GroupMetaService; -import org.apache.gravitino.storage.relational.service.NameIdMappingService; import org.apache.gravitino.storage.relational.service.MetalakeMetaService; +import org.apache.gravitino.storage.relational.service.NameIdMappingService; import org.apache.gravitino.storage.relational.service.RoleMetaService; import org.apache.gravitino.storage.relational.service.SchemaMetaService; import org.apache.gravitino.storage.relational.service.TableMetaService; @@ -249,7 +249,6 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea } finally { // Remove the entity from the cache again because we may add the cache during the deletion // process - NameIdMappingService.getInstance().invalidate(ident); NameIdMappingService.getInstance().invalidateWithPrefix(ident); } } From 9efd143436a84806ffd9f3fe669194c2f9c90518 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 17 Jul 2024 15:02:36 +0800 Subject: [PATCH 05/14] fix --- .../gravitino/catalog/hadoop/TestHadoopCatalogOperations.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java index 73b0c5dfc77..2cf188cd92d 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/TestHadoopCatalogOperations.java @@ -160,6 +160,9 @@ public static void setUp() { MetalakeMetaService metalakeMetaService = MetalakeMetaService.getInstance(); MetalakeMetaService spyMetaservice = Mockito.spy(metalakeMetaService); doReturn(1L).when(spyMetaservice).getMetalakeIdByName(Mockito.anyString()); + doReturn(1L) + .when(spyMetaservice) + .getMetalakeIdByNameIdentifier(Mockito.any(NameIdentifier.class)); CatalogMetaService catalogMetaService = CatalogMetaService.getInstance(); CatalogMetaService spyCatalogMetaService = Mockito.spy(catalogMetaService); From 36d420a01c0b564e856dc768511738effbe89925 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 17 Jul 2024 16:27:21 +0800 Subject: [PATCH 06/14] fix --- .../service/MetadataObjectService.java | 20 ++++++++++--------- .../relational/service/TagMetaService.java | 9 +++------ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index 0e7c94e9283..3f1922a903e 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -20,8 +20,9 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import java.util.List; import javax.annotation.Nullable; -import org.apache.commons.lang3.ArrayUtils; import org.apache.gravitino.Entity; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; @@ -56,20 +57,21 @@ public static long getMetadataObjectId( return MetalakeMetaService.getInstance().getMetalakeIdByName(fullName); } - String[] levelsWithoutMetalake = DOT_SPLITTER.splitToList(fullName).toArray(new String[0]); - String[] fullLevels = ArrayUtils.addFirst(levelsWithoutMetalake, metalakeName); - NameIdentifier identifier = NameIdentifier.of(fullLevels); + List names = DOT_SPLITTER.splitToList(fullName); + List realNames = Lists.newArrayList(metalakeName); + realNames.addAll(names); + NameIdentifier nameIdentifier = NameIdentifier.of(realNames.toArray(new String[0])); if (type == MetadataObject.Type.CATALOG) { - return CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(identifier); + return CatalogMetaService.getInstance().getCatalogIdByNameIdentifier(nameIdentifier); } else if (type == MetadataObject.Type.SCHEMA) { - return SchemaMetaService.getInstance().getSchemaIdByNameIdentifier(identifier); + return SchemaMetaService.getInstance().getSchemaIdByNameIdentifier(nameIdentifier); } else if (type == MetadataObject.Type.FILESET) { - return FilesetMetaService.getInstance().getFilesetIdByNameIdentifier(identifier); + return FilesetMetaService.getInstance().getFilesetIdByNameIdentifier(nameIdentifier); } else if (type == MetadataObject.Type.TOPIC) { - return TopicMetaService.getInstance().getTopicIdByNameIdentifier(identifier); + return TopicMetaService.getInstance().getTopicIdByNameIdentifier(nameIdentifier); } else if (type == MetadataObject.Type.TABLE) { - return TableMetaService.getInstance().getTableByNameIdentifier(identifier); + return TableMetaService.getInstance().getTableByNameIdentifier(nameIdentifier); } throw new IllegalArgumentException(String.format("Doesn't support the type %s", type)); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java index 71b8275275f..586b2535fca 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TagMetaService.java @@ -162,10 +162,9 @@ public List listTagsForMetadataObject( List tagPOs = null; try { - Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); Long metadataObjectId = MetadataObjectService.getMetadataObjectId( - metalakeId, metadataObject.fullName(), metadataObject.type()); + metalake, metadataObject.fullName(), metadataObject.type()); tagPOs = SessionUtils.doWithoutCommitAndFetchResult( @@ -191,10 +190,9 @@ public TagEntity getTagForMetadataObject( TagPO tagPO = null; try { - Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); Long metadataObjectId = MetadataObjectService.getMetadataObjectId( - metalakeId, metadataObject.fullName(), metadataObject.type()); + metalake, metadataObject.fullName(), metadataObject.type()); tagPO = SessionUtils.getWithoutCommit( @@ -262,10 +260,9 @@ public List associateTagsWithMetadataObject( String metalake = objectIdent.namespace().level(0); try { - Long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(metalake); Long metadataObjectId = MetadataObjectService.getMetadataObjectId( - metalakeId, metadataObject.fullName(), metadataObject.type()); + metalake, metadataObject.fullName(), metadataObject.type()); // Fetch all the tags need to associate with the metadata object. List tagNamesToAdd = From f70e1adf8ace6749db3c7646278144adf19af9ee Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 17 Jul 2024 18:22:16 +0800 Subject: [PATCH 07/14] fix --- .../gravitino/catalog/kafka/TestKafkaCatalogOperations.java | 1 + 1 file changed, 1 insertion(+) diff --git a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java index 58cd550425f..a40e246b7e4 100644 --- a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java @@ -157,6 +157,7 @@ public static void setUp() { MetalakeMetaService metalakeMetaService = MetalakeMetaService.getInstance(); MetalakeMetaService spyMetaservice = Mockito.spy(metalakeMetaService); doReturn(1L).when(spyMetaservice).getMetalakeIdByName(Mockito.anyString()); + doReturn(1L).when(spyMetaservice).getMetalakeIdByNameIdentifier(Mockito.any(NameIdentifier.class)); CatalogMetaService catalogMetaService = CatalogMetaService.getInstance(); CatalogMetaService spyCatalogMetaService = Mockito.spy(catalogMetaService); From ba965f0ee855ffc42e203c9c3b69eeb9159c0040 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 17 Jul 2024 18:52:48 +0800 Subject: [PATCH 08/14] fix the code style problem --- .../gravitino/catalog/kafka/TestKafkaCatalogOperations.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java index a40e246b7e4..8265b0585f5 100644 --- a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java +++ b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/TestKafkaCatalogOperations.java @@ -157,7 +157,9 @@ public static void setUp() { MetalakeMetaService metalakeMetaService = MetalakeMetaService.getInstance(); MetalakeMetaService spyMetaservice = Mockito.spy(metalakeMetaService); doReturn(1L).when(spyMetaservice).getMetalakeIdByName(Mockito.anyString()); - doReturn(1L).when(spyMetaservice).getMetalakeIdByNameIdentifier(Mockito.any(NameIdentifier.class)); + doReturn(1L) + .when(spyMetaservice) + .getMetalakeIdByNameIdentifier(Mockito.any(NameIdentifier.class)); CatalogMetaService catalogMetaService = CatalogMetaService.getInstance(); CatalogMetaService spyCatalogMetaService = Mockito.spy(catalogMetaService); From 57fae17bb3249121ca2b371b44ebc3d9b6e59f7a Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 6 Aug 2024 21:46:05 +0800 Subject: [PATCH 09/14] Fix --- .../TestDorisTablePartitionOperations.java | 2 +- .../relational/service/GroupMetaService.java | 2 +- .../relational/service/OwnerMetaService.java | 24 +++++++++---------- .../relational/service/UserMetaService.java | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java index 40cb3254bc6..ffbcb98d3c6 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTablePartitionOperations.java @@ -57,7 +57,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -@Tag("gravitino-docker-it") +@Tag("gravitino-docker-test") public class TestDorisTablePartitionOperations extends TestDoris { private static final String databaseName = GravitinoITUtils.genRandomName("doris_test_db"); private static final Integer DEFAULT_BUCKET_SIZE = 1; diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java index 7fe0e8323c4..8d4861b7114 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java @@ -83,7 +83,7 @@ public Long getGroupIdByMetalakeIdAndName(Long metalakeId, String groupName) { return groupId; } - private Long getGroupIdByNameIdentifier(NameIdentifier identifier) { + public Long getGroupIdByNameIdentifier(NameIdentifier identifier) { return NameIdMappingService.getInstance() .get( identifier, diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java index 3354e7a9591..f33775e7ec8 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/OwnerMetaService.java @@ -20,6 +20,7 @@ import java.util.Optional; import org.apache.gravitino.Entity; +import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; @@ -40,9 +41,8 @@ public static OwnerMetaService getInstance() { } public Optional getOwner(NameIdentifier identifier, Entity.EntityType type) { - long metalakeId = - MetalakeMetaService.getInstance().getMetalakeIdByName(getMetalake(identifier)); - Long entityId = getEntityId(metalakeId, identifier, type); + String metalakeName = getMetalake(identifier); + Long entityId = getEntityId(metalakeName, identifier, type); OwnerRelPO ownerRelPO = SessionUtils.getWithoutCommit( @@ -73,10 +73,12 @@ public void setOwner( Entity.EntityType entityType, NameIdentifier owner, Entity.EntityType ownerType) { - long metalakeId = MetalakeMetaService.getInstance().getMetalakeIdByName(getMetalake(entity)); - Long entityId = getEntityId(metalakeId, entity, entityType); - Long ownerId = getEntityId(metalakeId, owner, ownerType); + String metalakeName = getMetalake(entity); + Long metalakeId = + getEntityId(metalakeName, NameIdentifier.of(metalakeName), EntityType.METALAKE); + Long entityId = getEntityId(metalakeName, entity, entityType); + Long ownerId = getEntityId(metalakeName, owner, ownerType); OwnerRelPO ownerRelPO = POConverters.initializeOwnerRelPOsWithVersion( @@ -95,18 +97,16 @@ public void setOwner( } private static long getEntityId( - long metalakeId, NameIdentifier identifier, Entity.EntityType type) { + String metalake, NameIdentifier identifier, Entity.EntityType type) { switch (type) { case USER: - return UserMetaService.getInstance() - .getUserIdByMetalakeIdAndName(metalakeId, identifier.name()); + return UserMetaService.getInstance().getUserIdByNameIdentifier(identifier); case GROUP: - return GroupMetaService.getInstance() - .getGroupIdByMetalakeIdAndName(metalakeId, identifier.name()); + return GroupMetaService.getInstance().getGroupIdByNameIdentifier(identifier); default: MetadataObject object = NameIdentifierUtil.toMetadataObject(identifier, type); return MetadataObjectService.getMetadataObjectId( - metalakeId, object.fullName(), object.type()); + metalake, object.fullName(), object.type()); } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java index 804a1991f34..dcfacde7db3 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java @@ -83,7 +83,7 @@ public Long getUserIdByMetalakeIdAndName(Long metalakeId, String userName) { return userId; } - private Long getUserIdByNameIdentifier(NameIdentifier identifier) { + public Long getUserIdByNameIdentifier(NameIdentifier identifier) { AuthorizationUtils.checkUser(identifier); return NameIdMappingService.getInstance() From 1d13217e1edf5f02fc96f543bf3fe33d49723180 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 6 Aug 2024 21:55:00 +0800 Subject: [PATCH 10/14] Fix --- .../storage/relational/service/MetadataObjectService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java index ca46955d487..ddb8f39930d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetadataObjectService.java @@ -54,7 +54,8 @@ public static long getMetadataObjectId( } if (type == MetadataObject.Type.ROLE) { - NameIdentifier nameIdentifier = NameIdentifier.of(metalakeName, SYSTEM_CATALOG_RESERVED_NAME, ROLE_SCHEMA_NAME, fullName); + NameIdentifier nameIdentifier = + NameIdentifier.of(metalakeName, SYSTEM_CATALOG_RESERVED_NAME, ROLE_SCHEMA_NAME, fullName); return RoleMetaService.getInstance().getRoleIdByNameIdentifier(nameIdentifier); } From 593d84e2028ecc34a18d401ed6b174359945778d Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 7 Aug 2024 11:54:16 +0800 Subject: [PATCH 11/14] Change the cache key from NameIdentifier to EntityIdentifier --- .../storage/relational/JDBCBackend.java | 26 ++++++++++--- .../service/CatalogMetaService.java | 9 +++-- .../service/FilesetMetaService.java | 10 +++-- .../relational/service/GroupMetaService.java | 8 ++-- .../service/MetalakeMetaService.java | 12 ++++-- .../service/NameIdMappingService.java | 39 +++++++++++++------ .../relational/service/RoleMetaService.java | 10 +++-- .../relational/service/SchemaMetaService.java | 17 +++++--- .../relational/service/TableMetaService.java | 8 ++-- .../relational/service/TopicMetaService.java | 9 +++-- .../relational/service/UserMetaService.java | 9 +++-- .../service/TestIdNameMappingService.java | 25 +++++++----- 12 files changed, 126 insertions(+), 56 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index cab6753fc12..65eaabe2124 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -30,6 +30,7 @@ import org.apache.gravitino.Config; import org.apache.gravitino.Configs; import org.apache.gravitino.Entity; +import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.MetadataObject; @@ -55,6 +56,7 @@ import org.apache.gravitino.storage.relational.service.GroupMetaService; import org.apache.gravitino.storage.relational.service.MetalakeMetaService; import org.apache.gravitino.storage.relational.service.NameIdMappingService; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.service.OwnerMetaService; import org.apache.gravitino.storage.relational.service.RoleMetaService; import org.apache.gravitino.storage.relational.service.SchemaMetaService; @@ -124,32 +126,44 @@ public boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws @Override public void insert(E e, boolean overwritten) throws EntityAlreadyExistsException, IOException { + EntityType entityType; if (e instanceof BaseMetalake) { MetalakeMetaService.getInstance().insertMetalake((BaseMetalake) e, overwritten); + entityType = EntityType.METALAKE; } else if (e instanceof CatalogEntity) { CatalogMetaService.getInstance().insertCatalog((CatalogEntity) e, overwritten); + entityType = EntityType.CATALOG; } else if (e instanceof SchemaEntity) { SchemaMetaService.getInstance().insertSchema((SchemaEntity) e, overwritten); + entityType = EntityType.SCHEMA; } else if (e instanceof TableEntity) { TableMetaService.getInstance().insertTable((TableEntity) e, overwritten); + entityType = EntityType.TABLE; } else if (e instanceof FilesetEntity) { FilesetMetaService.getInstance().insertFileset((FilesetEntity) e, overwritten); + entityType = EntityType.FILESET; } else if (e instanceof TopicEntity) { TopicMetaService.getInstance().insertTopic((TopicEntity) e, overwritten); + entityType = EntityType.TOPIC; } else if (e instanceof UserEntity) { UserMetaService.getInstance().insertUser((UserEntity) e, overwritten); + entityType = EntityType.USER; } else if (e instanceof RoleEntity) { RoleMetaService.getInstance().insertRole((RoleEntity) e, overwritten); + entityType = EntityType.ROLE; } else if (e instanceof GroupEntity) { GroupMetaService.getInstance().insertGroup((GroupEntity) e, overwritten); + entityType = EntityType.GROUP; } else if (e instanceof TagEntity) { TagMetaService.getInstance().insertTag((TagEntity) e, overwritten); + entityType = EntityType.TAG; } else { throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for insert operation", e.getClass()); } - NameIdMappingService.getInstance().put(e.nameIdentifier(), e.id()); + NameIdMappingService.getInstance() + .put(EntityIdentifier.of(e.nameIdentifier(), entityType), e.id()); } @Override @@ -158,7 +172,8 @@ public E update( throws IOException, NoSuchEntityException, EntityAlreadyExistsException { // Remove all the children entities in the cache as we can't guarantee the children entities // are still valid after the parent entity is updated. - NameIdMappingService.getInstance().invalidateWithPrefix(ident); + EntityIdentifier entityIdentifier = EntityIdentifier.of(ident, entityType); + NameIdMappingService.getInstance().invalidateWithPrefix(entityIdentifier); switch (entityType) { case METALAKE: return (E) MetalakeMetaService.getInstance().updateMetalake(ident, updater); @@ -218,10 +233,11 @@ public E get( @Override public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolean cascade) { // Invalidate the cache first - NameIdMappingService.getInstance().invalidate(ident); + EntityIdentifier entityIdentifier = EntityIdentifier.of(ident, entityType); + NameIdMappingService.getInstance().invalidate(entityIdentifier); if (cascade) { // Remove all the children entities in the cache; - NameIdMappingService.getInstance().invalidateWithPrefix(ident); + NameIdMappingService.getInstance().invalidateWithPrefix(entityIdentifier); } try { @@ -253,7 +269,7 @@ public boolean delete(NameIdentifier ident, Entity.EntityType entityType, boolea } finally { // Remove the entity from the cache again because we may add the cache during the deletion // process - NameIdMappingService.getInstance().invalidateWithPrefix(ident); + NameIdMappingService.getInstance().invalidateWithPrefix(entityIdentifier); } } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java index 3cd7d28c5c3..fc04b678a36 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/CatalogMetaService.java @@ -42,6 +42,7 @@ import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper; import org.apache.gravitino.storage.relational.po.CatalogPO; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; @@ -103,14 +104,16 @@ public Long getCatalogIdByMetalakeIdAndName(Long metalakeId, String catalogName) public Long getCatalogIdByNameIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkCatalog(identifier); + EntityIdentifier catalogIdent = EntityIdentifier.of(identifier, Entity.EntityType.CATALOG); return NameIdMappingService.getInstance() .get( - identifier, + catalogIdent, ident -> { - String catalogName = ident.name(); + String catalogName = ident.ident.name(); Long metalakeId = - CommonMetaService.getInstance().getParentEntityIdByNamespace(ident.namespace()); + CommonMetaService.getInstance() + .getParentEntityIdByNamespace(ident.ident.namespace()); return getCatalogIdByMetalakeIdAndName(metalakeId, catalogName); }); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/FilesetMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/FilesetMetaService.java index 3cb8a51abe8..e64a1b08363 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/FilesetMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/FilesetMetaService.java @@ -25,6 +25,7 @@ import java.util.function.Function; import org.apache.commons.lang3.ArrayUtils; import org.apache.gravitino.Entity; +import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; @@ -36,6 +37,7 @@ import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.po.FilesetMaxVersionPO; import org.apache.gravitino.storage.relational.po.FilesetPO; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; @@ -85,13 +87,15 @@ public FilesetPO getFilesetPOById(Long filesetId) { public Long getFilesetIdByNameIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkFileset(identifier); + EntityIdentifier fileIdentifier = EntityIdentifier.of(identifier, EntityType.FILESET); return NameIdMappingService.getInstance() .get( - identifier, + fileIdentifier, ident -> { Long schemaId = - CommonMetaService.getInstance().getParentEntityIdByNamespace(ident.namespace()); - return getFilesetIdBySchemaIdAndName(schemaId, ident.name()); + CommonMetaService.getInstance() + .getParentEntityIdByNamespace(ident.ident.namespace()); + return getFilesetIdBySchemaIdAndName(schemaId, ident.ident.name()); }); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java index 8d4861b7114..3562d20a55c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/GroupMetaService.java @@ -39,6 +39,7 @@ import org.apache.gravitino.storage.relational.po.GroupPO; import org.apache.gravitino.storage.relational.po.GroupRoleRelPO; import org.apache.gravitino.storage.relational.po.RolePO; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; @@ -84,14 +85,15 @@ public Long getGroupIdByMetalakeIdAndName(Long metalakeId, String groupName) { } public Long getGroupIdByNameIdentifier(NameIdentifier identifier) { + EntityIdentifier groupIdentifier = EntityIdentifier.of(identifier, Entity.EntityType.GROUP); return NameIdMappingService.getInstance() .get( - identifier, + groupIdentifier, ident -> { Long metalakeId = MetalakeMetaService.getInstance() - .getMetalakeIdByName(identifier.namespace().level(0)); - return getGroupIdByMetalakeIdAndName(metalakeId, identifier.name()); + .getMetalakeIdByName(ident.ident.namespace().level(0)); + return getGroupIdByMetalakeIdAndName(metalakeId, ident.ident.name()); }); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java index c41a296fa2b..2a4f694cf9b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/MetalakeMetaService.java @@ -25,6 +25,7 @@ import java.util.Objects; import java.util.function.Function; import org.apache.gravitino.Entity; +import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.exceptions.NoSuchEntityException; @@ -48,6 +49,7 @@ import org.apache.gravitino.storage.relational.mapper.UserMetaMapper; import org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper; import org.apache.gravitino.storage.relational.po.MetalakePO; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; @@ -75,14 +77,15 @@ public List listMetalakes() { public Long getMetalakeIdByNameIdentifier(NameIdentifier nameIdentifier) { NameIdentifierUtil.checkMetalake(nameIdentifier); + EntityIdentifier metalakeIdentifier = EntityIdentifier.of(nameIdentifier, EntityType.METALAKE); return NameIdMappingService.getInstance() .get( - nameIdentifier, + metalakeIdentifier, ident -> { Long metalakeId = SessionUtils.getWithoutCommit( MetalakeMetaMapper.class, - mapper -> mapper.selectMetalakeIdMetaByName(nameIdentifier.name())); + mapper -> mapper.selectMetalakeIdMetaByName(ident.ident.name())); if (metalakeId == null) { throw new NoSuchEntityException( NoSuchEntityException.NO_SUCH_ENTITY_MESSAGE, @@ -94,9 +97,12 @@ public Long getMetalakeIdByNameIdentifier(NameIdentifier nameIdentifier) { } public Long getMetalakeIdByName(String metalakeName) { + EntityIdentifier metalakeIdentifier = + EntityIdentifier.of(NameIdentifier.of(metalakeName), EntityType.METALAKE); + return NameIdMappingService.getInstance() .get( - NameIdentifierUtil.ofMetalake(metalakeName), + metalakeIdentifier, ident -> { Long metalakeId = SessionUtils.getWithoutCommit( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java index 6b236074182..71c137e724d 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java @@ -29,13 +29,14 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.NameIdentifier; public class NameIdMappingService implements Closeable { private static volatile NameIdMappingService instance; - private Cache ident2IdCache; + private Cache ident2IdCache; private NameIdMappingService() { this.ident2IdCache = @@ -54,6 +55,20 @@ private NameIdMappingService() { .build(); } + public static class EntityIdentifier { + final NameIdentifier ident; + final EntityType type; + + private EntityIdentifier(NameIdentifier ident, EntityType type) { + this.ident = ident; + this.type = type; + } + + public static EntityIdentifier of(NameIdentifier ident, EntityType type) { + return new EntityIdentifier(ident, type); + } + } + public static NameIdMappingService getInstance() { if (instance == null) { synchronized (NameIdMappingService.class) { @@ -66,25 +81,25 @@ public static NameIdMappingService getInstance() { return instance; } - public void put(NameIdentifier key, Long value) { + public void put(EntityIdentifier key, Long value) { ident2IdCache.put(key, value); } - public Long get(NameIdentifier key, Function mappingFunction) { + public Long get(EntityIdentifier key, Function mappingFunction) { return ident2IdCache.get(key, mappingFunction); } - public Long get(NameIdentifier key) { + public Long get(EntityIdentifier key) { return ident2IdCache.getIfPresent(key); } - public NameIdentifier getById(Long value, Function mappingFunction) { + public EntityIdentifier getById(Long value, Function mappingFunction) { synchronized (this) { - BiMap map = HashBiMap.create(ident2IdCache.asMap()); + BiMap map = HashBiMap.create(ident2IdCache.asMap()); if (map.containsValue(value)) { return map.inverse().get(value); } else { - NameIdentifier nameIdentifier = mappingFunction.apply(value); + EntityIdentifier nameIdentifier = mappingFunction.apply(value); if (nameIdentifier != null) { ident2IdCache.put(nameIdentifier, value); } @@ -93,9 +108,9 @@ public NameIdentifier getById(Long value, Function mapping } } - public NameIdentifier getById(Long value) { + public EntityIdentifier getById(Long value) { synchronized (this) { - BiMap map = HashBiMap.create(ident2IdCache.asMap()); + BiMap map = HashBiMap.create(ident2IdCache.asMap()); if (map.containsValue(value)) { return map.inverse().get(value); } @@ -103,13 +118,13 @@ public NameIdentifier getById(Long value) { } } - public void invalidate(NameIdentifier key) { + public void invalidate(EntityIdentifier key) { ident2IdCache.invalidate(key); } - public void invalidateWithPrefix(NameIdentifier nameIdentifier) { + public void invalidateWithPrefix(EntityIdentifier nameIdentifier) { ident2IdCache.asMap().keySet().stream() - .filter(k -> k.toString().startsWith(nameIdentifier.toString())) + .filter(k -> k.ident.toString().startsWith(nameIdentifier.ident.toString())) .forEach(ident2IdCache::invalidate); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java index 6b97ac4e0bd..5ddc1633a75 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/RoleMetaService.java @@ -35,6 +35,7 @@ import org.apache.gravitino.storage.relational.mapper.UserRoleRelMapper; import org.apache.gravitino.storage.relational.po.RolePO; import org.apache.gravitino.storage.relational.po.SecurableObjectPO; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; @@ -86,13 +87,16 @@ public Long getRoleIdByMetalakeIdAndName(Long metalakeId, String roleName) { public Long getRoleIdByNameIdentifier(NameIdentifier identifier) { AuthorizationUtils.checkRole(identifier); + EntityIdentifier entityIdentifier = EntityIdentifier.of(identifier, Entity.EntityType.ROLE); + return NameIdMappingService.getInstance() .get( - identifier, + entityIdentifier, ident -> { Long metalakeId = - MetalakeMetaService.getInstance().getMetalakeIdByName(ident.namespace().level(0)); - return getRoleIdByMetalakeIdAndName(metalakeId, ident.name()); + MetalakeMetaService.getInstance() + .getMetalakeIdByName(ident.ident.namespace().level(0)); + return getRoleIdByMetalakeIdAndName(metalakeId, ident.ident.name()); }); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java index 1a056a223f1..10f22f8a5ed 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/SchemaMetaService.java @@ -41,6 +41,7 @@ import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper; import org.apache.gravitino.storage.relational.po.SchemaPO; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; @@ -94,15 +95,17 @@ private Long getSchemaIdByCatalogIdAndName(Long catalogId, String schemaName) { } public Long getSchemaIdByNameIdentifier(NameIdentifier identifier) { + EntityIdentifier entityIdentifier = EntityIdentifier.of(identifier, Entity.EntityType.SCHEMA); return NameIdMappingService.getInstance() .get( - identifier, - ident -> { - NameIdentifierUtil.checkSchema(ident); - String schemaName = ident.name(); + entityIdentifier, + entityIdent -> { + NameIdentifierUtil.checkSchema(entityIdent.ident); + String schemaName = entityIdent.ident.name(); Long catalogId = - CommonMetaService.getInstance().getParentEntityIdByNamespace(ident.namespace()); + CommonMetaService.getInstance() + .getParentEntityIdByNamespace(entityIdent.ident.namespace()); return getSchemaIdByCatalogIdAndName(catalogId, schemaName); }); @@ -199,8 +202,10 @@ public boolean deleteSchema(NameIdentifier identifier, boolean cascade) { String schemaName = identifier.name(); Long schemaId = getSchemaIdByNameIdentifier(identifier); + // Invalidate it in the cache. - NameIdMappingService.getInstance().invalidate(identifier); + NameIdMappingService.getInstance() + .invalidate(EntityIdentifier.of(identifier, Entity.EntityType.SCHEMA)); if (schemaId != null) { if (cascade) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java index 198a1d1cbfe..956ae359376 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableMetaService.java @@ -34,6 +34,7 @@ import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.mapper.TableMetaMapper; import org.apache.gravitino.storage.relational.po.TablePO; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; @@ -90,16 +91,17 @@ private Long getTableIdBySchemaIdAndName(Long schemaId, String tableName) { public Long getTableByNameIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkTable(identifier); + EntityIdentifier tableIdentifier = EntityIdentifier.of(identifier, Entity.EntityType.TABLE); return NameIdMappingService.getInstance() .get( - identifier, + tableIdentifier, ident -> { Long schemaId = CommonMetaService.getInstance() - .getParentEntityIdByNamespace(identifier.namespace()); + .getParentEntityIdByNamespace(ident.ident.namespace()); - return getTableIdBySchemaIdAndName(schemaId, identifier.name()); + return getTableIdBySchemaIdAndName(schemaId, ident.ident.name()); }); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TopicMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TopicMetaService.java index d61edeaefb7..a138435c73b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TopicMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TopicMetaService.java @@ -34,6 +34,7 @@ import org.apache.gravitino.storage.relational.mapper.OwnerMetaMapper; import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper; import org.apache.gravitino.storage.relational.po.TopicPO; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; @@ -217,13 +218,15 @@ public int deleteTopicMetasByLegacyTimeline(Long legacyTimeline, int limit) { public Long getTopicIdByNameIdentifier(NameIdentifier identifier) { NameIdentifierUtil.checkTopic(identifier); + EntityIdentifier topicEntity = EntityIdentifier.of(identifier, Entity.EntityType.TOPIC); return NameIdMappingService.getInstance() .get( - identifier, + topicEntity, ident -> { Long schemaId = - CommonMetaService.getInstance().getParentEntityIdByNamespace(ident.namespace()); - return getTopicIdBySchemaIdAndName(schemaId, ident.name()); + CommonMetaService.getInstance() + .getParentEntityIdByNamespace(ident.ident.namespace()); + return getTopicIdBySchemaIdAndName(schemaId, ident.ident.name()); }); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java index dcfacde7db3..d3119b76847 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/UserMetaService.java @@ -39,6 +39,7 @@ import org.apache.gravitino.storage.relational.po.RolePO; import org.apache.gravitino.storage.relational.po.UserPO; import org.apache.gravitino.storage.relational.po.UserRoleRelPO; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.apache.gravitino.storage.relational.utils.ExceptionUtils; import org.apache.gravitino.storage.relational.utils.POConverters; import org.apache.gravitino.storage.relational.utils.SessionUtils; @@ -86,13 +87,15 @@ public Long getUserIdByMetalakeIdAndName(Long metalakeId, String userName) { public Long getUserIdByNameIdentifier(NameIdentifier identifier) { AuthorizationUtils.checkUser(identifier); + EntityIdentifier userIdIdentifier = EntityIdentifier.of(identifier, Entity.EntityType.USER); return NameIdMappingService.getInstance() .get( - identifier, + userIdIdentifier, ident -> { Long metalakeId = - MetalakeMetaService.getInstance().getMetalakeIdByName(ident.namespace().level(0)); - return getUserIdByMetalakeIdAndName(metalakeId, ident.name()); + MetalakeMetaService.getInstance() + .getMetalakeIdByName(ident.ident.namespace().level(0)); + return getUserIdByMetalakeIdAndName(metalakeId, ident.ident.name()); }); } diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java index 36476e7cf3e..ddb74807c2a 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java @@ -19,7 +19,9 @@ package org.apache.gravitino.storage.relational.service; import java.io.IOException; +import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.storage.relational.service.NameIdMappingService.EntityIdentifier; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -28,19 +30,24 @@ public class TestIdNameMappingService { @Test public void testGetInstance() throws IOException { NameIdMappingService instance = NameIdMappingService.getInstance(); - Assertions.assertNull(instance.get(NameIdentifier.of("m1"))); - Assertions.assertEquals(1L, instance.get(NameIdentifier.of("m1"), (NameIdentifier key) -> 1L)); - instance.put(NameIdentifier.of("m2"), 2L); - Assertions.assertEquals(2L, instance.get(NameIdentifier.of("m2"))); + EntityIdentifier makeLakeIdent1 = + EntityIdentifier.of(NameIdentifier.of("m1"), EntityType.METALAKE); + EntityIdentifier makeLakeIdent2 = + EntityIdentifier.of(NameIdentifier.of("m2"), EntityType.METALAKE); - instance.invalidate(NameIdentifier.of("m2")); - Assertions.assertNull(instance.get(NameIdentifier.of("m2"))); + instance.put(makeLakeIdent1, 1L); + Assertions.assertEquals(1L, instance.get(makeLakeIdent1, (EntityIdentifier key) -> 1L)); - Assertions.assertEquals(NameIdentifier.of("m1"), instance.getById(1L)); + instance.put(makeLakeIdent2, 2L); + Assertions.assertEquals(2L, instance.get(makeLakeIdent2)); - Assertions.assertEquals( - NameIdentifier.of("m2"), instance.getById(2L, (Long value) -> NameIdentifier.of("m2"))); + instance.invalidate(makeLakeIdent2); + Assertions.assertNull(instance.get(makeLakeIdent2)); + + Assertions.assertEquals(makeLakeIdent1, instance.getById(1L)); + + Assertions.assertEquals(makeLakeIdent2, instance.getById(2L, (Long value) -> makeLakeIdent2)); instance.close(); } From 141969a9754ed446813baa22f3a8e00df9de14e1 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 7 Aug 2024 14:45:25 +0800 Subject: [PATCH 12/14] fix --- .../service/NameIdMappingService.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java index 71c137e724d..1fabfaab499 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java @@ -21,6 +21,7 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Scheduler; +import com.google.common.base.Objects; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -67,6 +68,23 @@ private EntityIdentifier(NameIdentifier ident, EntityType type) { public static EntityIdentifier of(NameIdentifier ident, EntityType type) { return new EntityIdentifier(ident, type); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof EntityIdentifier)) { + return false; + } + EntityIdentifier that = (EntityIdentifier) o; + return Objects.equal(ident, that.ident) && type == that.type; + } + + @Override + public int hashCode() { + return Objects.hashCode(ident, type); + } } public static NameIdMappingService getInstance() { From 5de00b74f886951a5f10a41139cfa41a217d9168 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 8 Aug 2024 14:49:29 +0800 Subject: [PATCH 13/14] Fix --- .../service/NameIdMappingService.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java index 1fabfaab499..b54630ab5b3 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/NameIdMappingService.java @@ -37,6 +37,11 @@ public class NameIdMappingService implements Closeable { private static volatile NameIdMappingService instance; + /** + * Cache to store the mapping between the entity identifier and the entity id. + * + *

Note: both the key and value are unique, so we can use BiMap here and get the key by value. + */ private Cache ident2IdCache; private NameIdMappingService() { @@ -111,6 +116,13 @@ public Long get(EntityIdentifier key) { return ident2IdCache.getIfPresent(key); } + /** + * Get the entity identifier by the entity id. + * + * @param value the entity id + * @param mappingFunction the function to get the entity identifier by the entity id + * @return the entity identifier + */ public EntityIdentifier getById(Long value, Function mappingFunction) { synchronized (this) { BiMap map = HashBiMap.create(ident2IdCache.asMap()); @@ -126,6 +138,12 @@ public EntityIdentifier getById(Long value, Function map } } + /** + * Get the entity identifier by the entity id. + * + * @param value the entity id + * @return the entity identifier + */ public EntityIdentifier getById(Long value) { synchronized (this) { BiMap map = HashBiMap.create(ident2IdCache.asMap()); From cdc5821778788100acfb37d5e7dc12b8122c344c Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 14 Aug 2024 19:34:29 +0800 Subject: [PATCH 14/14] Fix --- .../storage/relational/JDBCBackend.java | 12 +---------- .../service/TestIdNameMappingService.java | 20 +++++++++---------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java index 65eaabe2124..b162fee48ce 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/JDBCBackend.java @@ -126,37 +126,27 @@ public boolean exists(NameIdentifier ident, Entity.EntityType entityType) throws @Override public void insert(E e, boolean overwritten) throws EntityAlreadyExistsException, IOException { - EntityType entityType; + EntityType entityType = e.type(); if (e instanceof BaseMetalake) { MetalakeMetaService.getInstance().insertMetalake((BaseMetalake) e, overwritten); - entityType = EntityType.METALAKE; } else if (e instanceof CatalogEntity) { CatalogMetaService.getInstance().insertCatalog((CatalogEntity) e, overwritten); - entityType = EntityType.CATALOG; } else if (e instanceof SchemaEntity) { SchemaMetaService.getInstance().insertSchema((SchemaEntity) e, overwritten); - entityType = EntityType.SCHEMA; } else if (e instanceof TableEntity) { TableMetaService.getInstance().insertTable((TableEntity) e, overwritten); - entityType = EntityType.TABLE; } else if (e instanceof FilesetEntity) { FilesetMetaService.getInstance().insertFileset((FilesetEntity) e, overwritten); - entityType = EntityType.FILESET; } else if (e instanceof TopicEntity) { TopicMetaService.getInstance().insertTopic((TopicEntity) e, overwritten); - entityType = EntityType.TOPIC; } else if (e instanceof UserEntity) { UserMetaService.getInstance().insertUser((UserEntity) e, overwritten); - entityType = EntityType.USER; } else if (e instanceof RoleEntity) { RoleMetaService.getInstance().insertRole((RoleEntity) e, overwritten); - entityType = EntityType.ROLE; } else if (e instanceof GroupEntity) { GroupMetaService.getInstance().insertGroup((GroupEntity) e, overwritten); - entityType = EntityType.GROUP; } else if (e instanceof TagEntity) { TagMetaService.getInstance().insertTag((TagEntity) e, overwritten); - entityType = EntityType.TAG; } else { throw new UnsupportedEntityTypeException( "Unsupported entity type: %s for insert operation", e.getClass()); diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java index ddb74807c2a..64becc9b8f3 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestIdNameMappingService.java @@ -31,23 +31,23 @@ public class TestIdNameMappingService { public void testGetInstance() throws IOException { NameIdMappingService instance = NameIdMappingService.getInstance(); - EntityIdentifier makeLakeIdent1 = + EntityIdentifier metalakeIdent1 = EntityIdentifier.of(NameIdentifier.of("m1"), EntityType.METALAKE); - EntityIdentifier makeLakeIdent2 = + EntityIdentifier metalakeIdent2 = EntityIdentifier.of(NameIdentifier.of("m2"), EntityType.METALAKE); - instance.put(makeLakeIdent1, 1L); - Assertions.assertEquals(1L, instance.get(makeLakeIdent1, (EntityIdentifier key) -> 1L)); + instance.put(metalakeIdent1, 1L); + Assertions.assertEquals(1L, instance.get(metalakeIdent1, (EntityIdentifier key) -> 1L)); - instance.put(makeLakeIdent2, 2L); - Assertions.assertEquals(2L, instance.get(makeLakeIdent2)); + instance.put(metalakeIdent2, 2L); + Assertions.assertEquals(2L, instance.get(metalakeIdent2)); - instance.invalidate(makeLakeIdent2); - Assertions.assertNull(instance.get(makeLakeIdent2)); + instance.invalidate(metalakeIdent2); + Assertions.assertNull(instance.get(metalakeIdent2)); - Assertions.assertEquals(makeLakeIdent1, instance.getById(1L)); + Assertions.assertEquals(metalakeIdent1, instance.getById(1L)); - Assertions.assertEquals(makeLakeIdent2, instance.getById(2L, (Long value) -> makeLakeIdent2)); + Assertions.assertEquals(metalakeIdent2, instance.getById(2L, (Long value) -> metalakeIdent2)); instance.close(); }