Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#6474] improvement(storage): batch listing securable objects in RoleMetaService #6601

Merged
merged 24 commits into from
Mar 11, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
c5d2835
feat: move getFilesetObjectFullNames from RoleMetaService to Metadata…
unknowntpo Mar 4, 2025
b5a7953
feat(RoleMetaService.listSecurableObjects): metalake batch retrival
unknowntpo Mar 4, 2025
e6ff40e
feat: dynamically calling object fullname getter
unknowntpo Mar 4, 2025
2cf855e
feat(listSecurableObjects): implement batch retrieval for Catalog and…
unknowntpo Mar 5, 2025
d14f69c
feat(listSecurableObjects): implement batch retrieval for Model
unknowntpo Mar 5, 2025
82274c5
docs(SecurableObject.ofFileset): improve doc
unknowntpo Mar 5, 2025
2dc0d21
feat(listSecurableObjects): add support for TOPIC batch retrieval
unknowntpo Mar 6, 2025
cc4f523
refactor(listSecurableObjects): remove switch case
unknowntpo Mar 6, 2025
a52a8c5
fix(MetadataObjectService): fix wrong key in getCatalogObjectFullName…
unknowntpo Mar 6, 2025
018b035
feat(listSecurableObjects): add batch retrieval for SCHEMA
unknowntpo Mar 6, 2025
285e8b2
feat(listSecurableObjects): add support for listing column object ful…
unknowntpo Mar 7, 2025
53f3e03
docs: add javadoc
unknowntpo Mar 7, 2025
3bbdf52
refactor: set getXXXIdAndNameMap to private
unknowntpo Mar 7, 2025
24f1f24
refactor: remove MetadataObject.Type.TABLE in listSecurableObjects
unknowntpo Mar 7, 2025
3a7899a
fix(TableColumnBaseSQLProvider): fix wrong sql in listColumnPOsByColu…
unknowntpo Mar 7, 2025
bd41b48
fix: remove column and model
unknowntpo Mar 10, 2025
af46aa3
refactor(MetadataObjectService): remove unnecessary check of schema id
unknowntpo Mar 10, 2025
5129e6b
styles(MetadataObjectService): remove wrong comment
unknowntpo Mar 10, 2025
a93e38d
refactor(MetadataObjectSerivce): rename getXXXObjectFullNames to getX…
unknowntpo Mar 10, 2025
e862ede
refactor(MetadataObjectSerivce): remove unnecessary null check in get…
unknowntpo Mar 11, 2025
d4d6a50
refactor(MetadataObjectSerivce): change parameter name for readability
unknowntpo Mar 11, 2025
2077550
refactor(MetadataObjectSerivce): remove unnecessary null check in get…
unknowntpo Mar 11, 2025
ea393b5
refactor(MetadataObjectSerivce): remove getXXXIdAndNameMap method and…
unknowntpo Mar 11, 2025
14b15f1
refactor: remove unused code about COLUMN metadata object
unknowntpo Mar 11, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ public static SecurableObject ofTopic(
}

/**
* Create the table {@link SecurableObject} with the given securable schema object, fileset name
* Create the fileset {@link SecurableObject} with the given securable schema object, fileset name
* and privileges.
*
* @param schema The schema securable object
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ public interface MetalakeMetaMapper {
@SelectProvider(type = MetalakeMetaSQLProviderFactory.class, method = "selectMetalakeMetaById")
MetalakePO selectMetalakeMetaById(@Param("metalakeId") Long metalakeId);

@SelectProvider(
type = MetalakeMetaSQLProviderFactory.class,
method = "listMetalakePOsByMetalakeIds")
List<MetalakePO> listMetalakePOsByMetalakeIds(@Param("metalakeIds") List<Long> metalakeIds);

@SelectProvider(
type = MetalakeMetaSQLProviderFactory.class,
method = "selectMetalakeIdMetaByName")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.gravitino.storage.relational.mapper;

import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType;
import org.apache.gravitino.storage.relational.mapper.provider.base.MetalakeMetaBaseSQLProvider;
Expand Down Expand Up @@ -68,6 +69,10 @@ public static String selectMetalakeIdMetaByName(@Param("metalakeName") String me
return getProvider().selectMetalakeIdMetaByName(metalakeName);
}

public static String listMetalakePOsByMetalakeIds(@Param("metalakeIds") List<Long> metalakeIds) {
return getProvider().listMetalakePOsByMetalakeIds(metalakeIds);
}

public static String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) {
return getProvider().insertMetalakeMeta(metalakePO);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ public interface ModelMetaMapper {
@SelectProvider(type = ModelMetaSQLProviderFactory.class, method = "listModelPOsBySchemaId")
List<ModelPO> listModelPOsBySchemaId(@Param("schemaId") Long schemaId);

@SelectProvider(type = ModelMetaSQLProviderFactory.class, method = "listModelPOsByModelIds")
List<ModelPO> listModelPOsByModelIds(@Param("modelIds") List<Long> modelIds);

@SelectProvider(
type = ModelMetaSQLProviderFactory.class,
method = "selectModelMetaBySchemaIdAndModelName")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.gravitino.storage.relational.mapper;

import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType;
import org.apache.gravitino.storage.relational.mapper.provider.base.ModelMetaBaseSQLProvider;
Expand Down Expand Up @@ -62,6 +63,10 @@ public static String listModelPOsBySchemaId(@Param("schemaId") Long schemaId) {
return getProvider().listModelPOsBySchemaId(schemaId);
}

public static String listModelPOsByModelIds(@Param("modelIds") List<Long> modelIds) {
return getProvider().listModelPOsByModelIds(modelIds);
}

public static String selectModelMetaBySchemaIdAndModelName(
@Param("schemaId") Long schemaId, @Param("modelName") String modelName) {
return getProvider().selectModelMetaBySchemaIdAndModelName(schemaId, modelName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ public interface TableMetaMapper {
@SelectProvider(type = TableMetaSQLProviderFactory.class, method = "listTablePOsBySchemaId")
List<TablePO> listTablePOsBySchemaId(@Param("schemaId") Long schemaId);

@SelectProvider(type = TableMetaSQLProviderFactory.class, method = "listTablePOsByTableIds")
List<TablePO> listTablePOsByTableIds(@Param("tableIds") List<Long> tableIds);

@SelectProvider(
type = TableMetaSQLProviderFactory.class,
method = "selectTableIdBySchemaIdAndName")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.gravitino.storage.relational.mapper;

import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType;
import org.apache.gravitino.storage.relational.mapper.provider.base.TableMetaBaseSQLProvider;
Expand Down Expand Up @@ -54,6 +55,10 @@ public static String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) {
return getProvider().listTablePOsBySchemaId(schemaId);
}

public static String listTablePOsByTableIds(@Param("tableIds") List<Long> tableIds) {
return getProvider().listTablePOsByTableIds(tableIds);
}

public static String selectTableIdBySchemaIdAndName(
@Param("schemaId") Long schemaId, @Param("tableName") String name) {
return getProvider().selectTableIdBySchemaIdAndName(schemaId, name);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ public interface TopicMetaMapper {
@SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "listTopicPOsBySchemaId")
List<TopicPO> listTopicPOsBySchemaId(@Param("schemaId") Long schemaId);

@SelectProvider(type = TopicMetaSQLProviderFactory.class, method = "listTopicPOsByTopicIds")
List<TopicPO> listTopicPOsByTopicIds(@Param("topicIds") List<Long> topicIds);

@SelectProvider(
type = TopicMetaSQLProviderFactory.class,
method = "selectTopicMetaBySchemaIdAndName")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.apache.gravitino.storage.relational.mapper;

import com.google.common.collect.ImmutableMap;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.storage.relational.JDBCBackend.JDBCBackendType;
import org.apache.gravitino.storage.relational.mapper.provider.base.TopicMetaBaseSQLProvider;
Expand Down Expand Up @@ -63,6 +64,10 @@ public static String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) {
return getProvider().listTopicPOsBySchemaId(schemaId);
}

public static String listTopicPOsByTopicIds(@Param("topicIds") List<Long> topicIds) {
return getProvider().listTopicPOsByTopicIds(topicIds);
}

public static String selectTopicMetaBySchemaIdAndName(
@Param("schemaId") Long schemaId, @Param("topicName") String topicName) {
return getProvider().selectTopicMetaBySchemaIdAndName(schemaId, topicName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import static org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper.TABLE_NAME;

import java.util.List;
import org.apache.gravitino.storage.relational.po.MetalakePO;
import org.apache.ibatis.annotations.Param;

Expand Down Expand Up @@ -65,6 +66,24 @@ public String selectMetalakeIdMetaByName(@Param("metalakeName") String metalakeN
+ " WHERE metalake_name = #{metalakeName} and deleted_at = 0";
}

public String listMetalakePOsByMetalakeIds(@Param("metalakeIds") List<Long> metalakeIds) {
return "<script>"
+ " SELECT metalake_id as metalakeId, metalake_name as metalakeName,"
+ " metalake_comment as metalakeComment, properties,"
+ " audit_info as auditInfo, schema_version as schemaVersion,"
+ " current_version as currentVersion, last_version as lastVersion,"
+ " deleted_at as deletedAt"
+ " FROM "
+ TABLE_NAME
+ " WHERE deleted_at = 0"
+ " AND metalake_id in ("
+ "<foreach collection='metalakeIds' item='metalakeId' separator=','>"
+ "#{metalakeId}"
+ "</foreach>"
+ ") "
+ "</script>";
}

public String insertMetalakeMeta(@Param("metalakeMeta") MetalakePO metalakePO) {
return "INSERT INTO "
+ TABLE_NAME
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
*/
package org.apache.gravitino.storage.relational.mapper.provider.base;

import java.util.List;
import org.apache.gravitino.storage.relational.mapper.ModelMetaMapper;
import org.apache.gravitino.storage.relational.po.ModelPO;
import org.apache.ibatis.annotations.Param;
Expand Down Expand Up @@ -66,6 +67,23 @@ public String listModelPOsBySchemaId(@Param("schemaId") Long schemaId) {
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0";
}

public String listModelPOsByModelIds(List<Long> modelIds) {
return "<script>"
+ " SELECT model_id AS modelId, model_name AS modelName, metalake_id AS metalakeId,"
+ " catalog_id AS catalogId, schema_id AS schemaId, model_comment AS modelComment,"
+ " model_properties AS modelProperties, model_latest_version AS"
+ " modelLatestVersion, audit_info AS auditInfo, deleted_at AS deletedAt"
+ " FROM "
+ ModelMetaMapper.TABLE_NAME
+ " WHERE deleted_at = 0"
+ " AND model_id in ("
+ "<foreach collection='modelIds' item='modelId' separator=','>"
+ "#{modelId}"
+ "</foreach>"
+ ") "
+ "</script>";
}

public String selectModelMetaBySchemaIdAndModelName(
@Param("schemaId") Long schemaId, @Param("modelName") String modelName) {
return "SELECT model_id AS modelId, model_name AS modelName, metalake_id AS metalakeId,"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import static org.apache.gravitino.storage.relational.mapper.TableMetaMapper.TABLE_NAME;

import java.util.List;
import org.apache.gravitino.storage.relational.po.TablePO;
import org.apache.ibatis.annotations.Param;

Expand All @@ -36,6 +37,24 @@ public String listTablePOsBySchemaId(@Param("schemaId") Long schemaId) {
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0";
}

public String listTablePOsByTableIds(List<Long> tableIds) {
return "<script>"
+ " SELECT table_id as tableId, table_name as tableName,"
+ " metalake_id as metalakeId, catalog_id as catalogId,"
+ " schema_id as schemaId, audit_info as auditInfo,"
+ " current_version as currentVersion, last_version as lastVersion,"
+ " deleted_at as deletedAt"
+ " FROM "
+ TABLE_NAME
+ " WHERE deleted_at = 0"
+ " AND table_id in ("
+ "<foreach collection='tableIds' item='tableId' separator=','>"
+ "#{tableId}"
+ "</foreach>"
+ ") "
+ "</script>";
}

public String selectTableIdBySchemaIdAndName(
@Param("schemaId") Long schemaId, @Param("tableName") String name) {
return "SELECT table_id as tableId FROM "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import static org.apache.gravitino.storage.relational.mapper.TopicMetaMapper.TABLE_NAME;

import java.util.List;
import org.apache.gravitino.storage.relational.po.TopicPO;
import org.apache.ibatis.annotations.Param;

Expand Down Expand Up @@ -90,6 +91,24 @@ public String listTopicPOsBySchemaId(@Param("schemaId") Long schemaId) {
+ " WHERE schema_id = #{schemaId} AND deleted_at = 0";
}

public String listTopicPOsByTopicIds(@Param("topicIds") List<Long> topicIds) {
return "<script>"
+ " SELECT topic_id as topicId, topic_name as topicName, metalake_id as metalakeId,"
+ " catalog_id as catalogId, schema_id as schemaId,"
+ " comment as comment, properties as properties, audit_info as auditInfo,"
+ " current_version as currentVersion, last_version as lastVersion,"
+ " deleted_at as deletedAt"
+ " FROM "
+ TABLE_NAME
+ " WHERE deleted_at = 0"
+ " AND topic_id in ("
+ "<foreach collection='topicIds' item='topicId' separator=','>"
+ "#{topicId}"
+ "</foreach>"
+ ") "
+ "</script>";
}

public String selectTopicMetaBySchemaIdAndName(
@Param("schemaId") Long schemaId, @Param("topicName") String topicName) {
return "SELECT topic_id as topicId, topic_name as topicName,"
Expand Down
Loading