Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#6222]feat(tag): Add tag support for model #6228

Merged
merged 1 commit into from
Jan 22, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ public static MetadataObject parent(MetadataObject object) {
case TABLE:
case FILESET:
case TOPIC:
case MODEL:
parentType = MetadataObject.Type.SCHEMA;
break;
case SCHEMA:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,35 @@
*/
package org.apache.gravitino.client;

import com.google.common.collect.Lists;
import java.util.List;
import java.util.Map;
import org.apache.gravitino.Audit;
import org.apache.gravitino.MetadataObject;
import org.apache.gravitino.MetadataObjects;
import org.apache.gravitino.Namespace;
import org.apache.gravitino.authorization.SupportsRoles;
import org.apache.gravitino.dto.model.ModelDTO;
import org.apache.gravitino.exceptions.NoSuchTagException;
import org.apache.gravitino.exceptions.TagAlreadyAssociatedException;
import org.apache.gravitino.model.Model;
import org.apache.gravitino.tag.SupportsTags;
import org.apache.gravitino.tag.Tag;

/** Represents a generic model. */
class GenericModel implements Model {
class GenericModel implements Model, SupportsTags {

private final ModelDTO modelDTO;

GenericModel(ModelDTO modelDTO) {
private final MetadataObjectTagOperations objectTagOperations;

GenericModel(ModelDTO modelDTO, RESTClient restClient, Namespace modelNs) {
this.modelDTO = modelDTO;
List<String> modelFullName =
Lists.newArrayList(modelNs.level(1), modelNs.level(2), modelDTO.name());
MetadataObject modelObject = MetadataObjects.of(modelFullName, MetadataObject.Type.MODEL);
this.objectTagOperations =
new MetadataObjectTagOperations(modelNs.level(0), modelObject, restClient);
}

@Override
Expand Down Expand Up @@ -61,7 +76,7 @@ public int latestVersion() {

@Override
public SupportsTags supportsTags() {
throw new UnsupportedOperationException("Not supported yet.");
return this;
}

@Override
Expand All @@ -86,4 +101,25 @@ public boolean equals(Object o) {
public int hashCode() {
return modelDTO.hashCode();
}

@Override
public String[] listTags() {
return objectTagOperations.listTags();
}

@Override
public Tag[] listTagsInfo() {
return objectTagOperations.listTagsInfo();
}

@Override
public Tag getTag(String name) throws NoSuchTagException {
return objectTagOperations.getTag(name);
}

@Override
public String[] associateTags(String[] tagsToAdd, String[] tagsToRemove)
throws TagAlreadyAssociatedException {
return objectTagOperations.associateTags(tagsToAdd, tagsToRemove);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public Model getModel(NameIdentifier ident) throws NoSuchModelException {
ErrorHandlers.modelErrorHandler());
resp.validate();

return new GenericModel(resp.getModel());
return new GenericModel(resp.getModel(), restClient, modelFullNs);
}

@Override
Expand All @@ -118,7 +118,7 @@ public Model registerModel(NameIdentifier ident, String comment, Map<String, Str
ErrorHandlers.modelErrorHandler());

resp.validate();
return new GenericModel(resp.getModel());
return new GenericModel(resp.getModel(), restClient, modelFullNs);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.apache.gravitino.integration.test.container.HiveContainer;
import org.apache.gravitino.integration.test.util.BaseIT;
import org.apache.gravitino.integration.test.util.GravitinoITUtils;
import org.apache.gravitino.model.Model;
import org.apache.gravitino.rel.Column;
import org.apache.gravitino.rel.Table;
import org.apache.gravitino.rel.types.Types;
Expand All @@ -60,6 +61,10 @@ public class TagIT extends BaseIT {
private static Schema schema;
private static Table table;

private static Catalog modelCatalog;
private static Schema modelSchema;
private static Model model;

private static Column column;

@BeforeAll
Expand Down Expand Up @@ -108,13 +113,41 @@ public void setUp() {
"comment",
Collections.emptyMap());
column = Arrays.stream(table.columns()).filter(c -> c.name().equals("col1")).findFirst().get();

// Create model catalog
String modelCatalogName = GravitinoITUtils.genRandomName("tag_it_model_catalog");
Assertions.assertFalse(metalake.catalogExists(modelCatalogName));
modelCatalog =
metalake.createCatalog(
modelCatalogName, Catalog.Type.MODEL, "comment", Collections.emptyMap());

// Create model schema
String modelSchemaName = GravitinoITUtils.genRandomName("tag_it_model_schema");
Assertions.assertFalse(modelCatalog.asSchemas().schemaExists(modelSchemaName));
modelSchema =
modelCatalog.asSchemas().createSchema(modelSchemaName, "comment", Collections.emptyMap());

// Create model
String modelName = GravitinoITUtils.genRandomName("tag_it_model");
Assertions.assertFalse(
modelCatalog.asModelCatalog().modelExists(NameIdentifier.of(modelSchemaName, modelName)));
model =
modelCatalog
.asModelCatalog()
.registerModel(
NameIdentifier.of(modelSchemaName, modelName), "comment", Collections.emptyMap());
}

@AfterAll
public void tearDown() {
relationalCatalog.asTableCatalog().dropTable(NameIdentifier.of(schema.name(), table.name()));
relationalCatalog.asSchemas().dropSchema(schema.name(), true);
metalake.dropCatalog(relationalCatalog.name(), true);

modelCatalog.asModelCatalog().deleteModel(NameIdentifier.of(modelSchema.name(), model.name()));
modelCatalog.asSchemas().dropSchema(modelSchema.name(), true);
metalake.dropCatalog(modelCatalog.name(), true);

client.dropMetalake(metalakeName, true);

if (client != null) {
Expand Down Expand Up @@ -653,4 +686,84 @@ public void testAssociateAndDeleteTags() {
String[] associatedTags1 = relationalCatalog.supportsTags().listTags();
Assertions.assertArrayEquals(new String[] {tag2.name()}, associatedTags1);
}

@Test
public void testAssociateTagsToModel() {
Tag tag1 =
metalake.createTag(
GravitinoITUtils.genRandomName("tag_it_model_tag1"),
"comment1",
Collections.emptyMap());
Tag tag2 =
metalake.createTag(
GravitinoITUtils.genRandomName("tag_it_model_tag2"),
"comment2",
Collections.emptyMap());
Tag tag3 =
metalake.createTag(
GravitinoITUtils.genRandomName("tag_it_model_tag3"),
"comment3",
Collections.emptyMap());

// Associate tags to catalog
modelCatalog.supportsTags().associateTags(new String[] {tag1.name()}, null);

// Associate tags to schema
modelSchema.supportsTags().associateTags(new String[] {tag2.name()}, null);

// Associate tags to model
model.supportsTags().associateTags(new String[] {tag3.name()}, null);

// Test list associated tags for model
String[] tags1 = model.supportsTags().listTags();
Assertions.assertEquals(3, tags1.length);
Set<String> tagNames = Sets.newHashSet(tags1);
Assertions.assertTrue(tagNames.contains(tag1.name()));
Assertions.assertTrue(tagNames.contains(tag2.name()));
Assertions.assertTrue(tagNames.contains(tag3.name()));

// Test list associated tags with details for model
Tag[] tags2 = model.supportsTags().listTagsInfo();
Assertions.assertEquals(3, tags2.length);

Set<Tag> nonInheritedTags =
Arrays.stream(tags2).filter(tag -> !tag.inherited().get()).collect(Collectors.toSet());
Set<Tag> inheritedTags =
Arrays.stream(tags2).filter(tag -> tag.inherited().get()).collect(Collectors.toSet());

Assertions.assertEquals(1, nonInheritedTags.size());
Assertions.assertEquals(2, inheritedTags.size());
Assertions.assertTrue(nonInheritedTags.contains(tag3));
Assertions.assertTrue(inheritedTags.contains(tag1));
Assertions.assertTrue(inheritedTags.contains(tag2));

// Test get associated tag for model
Tag resultTag1 = model.supportsTags().getTag(tag1.name());
Assertions.assertEquals(tag1, resultTag1);
Assertions.assertTrue(resultTag1.inherited().get());

Tag resultTag2 = model.supportsTags().getTag(tag2.name());
Assertions.assertEquals(tag2, resultTag2);
Assertions.assertTrue(resultTag2.inherited().get());

Tag resultTag3 = model.supportsTags().getTag(tag3.name());
Assertions.assertEquals(tag3, resultTag3);
Assertions.assertFalse(resultTag3.inherited().get());

// Test get objects associated with tag
Assertions.assertEquals(1, tag1.associatedObjects().count());
Assertions.assertEquals(modelCatalog.name(), tag1.associatedObjects().objects()[0].name());
Assertions.assertEquals(
MetadataObject.Type.CATALOG, tag1.associatedObjects().objects()[0].type());

Assertions.assertEquals(1, tag2.associatedObjects().count());
Assertions.assertEquals(modelSchema.name(), tag2.associatedObjects().objects()[0].name());
Assertions.assertEquals(
MetadataObject.Type.SCHEMA, tag2.associatedObjects().objects()[0].type());

Assertions.assertEquals(1, tag3.associatedObjects().count());
Assertions.assertEquals(model.name(), tag3.associatedObjects().objects()[0].name());
Assertions.assertEquals(
MetadataObject.Type.MODEL, tag3.associatedObjects().objects()[0].type());
}
}
8 changes: 4 additions & 4 deletions docs/manage-tags-in-gravitino.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ the future versions.

:::info
1. Metadata objects are objects that are managed in Gravitino, such as `CATALOG`, `SCHEMA`, `TABLE`,
`COLUMN`, `FILESET`, `TOPIC`, `COLUMN`, etc. A metadata object is combined by a `type` and a
`COLUMN`, `FILESET`, `TOPIC`, `COLUMN`, `MODEL`, etc. A metadata object is combined by a `type` and a
comma-separated `name`. For example, a `CATAGLOG` object has a name "catalog1" with type
"CATALOG", a `SCHEMA` object has a name "catalog1.schema1" with type "SCHEMA", a `TABLE`
object has a name "catalog1.schema1.table1" with type "TABLE", a `COLUMN` object has a name
"catalog1.schema1.table1.column1" with type "COLUMN".
2. Currently, `CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC`, and `COLUMN` objects can be tagged.
2. Currently, `CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC`, `MODEL`, and `COLUMN` objects can be tagged.
3. Tags in Gravitino is inheritable, so listing tags of a metadata object will also list the
tags of its parent metadata objects. For example, listing tags of a `Table` will also list
the tags of its parent `Schema` and `Catalog`.
Expand Down Expand Up @@ -204,8 +204,8 @@ client.deleteTag("tag2");

## Tag associations

Gravitino allows you to associate and disassociate tags with metadata objects. Currently, only
`CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC` objects can be tagged.
Gravitino allows you to associate and disassociate tags with metadata objects. Currently,
`CATALOG`, `SCHEMA`, `TABLE`, `FILESET`, `TOPIC`, `MODEL`, and `COLUMN` objects can be tagged.

### Associate and disassociate tags with a metadata object

Expand Down
4 changes: 2 additions & 2 deletions docs/open-api/tags.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -394,8 +394,8 @@ components:
- "TABLE"
- "FILESET"
- "TOPIC"
- "ROLE"
- "METALAKE"
- "MODEL"
- "COLUMN"


requests:
Expand Down
Loading