Skip to content

Commit 77136b6

Browse files
jerryshaojerryshao
authored
[#5601] feat(core): Add model entity and model dispatcher in Gravitino (#5662)
### What changes were proposed in this pull request? 1. Define and add the model namespace, name identifier, entity definition in Gravitino. 2. Add the basic dispatcher framework in Gravitino. ### Why are the changes needed? This is the second PR to support model management in Gravitino. Fix: #5601 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add UTs to verify part of the code.
1 parent 0b059b9 commit 77136b6

18 files changed

+1093
-2
lines changed

‎core/src/main/java/org/apache/gravitino/Entity.java

+5
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ enum EntityType {
6969
GROUP("gr", 8),
7070
ROLE("ro", 9),
7171
TAG("ta", 10),
72+
MODEL("mo", 11),
73+
MODEL_VERSION("mv", 12),
7274

7375
AUDIT("au", 65534);
7476

@@ -109,12 +111,15 @@ public static List<EntityType> getParentEntityTypes(EntityType entityType) {
109111
case TABLE:
110112
case FILESET:
111113
case TOPIC:
114+
case MODEL:
112115
case USER:
113116
case GROUP:
114117
case ROLE:
115118
return ImmutableList.of(METALAKE, CATALOG, SCHEMA);
116119
case COLUMN:
117120
return ImmutableList.of(METALAKE, CATALOG, SCHEMA, TABLE);
121+
case MODEL_VERSION:
122+
return ImmutableList.of(METALAKE, CATALOG, SCHEMA, MODEL);
118123
default:
119124
throw new IllegalArgumentException("Unknown entity type: " + entityType);
120125
}

‎core/src/main/java/org/apache/gravitino/GravitinoEnv.java

+21
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@
3131
import org.apache.gravitino.catalog.FilesetDispatcher;
3232
import org.apache.gravitino.catalog.FilesetNormalizeDispatcher;
3333
import org.apache.gravitino.catalog.FilesetOperationDispatcher;
34+
import org.apache.gravitino.catalog.ModelDispatcher;
35+
import org.apache.gravitino.catalog.ModelNormalizeDispatcher;
36+
import org.apache.gravitino.catalog.ModelOperationDispatcher;
3437
import org.apache.gravitino.catalog.PartitionDispatcher;
3538
import org.apache.gravitino.catalog.PartitionNormalizeDispatcher;
3639
import org.apache.gravitino.catalog.PartitionOperationDispatcher;
@@ -98,6 +101,8 @@ public class GravitinoEnv {
98101

99102
private TopicDispatcher topicDispatcher;
100103

104+
private ModelDispatcher modelDispatcher;
105+
101106
private MetalakeDispatcher metalakeDispatcher;
102107

103108
private AccessControlDispatcher accessControlDispatcher;
@@ -207,6 +212,15 @@ public TableDispatcher tableDispatcher() {
207212
return tableDispatcher;
208213
}
209214

215+
/**
216+
* Get the ModelDispatcher associated with the Gravitino environment.
217+
*
218+
* @return The ModelDispatcher instance.
219+
*/
220+
public ModelDispatcher modelDispatcher() {
221+
return modelDispatcher;
222+
}
223+
210224
/**
211225
* Get the PartitionDispatcher associated with the Gravitino environment.
212226
*
@@ -440,6 +454,13 @@ private void initGravitinoServerComponents() {
440454
new TopicNormalizeDispatcher(topicHookDispatcher, catalogManager);
441455
this.topicDispatcher = new TopicEventDispatcher(eventBus, topicNormalizeDispatcher);
442456

457+
// TODO(jerryshao). Add Hook and event dispatcher support for Model.
458+
ModelOperationDispatcher modelOperationDispatcher =
459+
new ModelOperationDispatcher(catalogManager, entityStore, idGenerator);
460+
ModelNormalizeDispatcher modelNormalizeDispatcher =
461+
new ModelNormalizeDispatcher(modelOperationDispatcher, catalogManager);
462+
this.modelDispatcher = modelNormalizeDispatcher;
463+
443464
// Create and initialize access control related modules
444465
boolean enableAuthorization = config.get(Configs.ENABLE_AUTHORIZATION);
445466
if (enableAuthorization) {

‎core/src/main/java/org/apache/gravitino/catalog/CapabilityHelpers.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,8 @@ public static Namespace applyCaseSensitive(
130130
String catalog = namespace.level(1);
131131
if (identScope == Capability.Scope.TABLE
132132
|| identScope == Capability.Scope.FILESET
133-
|| identScope == Capability.Scope.TOPIC) {
133+
|| identScope == Capability.Scope.TOPIC
134+
|| identScope == Capability.Scope.MODEL) {
134135
String schema = namespace.level(namespace.length() - 1);
135136
schema = applyCaseSensitiveOnName(Capability.Scope.SCHEMA, schema, capabilities);
136137
return Namespace.of(metalake, catalog, schema);

‎core/src/main/java/org/apache/gravitino/catalog/ModelDispatcher.java

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.gravitino.catalog;
20+
21+
import org.apache.gravitino.model.ModelCatalog;
22+
23+
public interface ModelDispatcher extends ModelCatalog {}

‎core/src/main/java/org/apache/gravitino/catalog/ModelNormalizeDispatcher.java

+167
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.gravitino.catalog;
20+
21+
import static org.apache.gravitino.catalog.CapabilityHelpers.applyCapabilities;
22+
import static org.apache.gravitino.catalog.CapabilityHelpers.applyCaseSensitive;
23+
import static org.apache.gravitino.catalog.CapabilityHelpers.getCapability;
24+
25+
import java.util.Map;
26+
import org.apache.commons.lang3.ArrayUtils;
27+
import org.apache.gravitino.NameIdentifier;
28+
import org.apache.gravitino.Namespace;
29+
import org.apache.gravitino.connector.capability.Capability;
30+
import org.apache.gravitino.exceptions.ModelAlreadyExistsException;
31+
import org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException;
32+
import org.apache.gravitino.exceptions.NoSuchModelException;
33+
import org.apache.gravitino.exceptions.NoSuchModelVersionException;
34+
import org.apache.gravitino.exceptions.NoSuchSchemaException;
35+
import org.apache.gravitino.model.Model;
36+
import org.apache.gravitino.model.ModelVersion;
37+
38+
public class ModelNormalizeDispatcher implements ModelDispatcher {
39+
private final CatalogManager catalogManager;
40+
private final ModelDispatcher dispatcher;
41+
42+
public ModelNormalizeDispatcher(ModelDispatcher dispatcher, CatalogManager catalogManager) {
43+
this.dispatcher = dispatcher;
44+
this.catalogManager = catalogManager;
45+
}
46+
47+
@Override
48+
public NameIdentifier[] listModels(Namespace namespace) throws NoSuchSchemaException {
49+
// The constraints of the name spec may be more strict than underlying catalog,
50+
// and for compatibility reasons, we only apply case-sensitive capabilities here.
51+
Namespace caseSensitiveNs = normalizeCaseSensitive(namespace);
52+
NameIdentifier[] identifiers = dispatcher.listModels(caseSensitiveNs);
53+
return normalizeCaseSensitive(identifiers);
54+
}
55+
56+
@Override
57+
public Model getModel(NameIdentifier ident) throws NoSuchModelException {
58+
// The constraints of the name spec may be more strict than underlying catalog,
59+
// and for compatibility reasons, we only apply case-sensitive capabilities here.
60+
return dispatcher.getModel(normalizeCaseSensitive(ident));
61+
}
62+
63+
@Override
64+
public boolean modelExists(NameIdentifier ident) {
65+
// The constraints of the name spec may be more strict than underlying catalog,
66+
// and for compatibility reasons, we only apply case-sensitive capabilities here.
67+
return dispatcher.modelExists(normalizeCaseSensitive(ident));
68+
}
69+
70+
@Override
71+
public Model registerModel(NameIdentifier ident, String comment, Map<String, String> properties)
72+
throws ModelAlreadyExistsException {
73+
return dispatcher.registerModel(normalizeNameIdentifier(ident), comment, properties);
74+
}
75+
76+
@Override
77+
public Model registerModel(
78+
NameIdentifier ident,
79+
String uri,
80+
String[] aliases,
81+
String comment,
82+
Map<String, String> properties)
83+
throws ModelAlreadyExistsException, ModelVersionAliasesAlreadyExistException {
84+
return dispatcher.registerModel(
85+
normalizeNameIdentifier(ident), uri, aliases, comment, properties);
86+
}
87+
88+
@Override
89+
public boolean deleteModel(NameIdentifier ident) {
90+
// The constraints of the name spec may be more strict than underlying catalog,
91+
// and for compatibility reasons, we only apply case-sensitive capabilities here.
92+
return dispatcher.deleteModel(normalizeCaseSensitive(ident));
93+
}
94+
95+
@Override
96+
public int[] listModelVersions(NameIdentifier ident) throws NoSuchModelException {
97+
return dispatcher.listModelVersions(normalizeCaseSensitive(ident));
98+
}
99+
100+
@Override
101+
public ModelVersion getModelVersion(NameIdentifier ident, int version)
102+
throws NoSuchModelVersionException {
103+
return dispatcher.getModelVersion(normalizeCaseSensitive(ident), version);
104+
}
105+
106+
@Override
107+
public ModelVersion getModelVersion(NameIdentifier ident, String alias)
108+
throws NoSuchModelVersionException {
109+
return dispatcher.getModelVersion(normalizeCaseSensitive(ident), alias);
110+
}
111+
112+
@Override
113+
public boolean modelVersionExists(NameIdentifier ident, int version) {
114+
return dispatcher.modelVersionExists(normalizeCaseSensitive(ident), version);
115+
}
116+
117+
@Override
118+
public boolean modelVersionExists(NameIdentifier ident, String alias) {
119+
return dispatcher.modelVersionExists(normalizeCaseSensitive(ident), alias);
120+
}
121+
122+
@Override
123+
public ModelVersion linkModelVersion(
124+
NameIdentifier ident,
125+
String uri,
126+
String[] aliases,
127+
String comment,
128+
Map<String, String> properties)
129+
throws NoSuchModelException, ModelVersionAliasesAlreadyExistException {
130+
return dispatcher.linkModelVersion(
131+
normalizeCaseSensitive(ident), uri, aliases, comment, properties);
132+
}
133+
134+
@Override
135+
public boolean deleteModelVersion(NameIdentifier ident, int version) {
136+
return dispatcher.deleteModelVersion(normalizeCaseSensitive(ident), version);
137+
}
138+
139+
@Override
140+
public boolean deleteModelVersion(NameIdentifier ident, String alias) {
141+
return dispatcher.deleteModelVersion(normalizeCaseSensitive(ident), alias);
142+
}
143+
144+
private Namespace normalizeCaseSensitive(Namespace namespace) {
145+
Capability capabilities = getCapability(NameIdentifier.of(namespace.levels()), catalogManager);
146+
return applyCaseSensitive(namespace, Capability.Scope.MODEL, capabilities);
147+
}
148+
149+
private NameIdentifier normalizeCaseSensitive(NameIdentifier ident) {
150+
Capability capabilities = getCapability(ident, catalogManager);
151+
return applyCaseSensitive(ident, Capability.Scope.MODEL, capabilities);
152+
}
153+
154+
private NameIdentifier[] normalizeCaseSensitive(NameIdentifier[] idents) {
155+
if (ArrayUtils.isEmpty(idents)) {
156+
return idents;
157+
}
158+
159+
Capability capabilities = getCapability(idents[0], catalogManager);
160+
return applyCaseSensitive(idents, Capability.Scope.MODEL, capabilities);
161+
}
162+
163+
private NameIdentifier normalizeNameIdentifier(NameIdentifier ident) {
164+
Capability capability = getCapability(ident, catalogManager);
165+
return applyCapabilities(ident, Capability.Scope.MODEL, capability);
166+
}
167+
}

‎core/src/main/java/org/apache/gravitino/catalog/ModelOperationDispatcher.java

+110
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.gravitino.catalog;
20+
21+
import java.util.Map;
22+
import org.apache.gravitino.EntityStore;
23+
import org.apache.gravitino.NameIdentifier;
24+
import org.apache.gravitino.Namespace;
25+
import org.apache.gravitino.exceptions.ModelAlreadyExistsException;
26+
import org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException;
27+
import org.apache.gravitino.exceptions.NoSuchModelException;
28+
import org.apache.gravitino.exceptions.NoSuchModelVersionException;
29+
import org.apache.gravitino.exceptions.NoSuchSchemaException;
30+
import org.apache.gravitino.model.Model;
31+
import org.apache.gravitino.model.ModelVersion;
32+
import org.apache.gravitino.storage.IdGenerator;
33+
34+
public class ModelOperationDispatcher extends OperationDispatcher implements ModelDispatcher {
35+
36+
public ModelOperationDispatcher(
37+
CatalogManager catalogManager, EntityStore store, IdGenerator idGenerator) {
38+
super(catalogManager, store, idGenerator);
39+
}
40+
41+
@Override
42+
public NameIdentifier[] listModels(Namespace namespace) throws NoSuchSchemaException {
43+
throw new UnsupportedOperationException("Not implemented");
44+
}
45+
46+
@Override
47+
public Model getModel(NameIdentifier ident) throws NoSuchModelException {
48+
throw new UnsupportedOperationException("Not implemented");
49+
}
50+
51+
@Override
52+
public Model registerModel(NameIdentifier ident, String comment, Map<String, String> properties)
53+
throws ModelAlreadyExistsException {
54+
throw new UnsupportedOperationException("Not implemented");
55+
}
56+
57+
@Override
58+
public Model registerModel(
59+
NameIdentifier ident,
60+
String uri,
61+
String[] aliases,
62+
String comment,
63+
Map<String, String> properties)
64+
throws ModelAlreadyExistsException, ModelVersionAliasesAlreadyExistException {
65+
throw new UnsupportedOperationException("Not implemented");
66+
}
67+
68+
@Override
69+
public boolean deleteModel(NameIdentifier ident) {
70+
throw new UnsupportedOperationException("Not implemented");
71+
}
72+
73+
@Override
74+
public int[] listModelVersions(NameIdentifier ident) throws NoSuchModelException {
75+
throw new UnsupportedOperationException("Not implemented");
76+
}
77+
78+
@Override
79+
public ModelVersion getModelVersion(NameIdentifier ident, int version)
80+
throws NoSuchModelVersionException {
81+
throw new UnsupportedOperationException("Not implemented");
82+
}
83+
84+
@Override
85+
public ModelVersion getModelVersion(NameIdentifier ident, String alias)
86+
throws NoSuchModelVersionException {
87+
throw new UnsupportedOperationException("Not implemented");
88+
}
89+
90+
@Override
91+
public ModelVersion linkModelVersion(
92+
NameIdentifier ident,
93+
String uri,
94+
String[] aliases,
95+
String comment,
96+
Map<String, String> properties)
97+
throws NoSuchModelException, ModelVersionAliasesAlreadyExistException {
98+
throw new UnsupportedOperationException("Not implemented");
99+
}
100+
101+
@Override
102+
public boolean deleteModelVersion(NameIdentifier ident, int version) {
103+
throw new UnsupportedOperationException("Not implemented");
104+
}
105+
106+
@Override
107+
public boolean deleteModelVersion(NameIdentifier ident, String alias) {
108+
throw new UnsupportedOperationException("Not implemented");
109+
}
110+
}

‎core/src/main/java/org/apache/gravitino/connector/capability/Capability.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,8 @@ enum Scope {
3939
COLUMN,
4040
FILESET,
4141
TOPIC,
42-
PARTITION
42+
PARTITION,
43+
MODEL
4344
}
4445

4546
/**

0 commit comments

Comments
 (0)