Skip to content

Commit c855655

Browse files
suvaidkhansuvaidkhan
authored
Merge branch 'main' into suvaidkhan/improvement/removeAllTagsFromEntity_cli
2 parents d2e1ab7 + 5d6589c commit c855655

File tree

54 files changed

+2438
-155
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+2438
-155
lines changed

‎api/src/main/java/org/apache/gravitino/credential/OSSTokenCredential.java

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.gravitino.credential;
21+
22+
import com.google.common.base.Preconditions;
23+
import com.google.common.collect.ImmutableMap;
24+
import java.util.Map;
25+
import org.apache.commons.lang3.StringUtils;
26+
27+
/** OSS token credential. */
28+
public class OSSTokenCredential implements Credential {
29+
30+
/** OSS token credential type. */
31+
public static final String OSS_TOKEN_CREDENTIAL_TYPE = "oss-token";
32+
/** OSS access key ID used to access OSS data. */
33+
public static final String GRAVITINO_OSS_SESSION_ACCESS_KEY_ID = "oss-access-key-id";
34+
/** OSS secret access key used to access OSS data. */
35+
public static final String GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY = "oss-secret-access-key";
36+
/** OSS security token. */
37+
public static final String GRAVITINO_OSS_TOKEN = "oss-security-token";
38+
39+
private final String accessKeyId;
40+
private final String secretAccessKey;
41+
private final String securityToken;
42+
private final long expireTimeInMS;
43+
44+
/**
45+
* Constructs an instance of {@link OSSTokenCredential} with secret key and token.
46+
*
47+
* @param accessKeyId The oss access key ID.
48+
* @param secretAccessKey The oss secret access key.
49+
* @param securityToken The oss security token.
50+
* @param expireTimeInMS The oss token expire time in ms.
51+
*/
52+
public OSSTokenCredential(
53+
String accessKeyId, String secretAccessKey, String securityToken, long expireTimeInMS) {
54+
Preconditions.checkArgument(
55+
StringUtils.isNotBlank(accessKeyId), "OSS access key Id should not be empty");
56+
Preconditions.checkArgument(
57+
StringUtils.isNotBlank(secretAccessKey), "OSS access key secret should not be empty");
58+
Preconditions.checkArgument(
59+
StringUtils.isNotBlank(securityToken), "OSS security token should not be empty");
60+
61+
this.accessKeyId = accessKeyId;
62+
this.secretAccessKey = secretAccessKey;
63+
this.securityToken = securityToken;
64+
this.expireTimeInMS = expireTimeInMS;
65+
}
66+
67+
@Override
68+
public String credentialType() {
69+
return OSS_TOKEN_CREDENTIAL_TYPE;
70+
}
71+
72+
@Override
73+
public long expireTimeInMs() {
74+
return expireTimeInMS;
75+
}
76+
77+
@Override
78+
public Map<String, String> credentialInfo() {
79+
return (new ImmutableMap.Builder<String, String>())
80+
.put(GRAVITINO_OSS_SESSION_ACCESS_KEY_ID, accessKeyId)
81+
.put(GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY, secretAccessKey)
82+
.put(GRAVITINO_OSS_TOKEN, securityToken)
83+
.build();
84+
}
85+
86+
/**
87+
* Get oss access key ID.
88+
*
89+
* @return The oss access key ID.
90+
*/
91+
public String accessKeyId() {
92+
return accessKeyId;
93+
}
94+
95+
/**
96+
* Get oss secret access key.
97+
*
98+
* @return The oss secret access key.
99+
*/
100+
public String secretAccessKey() {
101+
return secretAccessKey;
102+
}
103+
104+
/**
105+
* Get oss security token.
106+
*
107+
* @return The oss security token.
108+
*/
109+
public String securityToken() {
110+
return securityToken;
111+
}
112+
}

‎bundles/aliyun-bundle/build.gradle.kts

+10
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,26 @@ plugins {
2525
}
2626

2727
dependencies {
28+
compileOnly(project(":api"))
29+
compileOnly(project(":core"))
30+
compileOnly(project(":catalogs:catalog-common"))
2831
compileOnly(project(":catalogs:catalog-hadoop"))
2932
compileOnly(libs.hadoop3.common)
33+
34+
implementation(libs.aliyun.credentials.sdk)
3035
implementation(libs.hadoop3.oss)
3136

37+
// Aliyun oss SDK depends on this package, and JDK >= 9 requires manual add
38+
// https://www.alibabacloud.com/help/en/oss/developer-reference/java-installation?spm=a2c63.p38356.0.i1
39+
implementation(libs.sun.activation)
40+
3241
// oss needs StringUtils from commons-lang3 or the following error will occur in 3.3.0
3342
// java.lang.NoClassDefFoundError: org/apache/commons/lang3/StringUtils
3443
// org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111)
3544
// org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323)
3645
// org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611)
3746
implementation(libs.commons.lang3)
47+
3848
implementation(project(":catalogs:catalog-common")) {
3949
exclude("*")
4050
}

‎bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java

+259
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.gravitino.oss.credential;
20+
21+
import com.aliyun.credentials.Client;
22+
import com.aliyun.credentials.models.Config;
23+
import com.aliyun.credentials.models.CredentialModel;
24+
import com.aliyun.credentials.utils.AuthConstant;
25+
import com.fasterxml.jackson.core.JsonProcessingException;
26+
import com.fasterxml.jackson.databind.ObjectMapper;
27+
import java.io.IOException;
28+
import java.net.URI;
29+
import java.util.HashMap;
30+
import java.util.Map;
31+
import java.util.Objects;
32+
import java.util.Set;
33+
import java.util.stream.Stream;
34+
import javax.annotation.Nullable;
35+
import org.apache.commons.lang3.StringUtils;
36+
import org.apache.gravitino.credential.Credential;
37+
import org.apache.gravitino.credential.CredentialContext;
38+
import org.apache.gravitino.credential.CredentialProvider;
39+
import org.apache.gravitino.credential.OSSTokenCredential;
40+
import org.apache.gravitino.credential.PathBasedCredentialContext;
41+
import org.apache.gravitino.credential.config.OSSCredentialConfig;
42+
import org.apache.gravitino.oss.credential.policy.Condition;
43+
import org.apache.gravitino.oss.credential.policy.Effect;
44+
import org.apache.gravitino.oss.credential.policy.Policy;
45+
import org.apache.gravitino.oss.credential.policy.Statement;
46+
import org.apache.gravitino.oss.credential.policy.StringLike;
47+
48+
/** Generates OSS token to access OSS data. */
49+
public class OSSTokenProvider implements CredentialProvider {
50+
private final ObjectMapper objectMapper = new ObjectMapper();
51+
private String accessKeyId;
52+
private String secretAccessKey;
53+
private String roleArn;
54+
private String externalID;
55+
private int tokenExpireSecs;
56+
private String region;
57+
58+
/**
59+
* Initializes the credential provider with catalog properties.
60+
*
61+
* @param properties catalog properties that can be used to configure the provider. The specific
62+
* properties required vary by implementation.
63+
*/
64+
@Override
65+
public void initialize(Map<String, String> properties) {
66+
OSSCredentialConfig credentialConfig = new OSSCredentialConfig(properties);
67+
this.roleArn = credentialConfig.ossRoleArn();
68+
this.externalID = credentialConfig.externalID();
69+
this.tokenExpireSecs = credentialConfig.tokenExpireInSecs();
70+
this.accessKeyId = credentialConfig.accessKeyID();
71+
this.secretAccessKey = credentialConfig.secretAccessKey();
72+
this.region = credentialConfig.region();
73+
}
74+
75+
/**
76+
* Returns the type of credential, it should be identical in Gravitino.
77+
*
78+
* @return A string identifying the type of credentials.
79+
*/
80+
@Override
81+
public String credentialType() {
82+
return OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE;
83+
}
84+
85+
/**
86+
* Obtains a credential based on the provided context information.
87+
*
88+
* @param context A context object providing necessary information for retrieving credentials.
89+
* @return A Credential object containing the authentication information needed to access a system
90+
* or resource. Null will be returned if no credential is available.
91+
*/
92+
@Nullable
93+
@Override
94+
public Credential getCredential(CredentialContext context) {
95+
if (!(context instanceof PathBasedCredentialContext)) {
96+
return null;
97+
}
98+
PathBasedCredentialContext pathBasedCredentialContext = (PathBasedCredentialContext) context;
99+
CredentialModel credentialModel =
100+
createOSSCredentialModel(
101+
roleArn,
102+
pathBasedCredentialContext.getReadPaths(),
103+
pathBasedCredentialContext.getWritePaths(),
104+
pathBasedCredentialContext.getUserName());
105+
return new OSSTokenCredential(
106+
credentialModel.accessKeyId,
107+
credentialModel.accessKeySecret,
108+
credentialModel.securityToken,
109+
credentialModel.expiration);
110+
}
111+
112+
private CredentialModel createOSSCredentialModel(
113+
String roleArn, Set<String> readLocations, Set<String> writeLocations, String userName) {
114+
Config config = new Config();
115+
config.setAccessKeyId(accessKeyId);
116+
config.setAccessKeySecret(secretAccessKey);
117+
config.setType(AuthConstant.RAM_ROLE_ARN);
118+
config.setRoleArn(roleArn);
119+
config.setRoleSessionName(getRoleName(userName));
120+
if (StringUtils.isNotBlank(externalID)) {
121+
config.setExternalId(externalID);
122+
}
123+
config.setRoleSessionExpiration(tokenExpireSecs);
124+
config.setPolicy(createPolicy(readLocations, writeLocations));
125+
// Local object and client is a simple proxy that does not require manual release
126+
Client client = new Client(config);
127+
return client.getCredential();
128+
}
129+
130+
// reference:
131+
// https://www.alibabacloud.com/help/en/oss/user-guide/tutorial-use-ram-policies-to-control-access-to-oss?spm=a2c63.p38356.help-menu-31815.d_2_4_5_1.5536471b56XPRQ
132+
private String createPolicy(Set<String> readLocations, Set<String> writeLocations) {
133+
Policy.Builder policyBuilder = Policy.builder().version("1");
134+
135+
// Allow read and write access to the specified locations
136+
Statement.Builder allowGetObjectStatementBuilder =
137+
Statement.builder()
138+
.effect(Effect.ALLOW)
139+
.addAction("oss:GetObject")
140+
.addAction("oss:GetObjectVersion");
141+
// Add support for bucket-level policies
142+
Map<String, Statement.Builder> bucketListStatementBuilder = new HashMap<>();
143+
Map<String, Statement.Builder> bucketGetLocationStatementBuilder = new HashMap<>();
144+
145+
String arnPrefix = getArnPrefix();
146+
Stream.concat(readLocations.stream(), writeLocations.stream())
147+
.distinct()
148+
.forEach(
149+
location -> {
150+
URI uri = URI.create(location);
151+
allowGetObjectStatementBuilder.addResource(getOssUriWithArn(arnPrefix, uri));
152+
String bucketArn = arnPrefix + getBucketName(uri);
153+
// ListBucket
154+
bucketListStatementBuilder.computeIfAbsent(
155+
bucketArn,
156+
key ->
157+
Statement.builder()
158+
.effect(Effect.ALLOW)
159+
.addAction("oss:ListBucket")
160+
.addResource(key)
161+
.condition(getCondition(uri)));
162+
// GetBucketLocation
163+
bucketGetLocationStatementBuilder.computeIfAbsent(
164+
bucketArn,
165+
key ->
166+
Statement.builder()
167+
.effect(Effect.ALLOW)
168+
.addAction("oss:GetBucketLocation")
169+
.addResource(key));
170+
});
171+
172+
if (!writeLocations.isEmpty()) {
173+
Statement.Builder allowPutObjectStatementBuilder =
174+
Statement.builder()
175+
.effect(Effect.ALLOW)
176+
.addAction("oss:PutObject")
177+
.addAction("oss:DeleteObject");
178+
writeLocations.forEach(
179+
location -> {
180+
URI uri = URI.create(location);
181+
allowPutObjectStatementBuilder.addResource(getOssUriWithArn(arnPrefix, uri));
182+
});
183+
policyBuilder.addStatement(allowPutObjectStatementBuilder.build());
184+
}
185+
186+
if (!bucketListStatementBuilder.isEmpty()) {
187+
bucketListStatementBuilder
188+
.values()
189+
.forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build()));
190+
} else {
191+
// add list privilege with 0 resources
192+
policyBuilder.addStatement(
193+
Statement.builder().effect(Effect.ALLOW).addAction("oss:ListBucket").build());
194+
}
195+
bucketGetLocationStatementBuilder
196+
.values()
197+
.forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build()));
198+
199+
policyBuilder.addStatement(allowGetObjectStatementBuilder.build());
200+
try {
201+
return objectMapper.writeValueAsString(policyBuilder.build());
202+
} catch (JsonProcessingException e) {
203+
throw new RuntimeException(e);
204+
}
205+
}
206+
207+
private Condition getCondition(URI uri) {
208+
return Condition.builder()
209+
.stringLike(
210+
StringLike.builder()
211+
.addPrefix(concatPathWithSep(trimLeadingSlash(uri.getPath()), "*", "/"))
212+
.build())
213+
.build();
214+
}
215+
216+
private String getArnPrefix() {
217+
if (StringUtils.isNotEmpty(region)) {
218+
return "acs:oss:" + region + ":*:";
219+
}
220+
return "acs:oss:*:*:";
221+
}
222+
223+
private String getBucketName(URI uri) {
224+
return uri.getHost();
225+
}
226+
227+
private String getOssUriWithArn(String arnPrefix, URI uri) {
228+
return arnPrefix + concatPathWithSep(removeSchemaFromOSSUri(uri), "*", "/");
229+
}
230+
231+
private static String concatPathWithSep(String leftPath, String rightPath, String fileSep) {
232+
if (leftPath.endsWith(fileSep) && rightPath.startsWith(fileSep)) {
233+
return leftPath + rightPath.substring(1);
234+
} else if (!leftPath.endsWith(fileSep) && !rightPath.startsWith(fileSep)) {
235+
return leftPath + fileSep + rightPath;
236+
} else {
237+
return leftPath + rightPath;
238+
}
239+
}
240+
241+
// Transform 'oss://bucket/path' to /bucket/path
242+
private String removeSchemaFromOSSUri(URI uri) {
243+
String bucket = uri.getHost();
244+
String path = trimLeadingSlash(uri.getPath());
245+
return String.join(
246+
"/", Stream.of(bucket, path).filter(Objects::nonNull).toArray(String[]::new));
247+
}
248+
249+
private String trimLeadingSlash(String path) {
250+
return path.startsWith("/") ? path.substring(1) : path;
251+
}
252+
253+
private String getRoleName(String userName) {
254+
return "gravitino_" + userName;
255+
}
256+
257+
@Override
258+
public void close() throws IOException {}
259+
}

0 commit comments

Comments
 (0)