Skip to content

Commit a09420d

Browse files
yuqi1129yuqi1129
authored
[#3537] improvement(hadooop-catalog): Add e2e user authentication tests for Hadoop catalog. (#3552)
### What changes were proposed in this pull request? Add e2e user authentication tests for the Hadoop catalog. ### Why are the changes needed? Fix: #3537 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? Test locally.
1 parent a24b55c commit a09420d

File tree

3 files changed

+246
-7
lines changed

3 files changed

+246
-7
lines changed

‎catalogs/catalog-hadoop/src/main/java/com/datastrato/gravitino/catalog/hadoop/kerberos/KerberosClient.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,10 @@
2323
import org.slf4j.LoggerFactory;
2424

2525
public class KerberosClient {
26+
private static final Logger LOG = LoggerFactory.getLogger(KerberosClient.class);
2627

2728
public static final String GRAVITINO_KEYTAB_FORMAT = "keytabs/gravitino-%s-keytab";
2829

29-
private static final Logger LOG = LoggerFactory.getLogger(KerberosClient.class);
30-
3130
private final ScheduledThreadPoolExecutor checkTgtExecutor;
3231
private final Map<String, String> conf;
3332
private final Configuration hadoopConf;

‎catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java

+216
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
/*
2+
* Copyright 2024 Datastrato Pvt Ltd.
3+
* This software is licensed under the Apache License version 2.
4+
*/
5+
6+
package com.datastrato.gravitino.catalog.hadoop.integration.test;
7+
8+
import static com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig.AUTH_TYPE_KEY;
9+
import static com.datastrato.gravitino.catalog.hadoop.kerberos.AuthenticationConfig.ENABLE_AUTH_KEY;
10+
import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.IMPERSONATION_ENABLE_KEY;
11+
import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.KEY_TAB_URI_KEY;
12+
import static com.datastrato.gravitino.catalog.hadoop.kerberos.KerberosConfig.PRINCIPAL_KEY;
13+
14+
import com.datastrato.gravitino.Catalog;
15+
import com.datastrato.gravitino.NameIdentifier;
16+
import com.datastrato.gravitino.SchemaChange;
17+
import com.datastrato.gravitino.client.GravitinoAdminClient;
18+
import com.datastrato.gravitino.client.GravitinoMetalake;
19+
import com.datastrato.gravitino.client.KerberosTokenProvider;
20+
import com.datastrato.gravitino.file.Fileset;
21+
import com.datastrato.gravitino.integration.test.container.ContainerSuite;
22+
import com.datastrato.gravitino.integration.test.container.HiveContainer;
23+
import com.datastrato.gravitino.integration.test.util.AbstractIT;
24+
import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
25+
import com.google.common.base.Throwables;
26+
import com.google.common.collect.ImmutableMap;
27+
import com.google.common.collect.Maps;
28+
import java.io.File;
29+
import java.io.IOException;
30+
import java.nio.charset.StandardCharsets;
31+
import java.nio.file.Files;
32+
import java.util.Map;
33+
import org.apache.commons.io.FileUtils;
34+
import org.apache.hadoop.security.UserGroupInformation;
35+
import org.junit.jupiter.api.AfterAll;
36+
import org.junit.jupiter.api.Assertions;
37+
import org.junit.jupiter.api.BeforeAll;
38+
import org.junit.jupiter.api.Tag;
39+
import org.junit.jupiter.api.Test;
40+
import org.slf4j.Logger;
41+
import org.slf4j.LoggerFactory;
42+
import sun.security.krb5.KrbException;
43+
44+
@Tag("gravitino-docker-it")
45+
public class HadoopUserAuthenticationIT extends AbstractIT {
46+
private static final Logger LOG = LoggerFactory.getLogger(HadoopUserAuthenticationIT.class);
47+
48+
private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
49+
50+
private static final String SDK_KERBEROS_PRINCIPAL_KEY = "client.kerberos.principal";
51+
private static final String SDK_KERBEROS_KEYTAB_KEY = "client.kerberos.keytab";
52+
53+
private static final String GRAVITINO_CLIENT_PRINCIPAL = "gravitino_client@HADOOPKRB";
54+
private static final String GRAVITINO_CLIENT_KEYTAB = "/gravitino_client.keytab";
55+
56+
private static final String GRAVITINO_SERVER_PRINCIPAL = "HTTP/localhost@HADOOPKRB";
57+
private static final String GRAVITINO_SERVER_KEYTAB = "/gravitino_server.keytab";
58+
59+
private static final String HADOOP_CLIENT_PRINCIPAL = "cli@HADOOPKRB";
60+
private static final String HADOOP_CLIENT_KEYTAB = "/client.keytab";
61+
62+
private static String TMP_DIR;
63+
64+
private static String HDFS_URL;
65+
66+
private static GravitinoAdminClient adminClient;
67+
68+
private static HiveContainer kerberosHiveContainer;
69+
70+
private static final String METALAKE_NAME =
71+
GravitinoITUtils.genRandomName("CatalogHadoop_metalake");
72+
private static final String CATALOG_NAME =
73+
GravitinoITUtils.genRandomName("CatalogHadoop_catalog");
74+
private static final String SCHEMA_NAME = GravitinoITUtils.genRandomName("CatalogHadoop_schema");
75+
76+
@SuppressWarnings("unused")
77+
private static final String TABLE_NAME = "test_table";
78+
79+
@BeforeAll
80+
public static void startIntegrationTest() throws Exception {
81+
containerSuite.startKerberosHiveContainer();
82+
kerberosHiveContainer = containerSuite.getKerberosHiveContainer();
83+
84+
File baseDir = new File(System.getProperty("java.io.tmpdir"));
85+
File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile();
86+
file.deleteOnExit();
87+
TMP_DIR = file.getAbsolutePath();
88+
89+
HDFS_URL = String.format("hdfs://%s:9000", kerberosHiveContainer.getContainerIpAddress());
90+
91+
// Prepare kerberos related-config;
92+
prepareKerberosConfig();
93+
94+
// Config kerberos configuration for Gravitino server
95+
addKerberosConfig();
96+
97+
// Start Gravitino server
98+
AbstractIT.startIntegrationTest();
99+
}
100+
101+
@AfterAll
102+
public static void stop() {
103+
// Reset the UGI
104+
UserGroupInformation.reset();
105+
106+
// Clean up the kerberos configuration
107+
System.clearProperty("java.security.krb5.conf");
108+
System.clearProperty("sun.security.krb5.debug");
109+
}
110+
111+
private static void prepareKerberosConfig() throws IOException, KrbException {
112+
// Keytab of the Gravitino SDK client
113+
kerberosHiveContainer
114+
.getContainer()
115+
.copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
116+
117+
// Keytab of the Gravitino server
118+
kerberosHiveContainer
119+
.getContainer()
120+
.copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
121+
122+
// Keytab of Gravitino server to connector to HDFS
123+
kerberosHiveContainer
124+
.getContainer()
125+
.copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HADOOP_CLIENT_KEYTAB);
126+
127+
String tmpKrb5Path = TMP_DIR + "krb5.conf_tmp";
128+
String krb5Path = TMP_DIR + "krb5.conf";
129+
kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
130+
131+
// Modify the krb5.conf and change the kdc and admin_server to the container IP
132+
String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
133+
String content = FileUtils.readFileToString(new File(tmpKrb5Path), StandardCharsets.UTF_8);
134+
content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88");
135+
content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749");
136+
FileUtils.write(new File(krb5Path), content, StandardCharsets.UTF_8);
137+
138+
LOG.info("Kerberos kdc config:\n{}", content);
139+
System.setProperty("java.security.krb5.conf", krb5Path);
140+
System.setProperty("sun.security.krb5.debug", "true");
141+
}
142+
143+
private static void addKerberosConfig() {
144+
AbstractIT.customConfigs.put("gravitino.authenticator", "kerberos");
145+
AbstractIT.customConfigs.put(
146+
"gravitino.authenticator.kerberos.principal", GRAVITINO_SERVER_PRINCIPAL);
147+
AbstractIT.customConfigs.put(
148+
"gravitino.authenticator.kerberos.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
149+
AbstractIT.customConfigs.put(SDK_KERBEROS_KEYTAB_KEY, TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
150+
AbstractIT.customConfigs.put(SDK_KERBEROS_PRINCIPAL_KEY, GRAVITINO_CLIENT_PRINCIPAL);
151+
}
152+
153+
@Test
154+
public void testUserAuthentication() {
155+
KerberosTokenProvider provider =
156+
KerberosTokenProvider.builder()
157+
.withClientPrincipal(GRAVITINO_CLIENT_PRINCIPAL)
158+
.withKeyTabFile(new File(TMP_DIR + GRAVITINO_CLIENT_KEYTAB))
159+
.build();
160+
adminClient = GravitinoAdminClient.builder(serverUri).withKerberosAuth(provider).build();
161+
162+
GravitinoMetalake[] metalakes = adminClient.listMetalakes();
163+
Assertions.assertEquals(0, metalakes.length);
164+
165+
GravitinoMetalake gravitinoMetalake =
166+
adminClient.createMetalake(METALAKE_NAME, null, ImmutableMap.of());
167+
168+
// Create a catalog
169+
Map<String, String> properties = Maps.newHashMap();
170+
171+
properties.put(ENABLE_AUTH_KEY, "true");
172+
properties.put(AUTH_TYPE_KEY, "kerberos");
173+
properties.put(IMPERSONATION_ENABLE_KEY, "true");
174+
properties.put(KEY_TAB_URI_KEY, TMP_DIR + HADOOP_CLIENT_KEYTAB);
175+
properties.put(PRINCIPAL_KEY, HADOOP_CLIENT_PRINCIPAL);
176+
properties.put("location", HDFS_URL + "/user/hadoop/");
177+
178+
kerberosHiveContainer.executeInContainer("hadoop", "fs", "-mkdir", "/user/hadoop");
179+
180+
Catalog catalog =
181+
gravitinoMetalake.createCatalog(
182+
CATALOG_NAME, Catalog.Type.FILESET, "hadoop", "comment", properties);
183+
184+
// Test create schema
185+
Exception exception =
186+
Assertions.assertThrows(
187+
Exception.class,
188+
() -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of()));
189+
String exceptionMessage = Throwables.getStackTraceAsString(exception);
190+
// Make sure real user is 'gravitino_client'
191+
Assertions.assertTrue(
192+
exceptionMessage.contains("Permission denied: user=gravitino_client, access=WRITE"));
193+
194+
// Now try to give the user the permission to create schema again
195+
kerberosHiveContainer.executeInContainer("hadoop", "fs", "-chmod", "-R", "777", "/user/hadoop");
196+
Assertions.assertDoesNotThrow(
197+
() -> catalog.asSchemas().createSchema(SCHEMA_NAME, "comment", ImmutableMap.of()));
198+
199+
catalog
200+
.asFilesetCatalog()
201+
.createFileset(
202+
NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, SCHEMA_NAME, TABLE_NAME),
203+
"comment",
204+
Fileset.Type.MANAGED,
205+
null,
206+
ImmutableMap.of());
207+
208+
catalog
209+
.asFilesetCatalog()
210+
.dropFileset(NameIdentifier.of(METALAKE_NAME, CATALOG_NAME, SCHEMA_NAME, TABLE_NAME));
211+
212+
catalog.asSchemas().alterSchema(SCHEMA_NAME, SchemaChange.setProperty("k1", "value1"));
213+
214+
catalog.asSchemas().dropSchema(SCHEMA_NAME, true);
215+
}
216+
}

‎catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java

+29-5
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.apache.hadoop.hdfs.MiniDFSCluster;
4343
import org.apache.hadoop.minikdc.MiniKdc;
4444
import org.apache.hadoop.security.UserGroupInformation;
45+
import org.apache.hadoop.security.authentication.util.KerberosName;
4546
import org.junit.jupiter.api.AfterAll;
4647
import org.junit.jupiter.api.Assertions;
4748
import org.junit.jupiter.api.BeforeAll;
@@ -54,11 +55,10 @@ public class HadoopUserImpersonationIT extends AbstractIT {
5455
private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogIT.class);
5556

5657
public static final String metalakeName =
57-
GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake");
58+
GravitinoITUtils.genRandomName("CatalogHadoopIT_metalake");
5859
public static final String catalogName =
59-
GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog");
60-
public static final String SCHEMA_PREFIX = "CatalogFilesetIT_schema";
61-
public static final String schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX);
60+
GravitinoITUtils.genRandomName("CatalogHadoopIT_catalog");
61+
public static final String schemaName = GravitinoITUtils.genRandomName("CatalogFilesetIT_schema");
6262
private static final String provider = "hadoop";
6363
private static GravitinoMetalake metalake;
6464
private static Catalog catalog;
@@ -78,6 +78,22 @@ public class HadoopUserImpersonationIT extends AbstractIT {
7878
private static String hdfsUri;
7979
private static UserGroupInformation clientUGI;
8080

81+
private static void refreshKerberosConfig() {
82+
Class<?> classRef;
83+
try {
84+
if (System.getProperty("java.vendor").contains("IBM")) {
85+
classRef = Class.forName("com.ibm.security.krb5.internal.Config");
86+
} else {
87+
classRef = Class.forName("sun.security.krb5.Config");
88+
}
89+
90+
Method refershMethod = classRef.getMethod("refresh");
91+
refershMethod.invoke(null);
92+
} catch (Exception e) {
93+
throw new RuntimeException(e);
94+
}
95+
}
96+
8197
@BeforeAll
8298
public static void setup() throws Exception {
8399
if (!isEmbedded()) {
@@ -93,7 +109,6 @@ public static void setup() throws Exception {
93109
kdc.start();
94110

95111
String krb5ConfFile = kdc.getKrb5conf().getAbsolutePath();
96-
System.setProperty("java.security.krb5.conf", krb5ConfFile);
97112

98113
// Reload config when krb5 conf is setup
99114
if (SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_1_8)) {
@@ -134,6 +149,13 @@ public static void setup() throws Exception {
134149
conf.set("hadoop.proxyuser.hdfs.users", "*");
135150
conf.set(
136151
"hadoop.security.auth_to_local", "RULE:[2:$1@$0](.*@EXAMPLE.COM)s/.*/hadoop/\nDEFAULT");
152+
153+
System.setProperty("java.security.krb5.conf", krb5ConfFile);
154+
refreshKerberosConfig();
155+
KerberosName.resetDefaultRealm();
156+
157+
LOG.info("Kerberos kdc config:\n{}", KerberosName.getDefaultRealm());
158+
137159
UserGroupInformation.setConfiguration(conf);
138160
UserGroupInformation.loginUserFromKeytab(
139161
SERVER_PRINCIPAL.replaceAll("_HOST", HOSTNAME) + "@" + kdc.getRealm(),
@@ -187,7 +209,9 @@ public static void stop() {
187209
kdcWorkDir.delete();
188210
}
189211

212+
UserGroupInformation.reset();
190213
System.clearProperty("sun.security.krb5.debug");
214+
System.clearProperty("java.security.krb5.conf");
191215
}
192216

193217
@Test

0 commit comments

Comments
 (0)