[HUDI-8616] DataHub meta sync improvements (#12456)

apache · Dec 18, 2024 · 3cb874f · 3cb874f
1 parent 63e014a
commit 3cb874f
Show file tree

Hide file tree

Showing 12 changed files with 891 additions and 129 deletions.
diff --git a/hudi-sync/hudi-datahub-sync/pom.xml b/hudi-sync/hudi-datahub-sync/pom.xml
@@ -34,14 +34,14 @@
   <packaging>jar</packaging>
 
   <properties>
-    <datahub.version>0.8.45</datahub.version>
+    <datahub.version>0.15.0rc20</datahub.version>
     <httpasync.version>4.1.5</httpasync.version>
   </properties>
 
   <dependencies>
     <dependency>
       <groupId>io.acryl</groupId>
-      <artifactId>datahub-client</artifactId>
+      <artifactId>datahub-client-java8</artifactId>
       <version>${datahub.version}</version>
     </dependency>
 
@@ -85,6 +85,14 @@
       <version>${project.version}</version>
     </dependency>
 
+    <!-- Hadoop to extract Properties -->
+    <dependency>
+      <groupId>org.apache.hudi</groupId>
+      <artifactId>hudi-hadoop-mr</artifactId>
+      <version>${project.version}</version>
+      <scope>compile</scope>
+    </dependency>
+
     <!-- Testing -->
     <dependency>
       <groupId>org.apache.hudi</groupId>
@@ -113,6 +121,14 @@
       <artifactId>kryo-shaded</artifactId>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.hudi</groupId>
+      <artifactId>hudi-hive-sync</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+
   </dependencies>
 
   <build>

diff --git a/...i/sync/datahub/DatahubResponseLogger.java → ...i/sync/datahub/DataHubResponseLogger.java b/...i/sync/datahub/DatahubResponseLogger.java → ...i/sync/datahub/DataHubResponseLogger.java
@@ -27,12 +27,12 @@
 /**
  * Handle responses to requests to Datahub Metastore. Just logs them.
  */
-public class DatahubResponseLogger implements Callback {
-  private static final Logger LOG = LoggerFactory.getLogger(DatahubResponseLogger.class);
+public class DataHubResponseLogger implements Callback {
+  private static final Logger LOG = LoggerFactory.getLogger(DataHubResponseLogger.class);
 
   @Override
   public void onCompletion(MetadataWriteResponse response) {
-    LOG.info("Completed Datahub RestEmitter request. "
+    LOG.info("Completed DataHub RestEmitter request. "
             + "Status: " + (response.isSuccess() ? " succeeded" : " failed"));
     if (!response.isSuccess()) {
       LOG.error("Request failed. " + response);