Datalake 0.15.0

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,7 +12,7 @@ _If medium or high, explain what verification was done to mitigate the risks._
 
 ### Documentation Update
 
-_Describe any necessary documentation update if there is any new feature, config, or user-facing change_
+_Describe any necessary documentation update if there is any new feature, config, or user-facing change. If not, put "none"._
 
 - _The config description must be updated if new configs are added or the default value of the configs are changed_
 - _Any new feature or user-facing change requires updating the Hudi website. Please create a Jira ticket, attach the

.github/workflows/azure_ci.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+async function checkAzureCiAndCreateCommitStatus({ github, context, prNumber, latestCommitHash }) {
+  console.log(`- Checking Azure CI status of PR: ${prNumber} ${latestCommitHash}`);
+  const botUsername = 'hudi-bot';
+
+  const comments = await github.paginate(github.rest.issues.listComments, {
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: prNumber,
+    sort: 'updated',
+    direction: 'desc',
+    per_page: 100
+  });
+
+  // Find the latest comment from hudi-bot containing the Azure CI report
+  const botComments = comments.filter(comment => comment.user.login === botUsername);
+
+  let status = 'pending';
+  let message = 'In progress';
+  let azureRunLink = '';
+
+  if (botComments.length > 0) {
+    const lastComment = botComments[0];
+    const reportPrefix = `${latestCommitHash} Azure: `
+    const successReportString = `${reportPrefix}[SUCCESS]`
+    const failureReportString = `${reportPrefix}[FAILURE]`
+
+    if (lastComment.body.includes(reportPrefix)) {
+      if (lastComment.body.includes(successReportString)) {
+        message = 'Successful on the latest commit';
+        status = 'success';
+      } else if (lastComment.body.includes(failureReportString)) {
+        message = 'Failed on the latest commit';
+        status = 'failure';
+      }
+    }
+
+    const linkRegex = /\[[a-zA-Z]+\]\((https?:\/\/[^\s]+)\)/;
+    const parts = lastComment.body.split(reportPrefix);
+    const secondPart = parts.length > 1 ? parts[1] : '';
+    const match = secondPart.match(linkRegex);
+
+    if (match) {
+      azureRunLink = match[1];
+    }
+  }
+
+  console.log(`Status: ${status}`);
+  console.log(`Azure Run Link: ${azureRunLink}`);
+  console.log(`${message}`);
+
+  console.log(`- Create commit status of PR based on Azure CI status: ${prNumber} ${latestCommitHash}`);
+  // Create or update the commit status for Azure CI
+  await github.rest.repos.createCommitStatus({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    sha: latestCommitHash,
+    state: status,
+    target_url: azureRunLink,
+    description: message,
+    context: 'Azure CI'
+  });
+
+  return { status, message, azureRunLink };
+}
+
+module.exports = checkAzureCiAndCreateCommitStatus;

.github/workflows/azure_ci_check.yml

@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Azure CI
+
+on:
+  issue_comment:
+    types: [ created, edited, deleted ]
+
+permissions:
+  statuses: write
+  pull-requests: read
+  issues: read
+
+jobs:
+  check-azure-ci-and-add-commit-status:
+    if: |
+      github.event.issue.pull_request != null &&
+      github.event.comment.user.login == 'hudi-bot'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Check PR state
+        id: check_pr_state
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const issueNumber = context.issue.number;
+            const { data: pullRequest } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: issueNumber
+            });
+
+            // Only check open PRs and a PR that is not a HOTFIX
+            const shouldSkip = (pullRequest.body.includes('HOTFIX: SKIP AZURE CI')
+              || pullRequest.state != 'open');
+
+            if (!shouldSkip) {
+              const commitHash = pullRequest.head.sha;
+              console.log(`Latest commit hash: ${commitHash}`);
+              // Set the output variable to be used in subsequent step
+              core.setOutput("latest_commit_hash", commitHash);
+            }
+            console.log(`Should skip Azure CI? ${shouldSkip}`);
+            return shouldSkip;
+
+      - name: Check Azure CI report and create commit status to PR
+        if: steps.check_pr_state.outputs.result != 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const latestCommitHash = '${{ steps.check_pr_state.outputs.latest_commit_hash }}'            
+            const issueNumber = context.issue.number;
+            const checkAzureCiAndCreateCommitStatus = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/azure_ci.js`);
+
+            await checkAzureCiAndCreateCommitStatus({
+              github,
+              context,
+              prNumber: issueNumber,
+              latestCommitHash: latestCommitHash
+            });