diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 31fb940740b..03bd77946c2 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -32,6 +32,11 @@ on: description: 'Publish Docker token' required: true type: string + pat: + description: 'GitHub private access token' + required: true + type: string + publish-latest-tag: description: 'Whether to update the latest tag. This operation is only applicable to official releases and should not be used for Release Candidate (RC).' required: false @@ -132,6 +137,8 @@ jobs: full_tag_name="${{ github.event.inputs.version }}" fi + export PRIVATE_ACCESS_TOKEN=${{ github.event.inputs.pat }} + if [[ "${publish_latest}" == "true" ]]; then echo "Publish tag ${full_tag_name}, and update latest too." ./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${full_tag_name} --latest diff --git a/build.gradle.kts b/build.gradle.kts index c0d27f38eec..a9254fe8681 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -174,7 +174,7 @@ allprojects { param.environment("PROJECT_VERSION", project.version) // Gravitino CI Docker image - param.environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "apache/gravitino-ci:hive-0.1.17") + param.environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "apache/gravitino-ci:hive-0.1.18") param.environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "apache/gravitino-ci:kerberos-hive-0.1.5") param.environment("GRAVITINO_CI_DORIS_DOCKER_IMAGE", "apache/gravitino-ci:doris-0.1.5") param.environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "apache/gravitino-ci:trino-0.1.6") diff --git a/dev/docker/hive/download-release.sh b/dev/docker/hive/download-release.sh new file mode 100755 index 00000000000..bf03d3c35b1 --- /dev/null +++ b/dev/docker/hive/download-release.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The first parameter is the version of the release, e.g. 2.4.0 +# The second parameter is the name of the asset file, e.g. ranger-2.4.0-admin.tar.gz +# The third parameter is the location and name of the output file, e.g. /tmp/ranger-2.4.0-admin.tar.gz + +set -ex + +if [ $# -ne 3 ]; then + echo "Usage: $0 " + exit 1 +fi + + +TOKEN=${PRIVATE_ACCESS_TOKEN} +REPO="datastrato/ranger" +FILE=$2 # the name of your release asset file, e.g. build.tar.gz +VERSION=$1 # tag name or the word "latest" +GITHUB="https://api.github.com" + +alias err_echo='>&2 echo' + +function gh_curl() { + curl -H "Authorization: token $TOKEN" \ + -H "Accept: application/vnd.github.v3.raw" \ + $@ +} + +if [ "$VERSION" = "latest" ]; then + # Github should return the latest release first. + parser=".[0].assets | map(select(.name == \"$FILE\"))[0].id" +else + parser=". | map(select(.tag_name == \"$VERSION\"))[0].assets | map(select(.name == \"$FILE\"))[0].id" +fi; + +asset_id=`gh_curl -s $GITHUB/repos/$REPO/releases | jq "$parser"` +if [ "$asset_id" = "null" ]; then + err_echo "ERROR: version not found $VERSION" + exit 1 +fi; + +echo $2 + +curl -sL --header "Authorization: token $TOKEN" --header 'Accept: application/octet-stream' https://$TOKEN:@api.github.com/repos/$REPO/releases/assets/$asset_id -o $3 diff --git a/dev/docker/hive/hive-dependency.sh b/dev/docker/hive/hive-dependency.sh index e93361c3c9f..ff138c51712 100755 --- a/dev/docker/hive/hive-dependency.sh +++ b/dev/docker/hive/hive-dependency.sh @@ -54,10 +54,10 @@ ZOOKEEPER_PACKAGE_NAME="zookeeper-${ZOOKEEPER_VERSION}.tar.gz" ZOOKEEPER_DOWNLOAD_URL="https://archive.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/${ZOOKEEPER_PACKAGE_NAME}" RANGER_HIVE_PACKAGE_NAME="ranger-${RANGER_VERSION}-hive-plugin.tar.gz" -RANGER_HIVE_DOWNLOAD_URL=https://github.com/datastrato/apache-ranger/releases/download/release-ranger-${RANGER_VERSION}/ranger-${RANGER_VERSION}-hive-plugin.tar.gz +RANGER_HIVE_DOWNLOAD_URL=https://github.com/datastrato/ranger/releases/download/v${RANGER_VERSION}/ranger-${RANGER_VERSION}-hive-plugin.tar.gz RANGER_HDFS_PACKAGE_NAME="ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz" -RANGER_HDFS_DOWNLOAD_URL=https://github.com/datastrato/apache-ranger/releases/download/release-ranger-${RANGER_VERSION}/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz +RANGER_HDFS_DOWNLOAD_URL=https://github.com/datastrato/ranger/releases/download/v${RANGER_VERSION}/ranger-${RANGER_VERSION}-hdfs-plugin.tar.gz # Prepare download packages if [[ ! -d "${hive_dir}/packages" ]]; then @@ -89,11 +89,21 @@ if [ ! -f "${hive_dir}/packages/${ZOOKEEPER_PACKAGE_NAME}" ]; then fi if [ ! -f "${hive_dir}/packages/${RANGER_HDFS_PACKAGE_NAME}" ]; then - curl -L -s -o "${hive_dir}/packages/${RANGER_HDFS_PACKAGE_NAME}" ${RANGER_HDFS_DOWNLOAD_URL} + bash ${hive_dir}/download-release.sh "v${RANGER_VERSION}" ${RANGER_HDFS_PACKAGE_NAME} "${hive_dir}/packages/${RANGER_HDFS_PACKAGE_NAME}" +fi + +if [[ $? -ne 0 ]]; then + echo "Failed to download Ranger HDFS plugin package" + exit 1 fi if [ ! -f "${hive_dir}/packages/${RANGER_HIVE_PACKAGE_NAME}" ]; then - curl -L -s -o "${hive_dir}/packages/${RANGER_HIVE_PACKAGE_NAME}" ${RANGER_HIVE_DOWNLOAD_URL} + bash ${hive_dir}/download-release.sh "v${RANGER_VERSION}" ${RANGER_HIVE_PACKAGE_NAME} "${hive_dir}/packages/${RANGER_HIVE_PACKAGE_NAME}" +fi + +if [[ $? -ne 0 ]]; then + echo "Failed to download Ranger Hive plugin package" + exit 1 fi if [ ! -f "${hive_dir}/packages/${HADOOP2_GCS_PACKAGE_NAME}" ]; then diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md index a137923a694..6540b87365c 100644 --- a/docs/docker-image-details.md +++ b/docs/docker-image-details.md @@ -185,6 +185,12 @@ You can use this kind of image to test the catalog of Apache Hive. Changelog + +- apache/gravitino-ci:hive-0.1.18 + - Support UTF-8 encoding for the `hive-site.xml` file and Hive Metastore. + For more information, please see [PR](https://github.com/apache/gravitino/pull/6625) + - Change ranger-hive-plugin and ranger-hdfs-plugin download URL. + - apache/gravitino-ci:hive-0.1.17 - Add support for JDBC SQL standard authorization - Add JDBC SQL standard authorization related configuration in the `hive-site-for-sql-base-auth.xml` and `hiveserver2-site-for-sql-base-auth.xml`