From 4fe042bd27c3453e77c28e0c55ffa90bf6262108 Mon Sep 17 00:00:00 2001 From: Mario Inga Date: Wed, 9 Dec 2020 16:13:06 -0500 Subject: [PATCH 01/11] feat(automation) jenkins and docker scripts were improved --- ci/cpu/upload.sh | 6 ++-- ci/gpu/build.sh | 4 +-- conda-build-docker.sh | 77 ++++++++++++++++++----------------------- conda_build.jenkinsfile | 50 ++++++++++---------------- 4 files changed, 58 insertions(+), 79 deletions(-) diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index f0220502d..815adec01 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -53,6 +53,8 @@ if [ "$UPLOAD_BLAZING" == "1" ]; then echo "Upload BlazingSQL to ${CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} - echo "Upload BlazingSQL to ${RAPIDS_CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" - anaconda -t ${RAPIDS_CONDA_KEY} upload -u ${RAPIDS_CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} + if [ ! -z "$RAPIDS_CONDA_USERNAME" ]; then + echo "Upload BlazingSQL to ${RAPIDS_CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" + anaconda -t ${RAPIDS_CONDA_KEY} upload -u ${RAPIDS_CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} + fi fi diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 2ff6fefda..d88158159 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -42,8 +42,8 @@ gpuci_logger "Check GPU usage" nvidia-smi gpuci_logger "Activate conda env" -conda create python=$PYTHON_VER -y -n bsql -source activate bsql +. /opt/conda/etc/profile.d/conda.sh +conda activate rapids conda config --set ssl_verify False gpuci_logger "Installing BlazingSQL dev environment" diff --git a/conda-build-docker.sh b/conda-build-docker.sh index 86b775c42..9d67530f4 100755 --- a/conda-build-docker.sh +++ b/conda-build-docker.sh @@ -21,10 +21,10 @@ # ================================================================ # NOTE Examples: # Run GPUCI jobs (first the gpu-build and then the cpu-build): -# ./conda-build-docker.sh cudf_version cuda_version python_version conda_token custom_label conda_username +# ./conda-build-docker.sh cudf_version cuda_version python_version conda_token conda_username custom_label # # Run only the CPU BUILD job (use this one if you want to debug issues with conda build on gpuci) -# BLAZING_GPUCI_JOB=cpu-build ./conda-build-docker.sh cudf_version cuda_version python_version conda_token custom_label conda_username +# BLAZING_GPUCI_JOB=cpu-build ./conda-build-docker.sh cudf_version cuda_version python_version conda_token conda_username custom_label # # Run only the GPU BUILD job (use this one if you want to debug issues with tests on gpuci/gpu build) # BLAZING_GPUCI_JOB=gpu-build ./conda-build-docker.sh cudf_version cuda_version python_version @@ -36,12 +36,12 @@ # ./conda-build-docker.sh # ================================================================ # NOTE Defaults: -# cudf_version=0.16 -# cuda_version=10.0 +# cudf_version=0.17 +# cuda_version=10.2 # python_version=3.7 # conda_token="" -# custom_label="" # conda_username="blazingsql-nightly" +# custom_label="" # ================================================================ # NOTE Remarks: # - In case a job fails then you will go automatically into the docker for debugging @@ -126,13 +126,13 @@ if [ -z $BLAZING_GPUCI_OS ]; then echo "BLAZING_GPUCI_OS: $BLAZING_GPUCI_OS" fi -CUDF_VERSION="0.16" +CUDF_VERSION="0.17" if [ ! -z $1 ]; then CUDF_VERSION=$1 fi echo "CUDF_VERSION: $CUDF_VERSION" -CUDA_VERSION="10.0" +CUDA_VERSION="10.2" if [ ! -z $2 ]; then CUDA_VERSION=$2 fi @@ -144,6 +144,10 @@ if [ ! -z $3 ]; then fi echo "PYTHON_VERSION: $PYTHON_VERSION" +#USER=$(id -u):$(id -g) +USER="0:0" +echo "USER: $USER" + if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then MY_UPLOAD_KEY="" UPLOAD_BLAZING="0" @@ -151,20 +155,20 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then MY_UPLOAD_KEY=$4 UPLOAD_BLAZING=1 fi - echo "MY_UPLOAD_KEY: $MY_UPLOAD_KEY" + echo "MY_UPLOAD_KEY: ${MY_UPLOAD_KEY:(-4)}" echo "UPLOAD_BLAZING: $UPLOAD_BLAZING" - CUSTOM_LABEL="" + CONDA_USERNAME="blazingsql-nightly" if [ ! -z $5 ]; then - CUSTOM_LABEL=$5 + CONDA_USERNAME=$5 fi - echo "CUSTOM_LABEL: $CUSTOM_LABEL" + echo "CONDA_USERNAME: $CONDA_USERNAME" - CONDA_USERNAME="blazingsql-nightly" + CUSTOM_LABEL="" if [ ! -z $6 ]; then - CONDA_USERNAME=$6 + CUSTOM_LABEL=$6 fi - echo "CONDA_USERNAME: $CONDA_USERNAME" + echo "CUSTOM_LABEL: $CUSTOM_LABEL" fi if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "gpu-build" ]; then @@ -180,27 +184,20 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "gpu-build" ]; then echo "docker pull $gpu_build_img" docker pull $gpu_build_img - gpu_container="blazingsql-gpuci-gpu-build-container" + gpu_container="blazingsql-gpuci-gpu-build-"$RANDOM logger "Running the docker container for the GPU BUILD job ..." - echo "docker run --name $gpu_container --rm -dti \ + GPU_DOCKER="docker run --name $gpu_container --rm -dti \ --runtime=nvidia \ - -u $(id -u):$(id -g) \ + -u $USER \ -e CUDA_VER=${CUDA_VERSION} -e PYTHON_VER=$PYTHON_VERSION \ -e WORKSPACE=$WORKSPACE \ -v /etc/passwd:/etc/passwd \ -v ${WORKSPACE}:${WORKSPACE} -w ${WORKSPACE} \ $gpu_build_img \ bash" - docker run --name $gpu_container --rm -dti \ - --runtime=nvidia \ - -u $(id -u):$(id -g) \ - -e CUDA_VER=${CUDA_VERSION} -e PYTHON_VER=$PYTHON_VERSION \ - -e WORKSPACE=$WORKSPACE \ - -v /etc/passwd:/etc/passwd \ - -v ${WORKSPACE}:${WORKSPACE} -w ${WORKSPACE} \ - $gpu_build_img \ - bash + echo "GPU_DOCKER: "$GPU_DOCKER + eval $GPU_DOCKER logger "Running the GPU BUILD job ..." echo "docker exec -ti $gpu_container $gpu_build_cmd" @@ -211,8 +208,9 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "gpu-build" ]; then echo "docker exec -ti $gpu_container bash" docker exec -ti $gpu_container bash #docker stop $gpu_container - #docker rm $gpu_container #docker ps -a + else + docker rm $gpu_container fi fi @@ -222,6 +220,7 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then ./build.sh clean ./build.sh clean thirdparty + #cpu_build_cmd="bash" cpu_build_cmd="./ci/cpu/build.sh" cpu_build_img=gpuci/rapidsai-driver:$CUDF_VERSION-cuda${CUDA_VERSION}-devel-$BLAZING_GPUCI_OS-py$PYTHON_VERSION @@ -229,31 +228,22 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then echo "docker pull $cpu_build_img" docker pull $cpu_build_img - cpu_container="blazingsql-gpuci-cpu-build-container" + cpu_container="blazingsql-gpuci-cpu-build-"$RANDOM logger "Running the docker container for the CPU BUILD job ..." - echo "docker run --name $cpu_container --rm -dti \ - -u $(id -u):$(id -g) \ + CPU_DOCKER="docker run --name $cpu_container --rm -dti \ + -u $USER \ -e CUDA_VER=${CUDA_VERSION} -e PYTHON_VER=$PYTHON_VERSION \ -e CONDA_USERNAME=$CONDA_USERNAME -e MY_UPLOAD_KEY=$MY_UPLOAD_KEY \ -e UPLOAD_BLAZING=$UPLOAD_BLAZING -e CUSTOM_LABEL=$CUSTOM_LABEL \ - -e GIT_BRANCH="main" -e SOURCE_BRANCH="main" \ + -e BUILD_MODE=branch -e UPLOAD_BLAZING=1 \ -e WORKSPACE=$WORKSPACE \ -v /etc/passwd:/etc/passwd \ -v ${WORKSPACE}:${WORKSPACE} -w ${WORKSPACE} \ $cpu_build_img \ bash" - docker run --name $cpu_container --rm -dti \ - -u $(id -u):$(id -g) \ - -e CUDA_VER=${CUDA_VERSION} -e PYTHON_VER=$PYTHON_VERSION \ - -e CONDA_USERNAME=$CONDA_USERNAME -e MY_UPLOAD_KEY=$MY_UPLOAD_KEY \ - -e UPLOAD_BLAZING=$UPLOAD_BLAZING -e CUSTOM_LABEL=$CUSTOM_LABEL \ - -e GIT_BRANCH="main" -e SOURCE_BRANCH="main" \ - -e WORKSPACE=$WORKSPACE \ - -v /etc/passwd:/etc/passwd \ - -v ${WORKSPACE}:${WORKSPACE} -w ${WORKSPACE} \ - $cpu_build_img \ - bash + echo "CPU_DOCKER: "$CPU_DOCKER + eval $CPU_DOCKER logger "Running the CPU BUILD job ..." echo "docker exec -ti $cpu_container $cpu_build_cmd" @@ -264,8 +254,9 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then echo "docker exec -ti $cpu_container bash" docker exec -ti $cpu_container bash #docker stop $cpu_container - #docker rm $cpu_container #docker ps -a + else + docker rm $cpu_container fi fi diff --git a/conda_build.jenkinsfile b/conda_build.jenkinsfile index a334608fb..bcdc7d216 100644 --- a/conda_build.jenkinsfile +++ b/conda_build.jenkinsfile @@ -8,14 +8,12 @@ pipeline { } parameters { - string(name: 'BRANCH_BUILD', defaultValue: 'develop', description: 'Branch from build') - choice(name: 'CUDA_VERSION', choices: '10.0\n9.2', description: 'Cuda version') - choice(name: 'PYTHON_VERSION', choices: '3.7\n3.6', description: 'Python version') - string(name: 'RAPIDS_BUILD_VERSION', defaultValue: '0.11', description: 'Version of Rapids dependencies, example: 0.10') - string(name: 'CONDA_BUILD', defaultValue: 'blazingsql-nightly,rapidsai-nightly', description: 'Channels from build, example: blazingsql-nightly,rapidsai-nightly') - string(name: 'CONDA_UPLOAD', defaultValue: 'blazingsql-nightly', description: 'The Anaconda repository could be: blazingsql, blazingsql-nightly, mario21ic, etc') + string(name: 'RAPIDS_BUILD_VERSION', defaultValue: '0.17', description: 'Version of Rapids, example: 0.18') + choice(name: 'CUDA_VERSION', choices: '10.1\10.2\n11.0', description: 'Cuda version') + choice(name: 'PYTHON_VERSION', choices: '3.7\n3.8', description: 'Python version') + string(name: 'CONDA_USERNAME', defaultValue: 'blazingsql-nightly', description: 'The Anaconda account, example: blazingsql, blazingsql-nightly, etc') string(name: 'CONDA_TOKEN', defaultValue: 'conda-token-nightly', description: 'Anaconda account token') - string(name: 'SLACK_CHANNEL', defaultValue: 'jenkins-blazingsql', description: 'Slack channel to post notifications') + booleanParam(name: 'TEST', defaultValue: false, description: 'Run GPU tests') } environment { @@ -30,38 +28,26 @@ pipeline { } } - stage("Compile & Publish") { + stage("Test") { + when { + expression { + return params.TEST ==~ /(?i)(Y|YES|T|TRUE|ON|RUN)/ + } + } + steps { + sh "BLAZING_GPUCI_JOB=gpu-build ./conda-build-docker.sh ${params.RAPIDS_BUILD_VERSION} ${params.CUDA_VERSION} ${params.PYTHON_VERSION}" + } + } + + stage("Build && Upload") { steps { withCredentials([string(credentialsId: "${params.CONDA_TOKEN}", variable: 'TOKEN')]) { - sh "./conda-build-docker.sh ${params.CUDA_VERSION} ${params.PYTHON_VERSION} ${params.RAPIDS_BUILD_VERSION} ${params.CONDA_BUILD} ${params.CONDA_UPLOAD} $TOKEN" + sh "BLAZING_GPUCI_JOB=cpu-build ./conda-build-docker.sh ${params.RAPIDS_BUILD_VERSION} ${params.CUDA_VERSION} ${params.PYTHON_VERSION} $TOKEN ${params.CONDA_USERNAME}" } } } } - // Method to post jenkinsfile - post { - always { - echo "Job has finished" - } - success { - slackSendMessage("Success", "good") - } - failure { - slackSendMessage("Failure", "danger") - } - unstable { - slackSendMessage("Warning", "warning") - } - } - } -// Method to send notifications for slack -def slackSendMessage(String status="Started", String color){ - slackSend channel: "${params.SLACK_CHANNEL}", - color: color, - failOnError: true, - message: "${status}" + "${env.SLACK_MESSAGE}" -} From ef20a4ac863b6a2dc5cbca74411e690ec52eed87 Mon Sep 17 00:00:00 2001 From: Mario Inga Date: Wed, 9 Dec 2020 16:41:53 -0500 Subject: [PATCH 02/11] feat(automation) updating changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96ba53e8c..5894e7acb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ - #1224 Added cudaSetDevice to thread initialization so that the cuda context is available to UCX - #1229 Change hardcoded version from setup.py - #1231 Adding docker support for gpuCI scripts +- #1248 Jenkins and Docker scripts were improved for building ## Bug Fixes From f099e66d8776275f47bfee8b076bbe5f45feef58 Mon Sep 17 00:00:00 2001 From: Mario Inga Date: Wed, 9 Dec 2020 17:59:03 -0500 Subject: [PATCH 03/11] feat(automation) fixing gpu test script --- ci/gpu/build.sh | 4 ++-- conda-build-docker.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index d88158159..2ff6fefda 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -42,8 +42,8 @@ gpuci_logger "Check GPU usage" nvidia-smi gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids +conda create python=$PYTHON_VER -y -n bsql +source activate bsql conda config --set ssl_verify False gpuci_logger "Installing BlazingSQL dev environment" diff --git a/conda-build-docker.sh b/conda-build-docker.sh index 9d67530f4..74c381343 100755 --- a/conda-build-docker.sh +++ b/conda-build-docker.sh @@ -235,8 +235,8 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then -u $USER \ -e CUDA_VER=${CUDA_VERSION} -e PYTHON_VER=$PYTHON_VERSION \ -e CONDA_USERNAME=$CONDA_USERNAME -e MY_UPLOAD_KEY=$MY_UPLOAD_KEY \ + -e BUILD_MODE=branch \ -e UPLOAD_BLAZING=$UPLOAD_BLAZING -e CUSTOM_LABEL=$CUSTOM_LABEL \ - -e BUILD_MODE=branch -e UPLOAD_BLAZING=1 \ -e WORKSPACE=$WORKSPACE \ -v /etc/passwd:/etc/passwd \ -v ${WORKSPACE}:${WORKSPACE} -w ${WORKSPACE} \ From 6c08d34a3dfe500e885232f354fdc17f49759dec Mon Sep 17 00:00:00 2001 From: William Malpica Date: Thu, 10 Dec 2020 11:56:18 -0600 Subject: [PATCH 04/11] updated README.md and CHANGELOG and others preparing for 0.17 release --- CHANGELOG.md | 2 +- README.md | 10 +++++----- conda-build-docker.sh | 16 ++++++++-------- docs/conf.py | 4 ++-- powerpc/build.sh | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96ba53e8c..86dd99f0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# BlazingSQL 0.17.0 (Date TBS) +# BlazingSQL 0.17.0 (December 10, 2020) ## New Features diff --git a/README.md b/README.md index d158838f0..429d5fee6 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ conda install -c blazingsql/label/cuda$CUDA_VERSION -c rapidsai -c nvidia -c con Where $CUDA_VERSION is 10.1, 10.2 or 11.0 and $PYTHON_VERSION is 3.7 or 3.8 *For example for CUDA 10.1 and Python 3.7:* ```bash -conda install -c blazingsql/label/cuda10.1 -c rapidsai -c nvidia -c conda-forge -c defaults blazingsql python=3.7 cudatoolkit=10.1 +conda install -c blazingsql -c rapidsai -c nvidia -c conda-forge -c defaults blazingsql python=3.7 cudatoolkit=10.1 ``` ## Nightly Version @@ -118,7 +118,7 @@ This is the recommended way of building all of the BlazingSQL components and dep conda create -n bsql python=$PYTHON_VERSION conda activate bsql conda install --yes -c conda-forge spdlog=1.7.0 google-cloud-cpp=1.16 ninja -conda install --yes -c rapidsai -c nvidia -c conda-forge -c defaults dask-cuda=0.16 dask-cudf=0.16 cudf=0.16 python=3.7 cudatoolkit=$CUDA_VERSION +conda install --yes -c rapidsai -c nvidia -c conda-forge -c defaults dask-cuda=0.17 dask-cudf=0.17 cudf=0.17 ucx-py=0.17 ucx-proc=*=gpu python=3.7 cudatoolkit=$CUDA_VERSION conda install --yes -c conda-forge cmake=3.18 gtest gmock cppzmq cython=0.29 openjdk=8.0 maven jpype1 netifaces pyhive ``` Where $CUDA_VERSION is is 10.1, 10.2 or 11.0 and $PYTHON_VERSION is 3.7 or 3.8 @@ -127,7 +127,7 @@ Where $CUDA_VERSION is is 10.1, 10.2 or 11.0 and $PYTHON_VERSION is 3.7 or 3.8 conda create -n bsql python=3.7 conda activate bsql conda install --yes -c conda-forge spdlog=1.7.0 google-cloud-cpp=1.16 ninja -conda install --yes -c rapidsai -c nvidia -c conda-forge -c defaults dask-cuda=0.16 dask-cudf=0.16 cudf=0.16 python=3.7 cudatoolkit=10.1 +conda install --yes -c rapidsai -c nvidia -c conda-forge -c defaults dask-cuda=0.17 dask-cudf=0.17 cudf=0.17 ucx-py=0.17 ucx-proc=*=gpu python=3.7 cudatoolkit=10.1 conda install --yes -c conda-forge cmake=3.18 gtest gmock cppzmq cython=0.29 openjdk=8.0 maven jpype1 netifaces pyhive ``` @@ -153,7 +153,7 @@ $CONDA_PREFIX now has a folder for the blazingsql repository. conda create -n bsql python=$PYTHON_VERSION conda activate bsql conda install --yes -c conda-forge spdlog=1.7.0 google-cloud-cpp=1.16 ninja -conda install --yes -c rapidsai-nightly -c nvidia -c conda-forge -c defaults dask-cuda=0.17 dask-cudf=0.17 cudf=0.17 ucx-py=0.17 ucx-proc=*=gpu python=3.7 cudatoolkit=$CUDA_VERSION +conda install --yes -c rapidsai-nightly -c nvidia -c conda-forge -c defaults dask-cuda=0.18 dask-cudf=0.18 cudf=0.18 ucx-py=0.18 ucx-proc=*=gpu python=3.7 cudatoolkit=$CUDA_VERSION conda install --yes -c conda-forge cmake=3.18 gtest gmock cppzmq cython=0.29 openjdk=8.0 maven jpype1 netifaces pyhive ``` Where $CUDA_VERSION is is 10.1, 10.2 or 11.0 and $PYTHON_VERSION is 3.7 or 3.8 @@ -162,7 +162,7 @@ Where $CUDA_VERSION is is 10.1, 10.2 or 11.0 and $PYTHON_VERSION is 3.7 or 3.8 conda create -n bsql python=3.7 conda activate bsql conda install --yes -c conda-forge spdlog=1.7.0 google-cloud-cpp=1.16 ninja -conda install --yes -c rapidsai-nightly -c nvidia -c conda-forge -c defaults dask-cuda=0.17 dask-cudf=0.17 cudf=0.17 ucx-py=0.17 ucx-proc=*=gpu python=3.7 cudatoolkit=10.1 +conda install --yes -c rapidsai-nightly -c nvidia -c conda-forge -c defaults dask-cuda=0.18 dask-cudf=0.18 cudf=0.18 ucx-py=0.18 ucx-proc=*=gpu python=3.7 cudatoolkit=10.1 conda install --yes -c conda-forge cmake=3.18 gtest gmock cppzmq cython=0.29 openjdk=8.0 maven jpype1 netifaces pyhive ``` diff --git a/conda-build-docker.sh b/conda-build-docker.sh index 86b775c42..bf70c2257 100755 --- a/conda-build-docker.sh +++ b/conda-build-docker.sh @@ -30,14 +30,14 @@ # BLAZING_GPUCI_JOB=gpu-build ./conda-build-docker.sh cudf_version cuda_version python_version # # Run only the CPU BUILD job and upload the blazingsql package to your conda channel with the label main -# BLAZING_GPUCI_JOB=cpu-build ./conda-build-docker.sh 0.16 10.0 3.7 conda_token main conda_username +# BLAZING_GPUCI_JOB=cpu-build ./conda-build-docker.sh 0.17 10.0 3.7 conda_token main conda_username # # Run GPUCI jobs with defaults: # ./conda-build-docker.sh # ================================================================ # NOTE Defaults: -# cudf_version=0.16 -# cuda_version=10.0 +# cudf_version=0.17 +# cuda_version=10.2 # python_version=3.7 # conda_token="" # custom_label="" @@ -82,14 +82,14 @@ HELP="# ================================================================ # BLAZING_GPUCI_JOB=gpu-build ./conda-build-docker.sh cudf_version cuda_version python_version # # Run only the CPU BUILD job and upload the blazingsql package to your conda channel with the label main -# BLAZING_GPUCI_JOB=cpu-build ./conda-build-docker.sh 0.16 10.0 3.7 conda_token main conda_username +# BLAZING_GPUCI_JOB=cpu-build ./conda-build-docker.sh 0.17 10.0 3.7 conda_token main conda_username # # Run GPUCI jobs with defaults: # ./conda-build-docker.sh # ================================================================ # NOTE Defaults: -# cudf_version=0.16 -# cuda_version=10.0 +# cudf_version=0.17 +# cuda_version=10.1 # python_version=3.7 # conda_token="" # custom_label="" @@ -126,13 +126,13 @@ if [ -z $BLAZING_GPUCI_OS ]; then echo "BLAZING_GPUCI_OS: $BLAZING_GPUCI_OS" fi -CUDF_VERSION="0.16" +CUDF_VERSION="0.17" if [ ! -z $1 ]; then CUDF_VERSION=$1 fi echo "CUDF_VERSION: $CUDF_VERSION" -CUDA_VERSION="10.0" +CUDA_VERSION="10.1" if [ ! -z $2 ]; then CUDA_VERSION=$2 fi diff --git a/docs/conf.py b/docs/conf.py index 5980aeb13..6cc362fc8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ author = 'BlazingDB, Inc.' # The full version, including alpha/beta/rc tags -release = 'v0.16' +release = 'v0.17' # -- General configuration --------------------------------------------------- @@ -83,5 +83,5 @@ html_static_path = ['_static'] -extlinks = {'blazing_repo': ('https://github.com/BlazingDB/blazingsql/blob/branch-0.16/%s', +extlinks = {'blazing_repo': ('https://github.com/BlazingDB/blazingsql/blob/branch-0.17/%s', 'View on github')} diff --git a/powerpc/build.sh b/powerpc/build.sh index 0069d213e..508304b5b 100755 --- a/powerpc/build.sh +++ b/powerpc/build.sh @@ -462,7 +462,7 @@ if [ ! -d cudf ]; then git clone https://github.com/rapidsai/cudf.git --branch "branch-$cudf_version" --single-branch cd cudf # need to pin to a specific commit to keep this build script stable - git checkout 88821fb7fd4b81a98b8efa2f2ab8c7871d02bdef + git checkout v0.17.0 #git submodule update --init --remote --recursive #export CUDA_HOME=/usr/local/cuda/ From 874c94664672f2e57d1d498be18724efce8307d8 Mon Sep 17 00:00:00 2001 From: William Malpica Date: Thu, 10 Dec 2020 11:58:39 -0600 Subject: [PATCH 05/11] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86dd99f0f..293d8389e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - #1216 Add unit test for DAYOFWEEK - #1205 Implement string REVERSE - #1220 Implement string LEFT and RIGHT +- #1250 updated README.md and CHANGELOG and others preparing for 0.17 release ## Improvements From 8f8aafe942a0c0b0da023f0be32008b337133a53 Mon Sep 17 00:00:00 2001 From: Mario Inga Date: Thu, 10 Dec 2020 14:15:25 -0500 Subject: [PATCH 06/11] feat(automation) improving jenkinsfile --- ci/cpu/upload.sh | 4 +++- conda-build-docker.sh | 24 ++++++++++++------------ conda_build.jenkinsfile | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index 815adec01..e377393ee 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -32,7 +32,7 @@ export BLAZINGSQL_FILE=`conda build conda/recipes/blazingsql/ --python=$PYTHON - ################################################################################ # UPLOAD - Conda packages ################################################################################ - +echo "### UPLOAD_BLAZING: $UPLOAD_BLAZING" if [ "$UPLOAD_BLAZING" == "1" ]; then LABEL_OPTION="--label main" if [ ! -z "$CUSTOM_LABEL" ]; then @@ -48,7 +48,9 @@ if [ "$UPLOAD_BLAZING" == "1" ]; then RAPIDS_CONDA_KEY=${RAPIDS_NIGHTLY_UPLOAD_KEY} fi + echo "before test file" test -e ${BLAZINGSQL_FILE} + echo "after test file" echo "Upload BlazingSQL to ${CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} diff --git a/conda-build-docker.sh b/conda-build-docker.sh index 74c381343..8ed75add0 100755 --- a/conda-build-docker.sh +++ b/conda-build-docker.sh @@ -231,7 +231,7 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then cpu_container="blazingsql-gpuci-cpu-build-"$RANDOM logger "Running the docker container for the CPU BUILD job ..." - CPU_DOCKER="docker run --name $cpu_container --rm -dti \ + CPU_DOCKER="docker run --name $cpu_container --rm \ -u $USER \ -e CUDA_VER=${CUDA_VERSION} -e PYTHON_VER=$PYTHON_VERSION \ -e CONDA_USERNAME=$CONDA_USERNAME -e MY_UPLOAD_KEY=$MY_UPLOAD_KEY \ @@ -241,23 +241,23 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then -v /etc/passwd:/etc/passwd \ -v ${WORKSPACE}:${WORKSPACE} -w ${WORKSPACE} \ $cpu_build_img \ - bash" + $cpu_build_cmd" echo "CPU_DOCKER: "$CPU_DOCKER eval $CPU_DOCKER - logger "Running the CPU BUILD job ..." - echo "docker exec -ti $cpu_container $cpu_build_cmd" - docker exec -ti $cpu_container $cpu_build_cmd + #logger "Running the CPU BUILD job ..." + #echo "docker exec -ti $cpu_container $cpu_build_cmd" + #docker exec -ti $cpu_container $cpu_build_cmd - if [ $? != 0 ]; then - logger "Debugging the CPU BUILD job ... " - echo "docker exec -ti $cpu_container bash" - docker exec -ti $cpu_container bash + #if [ $? != 0 ]; then + # logger "Debugging the CPU BUILD job ... " + # echo "docker exec -ti $cpu_container bash" + # docker exec -ti $cpu_container bash #docker stop $cpu_container #docker ps -a - else - docker rm $cpu_container - fi + #else + # docker rm $cpu_container + #fi fi logger "You can run docker exec on the containers, if not needed then just kill them!" diff --git a/conda_build.jenkinsfile b/conda_build.jenkinsfile index bcdc7d216..5eea3ed74 100644 --- a/conda_build.jenkinsfile +++ b/conda_build.jenkinsfile @@ -9,7 +9,7 @@ pipeline { parameters { string(name: 'RAPIDS_BUILD_VERSION', defaultValue: '0.17', description: 'Version of Rapids, example: 0.18') - choice(name: 'CUDA_VERSION', choices: '10.1\10.2\n11.0', description: 'Cuda version') + choice(name: 'CUDA_VERSION', choices: '10.1\n10.2\n11.0', description: 'Cuda version') choice(name: 'PYTHON_VERSION', choices: '3.7\n3.8', description: 'Python version') string(name: 'CONDA_USERNAME', defaultValue: 'blazingsql-nightly', description: 'The Anaconda account, example: blazingsql, blazingsql-nightly, etc') string(name: 'CONDA_TOKEN', defaultValue: 'conda-token-nightly', description: 'Anaconda account token') From 1448c9c7a1ed309063ab100f007cbf05d67facbb Mon Sep 17 00:00:00 2001 From: Mario Inga Date: Thu, 10 Dec 2020 16:34:34 -0500 Subject: [PATCH 07/11] feat(automation) improving docker script --- ci/cpu/build.sh | 1 + conda-build-docker.sh | 35 ++--------------------------------- 2 files changed, 3 insertions(+), 33 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 4db0b2ed1..a27c16e2e 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -32,6 +32,7 @@ fi gpuci_logger "Get env" env echo " - blazingsql-nightly" >> /conda/.condarc +echo " - blazingsql" >> /conda/.condarc gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh diff --git a/conda-build-docker.sh b/conda-build-docker.sh index 8ed75add0..2d0108868 100755 --- a/conda-build-docker.sh +++ b/conda-build-docker.sh @@ -187,7 +187,7 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "gpu-build" ]; then gpu_container="blazingsql-gpuci-gpu-build-"$RANDOM logger "Running the docker container for the GPU BUILD job ..." - GPU_DOCKER="docker run --name $gpu_container --rm -dti \ + GPU_DOCKER="docker run --name $gpu_container --rm \ --runtime=nvidia \ -u $USER \ -e CUDA_VER=${CUDA_VERSION} -e PYTHON_VER=$PYTHON_VERSION \ @@ -195,23 +195,9 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "gpu-build" ]; then -v /etc/passwd:/etc/passwd \ -v ${WORKSPACE}:${WORKSPACE} -w ${WORKSPACE} \ $gpu_build_img \ - bash" + $gpu_build_cmd" echo "GPU_DOCKER: "$GPU_DOCKER eval $GPU_DOCKER - - logger "Running the GPU BUILD job ..." - echo "docker exec -ti $gpu_container $gpu_build_cmd" - docker exec -ti $gpu_container $gpu_build_cmd - - if [ $? != 0 ]; then - logger "Debugging the GPU BUILD job ... " - echo "docker exec -ti $gpu_container bash" - docker exec -ti $gpu_container bash - #docker stop $gpu_container - #docker ps -a - else - docker rm $gpu_container - fi fi if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then @@ -244,22 +230,5 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then $cpu_build_cmd" echo "CPU_DOCKER: "$CPU_DOCKER eval $CPU_DOCKER - - #logger "Running the CPU BUILD job ..." - #echo "docker exec -ti $cpu_container $cpu_build_cmd" - #docker exec -ti $cpu_container $cpu_build_cmd - - #if [ $? != 0 ]; then - # logger "Debugging the CPU BUILD job ... " - # echo "docker exec -ti $cpu_container bash" - # docker exec -ti $cpu_container bash - #docker stop $cpu_container - #docker ps -a - #else - # docker rm $cpu_container - #fi fi -logger "You can run docker exec on the containers, if not needed then just kill them!" -docker ps -a - From 230c9c6fac4909baeaee84b253bdd5d7cae1652d Mon Sep 17 00:00:00 2001 From: Mario Inga Date: Thu, 10 Dec 2020 21:05:56 -0500 Subject: [PATCH 08/11] feat(automation) supporting stable and nightly version --- ci/cpu/build.sh | 8 ++++++-- ci/cpu/upload.sh | 10 ++++------ conda-build-docker.sh | 12 +++++++++--- conda_build.jenkinsfile | 3 ++- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index a27c16e2e..f3443b153 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -31,8 +31,12 @@ fi gpuci_logger "Get env" env -echo " - blazingsql-nightly" >> /conda/.condarc -echo " - blazingsql" >> /conda/.condarc +if [ ! -z "$TYPE" ] && [ "$TYPE" == "stable" ]; then + sed -i "/- rapidsai-nightly/g" /conda/.condarc +else + echo " - blazingsql-nightly" >> /conda/.condarc + echo " - blazingsql" >> /conda/.condarc +fi gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index e377393ee..0fa17061f 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -48,15 +48,13 @@ if [ "$UPLOAD_BLAZING" == "1" ]; then RAPIDS_CONDA_KEY=${RAPIDS_NIGHTLY_UPLOAD_KEY} fi - echo "before test file" test -e ${BLAZINGSQL_FILE} - echo "after test file" echo "Upload BlazingSQL to ${CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} - if [ ! -z "$RAPIDS_CONDA_USERNAME" ]; then - echo "Upload BlazingSQL to ${RAPIDS_CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" - anaconda -t ${RAPIDS_CONDA_KEY} upload -u ${RAPIDS_CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} - fi + #if [ ! -z "$RAPIDS_CONDA_USERNAME" ]; then + # echo "Upload BlazingSQL to ${RAPIDS_CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" + # anaconda -t ${RAPIDS_CONDA_KEY} upload -u ${RAPIDS_CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} + #fi fi diff --git a/conda-build-docker.sh b/conda-build-docker.sh index e36b23aba..259ffe67e 100755 --- a/conda-build-docker.sh +++ b/conda-build-docker.sh @@ -164,9 +164,15 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then fi echo "CONDA_USERNAME: $CONDA_USERNAME" - CUSTOM_LABEL="" + TYPE="nightly" if [ ! -z $6 ]; then - CUSTOM_LABEL=$6 + TYPE=$6 + fi + echo "TYPE: $TYPE" + + CUSTOM_LABEL="" + if [ ! -z $7 ]; then + CUSTOM_LABEL=$7 fi echo "CUSTOM_LABEL: $CUSTOM_LABEL" fi @@ -221,7 +227,7 @@ if [ "$BLAZING_GPUCI_JOB" = "" ] || [ "$BLAZING_GPUCI_JOB" = "cpu-build" ]; then -u $USER \ -e CUDA_VER=${CUDA_VERSION} -e PYTHON_VER=$PYTHON_VERSION \ -e CONDA_USERNAME=$CONDA_USERNAME -e MY_UPLOAD_KEY=$MY_UPLOAD_KEY \ - -e BUILD_MODE=branch \ + -e BUILD_MODE=branch -e TYPE=$TYPE \ -e UPLOAD_BLAZING=$UPLOAD_BLAZING -e CUSTOM_LABEL=$CUSTOM_LABEL \ -e WORKSPACE=$WORKSPACE \ -v /etc/passwd:/etc/passwd \ diff --git a/conda_build.jenkinsfile b/conda_build.jenkinsfile index 5eea3ed74..f1c2d423f 100644 --- a/conda_build.jenkinsfile +++ b/conda_build.jenkinsfile @@ -8,6 +8,7 @@ pipeline { } parameters { + choice(name: 'TYPE', choices: 'nightly\nstable', description: 'Nightly or Stable') string(name: 'RAPIDS_BUILD_VERSION', defaultValue: '0.17', description: 'Version of Rapids, example: 0.18') choice(name: 'CUDA_VERSION', choices: '10.1\n10.2\n11.0', description: 'Cuda version') choice(name: 'PYTHON_VERSION', choices: '3.7\n3.8', description: 'Python version') @@ -42,7 +43,7 @@ pipeline { stage("Build && Upload") { steps { withCredentials([string(credentialsId: "${params.CONDA_TOKEN}", variable: 'TOKEN')]) { - sh "BLAZING_GPUCI_JOB=cpu-build ./conda-build-docker.sh ${params.RAPIDS_BUILD_VERSION} ${params.CUDA_VERSION} ${params.PYTHON_VERSION} $TOKEN ${params.CONDA_USERNAME}" + sh "BLAZING_GPUCI_JOB=cpu-build ./conda-build-docker.sh ${params.RAPIDS_BUILD_VERSION} ${params.CUDA_VERSION} ${params.PYTHON_VERSION} $TOKEN ${params.CONDA_USERNAME} ${params.TYPE}" } } } From 4f32a846b50544ceb045600af8fe5e41ae1cf65d Mon Sep 17 00:00:00 2001 From: Mario Inga Date: Fri, 11 Dec 2020 02:26:37 -0500 Subject: [PATCH 09/11] feat(automation) fixing scripts --- ci/cpu/build.sh | 1 - ci/cpu/upload.sh | 8 ++++---- conda_build.jenkinsfile | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index f3443b153..b4db0e305 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -35,7 +35,6 @@ if [ ! -z "$TYPE" ] && [ "$TYPE" == "stable" ]; then sed -i "/- rapidsai-nightly/g" /conda/.condarc else echo " - blazingsql-nightly" >> /conda/.condarc - echo " - blazingsql" >> /conda/.condarc fi gpuci_logger "Activate conda env" diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh index 0fa17061f..643fbf3a7 100755 --- a/ci/cpu/upload.sh +++ b/ci/cpu/upload.sh @@ -53,8 +53,8 @@ if [ "$UPLOAD_BLAZING" == "1" ]; then echo "Upload BlazingSQL to ${CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} - #if [ ! -z "$RAPIDS_CONDA_USERNAME" ]; then - # echo "Upload BlazingSQL to ${RAPIDS_CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" - # anaconda -t ${RAPIDS_CONDA_KEY} upload -u ${RAPIDS_CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} - #fi + if [ ! -z "$RAPIDS_CONDA_USERNAME" ]; then + echo "Upload BlazingSQL to ${RAPIDS_CONDA_USERNAME} channel: ${BLAZINGSQL_FILE}" + anaconda -t ${RAPIDS_CONDA_KEY} upload -u ${RAPIDS_CONDA_USERNAME} ${LABEL_OPTION} --force ${BLAZINGSQL_FILE} + fi fi diff --git a/conda_build.jenkinsfile b/conda_build.jenkinsfile index f1c2d423f..6fc56d6b1 100644 --- a/conda_build.jenkinsfile +++ b/conda_build.jenkinsfile @@ -12,6 +12,7 @@ pipeline { string(name: 'RAPIDS_BUILD_VERSION', defaultValue: '0.17', description: 'Version of Rapids, example: 0.18') choice(name: 'CUDA_VERSION', choices: '10.1\n10.2\n11.0', description: 'Cuda version') choice(name: 'PYTHON_VERSION', choices: '3.7\n3.8', description: 'Python version') + string(name: 'CONDA_USERNAME', defaultValue: 'blazingsql-nightly', description: 'The Anaconda account, example: blazingsql, blazingsql-nightly, etc') string(name: 'CONDA_TOKEN', defaultValue: 'conda-token-nightly', description: 'Anaconda account token') booleanParam(name: 'TEST', defaultValue: false, description: 'Run GPU tests') From 0d12c370069cbc7d28083bc01cb4a23813eee2d6 Mon Sep 17 00:00:00 2001 From: Mario Inga Date: Fri, 11 Dec 2020 02:43:27 -0500 Subject: [PATCH 10/11] feat(automation) fixing scripts --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53ee6389b..d66e19356 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ - #1207 Calcite uses literal as int32 if not explicit CAST was provided - #1212 Fixed issue when building the thirdpart, cmake version set to 3.18.4 - #1225 Fixed issue due to change in gather API +- #1254 Fixing support of nightly and stable on localhost # BlazingSQL 0.16.0 (October 22, 2020) From 6bdc97e145da9bac24275b0942755ba5c44125b7 Mon Sep 17 00:00:00 2001 From: Mario Inga Date: Fri, 11 Dec 2020 14:08:00 -0500 Subject: [PATCH 11/11] feat(automation) fixing gtest build version --- ci/gpu/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 2ff6fefda..d68c4dfb9 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -54,7 +54,7 @@ gpuci_logger "Install Dependencies" gpuci_conda_retry install --yes -c conda-forge spdlog=1.7.0 google-cloud-cpp=1.16 ninja # NOTE cython must be the same of cudf (for 0.11 and 0.12 cython is >=0.29,<0.30) -gpuci_conda_retry install --yes openjdk=8.0 maven cmake=3.18.4 gtest gmock rapidjson cppzmq cython=0.29 jpype1 netifaces pyhive pytest numpy=1.19 +gpuci_conda_retry install --yes openjdk=8.0 maven cmake=3.18.4 gtest==1.10.0=h0efe328_4 gmock rapidjson cppzmq cython=0.29 jpype1 netifaces pyhive pytest numpy=1.19 # install cudf gpuci_logger "Install RAPIDS dependencies"