From bf8755b5ef48ba78c2599f33e9ea93fb5329f9ec Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Mon, 3 Feb 2025 11:48:38 -0700 Subject: [PATCH 1/7] Added Upload Artifact in CPP build Signed-off-by: VenkateshJaya --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 699efc636..237c954b6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -86,7 +86,7 @@ jobs: conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@nvks-runners + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@github-artifact-store with: build_type: pull-request conda-cpp-tests: From ce0ac2b2c21bc9e14507d10c767a5ef42c802777 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Mon, 3 Feb 2025 15:06:59 -0700 Subject: [PATCH 2/7] commented out unchanged steps in CI --- .github/workflows/pr.yaml | 150 +++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 237c954b6..3a76aa539 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -86,81 +86,81 @@ jobs: conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@github-artifact-store + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@test-github-artifacts with: build_type: pull-request - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@nvks-runners - with: - build_type: pull-request - conda-python-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" - wheel-build-cpp: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners - with: - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) - build_type: pull-request - script: ci/build_wheel_cpp.sh - wheel-build-python: - needs: wheel-build-cpp - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners - with: - build_type: pull-request - script: ci/build_wheel_python.sh - wheel-tests: - needs: [wheel-build-python, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@nvks-runners - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel.sh - devcontainer: - secrets: inherit - needs: - - telemetry-setup - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@nvks-runners - with: - arch: '["amd64"]' - cuda: '["12.8"]' - build_command: | - sccache -z; - build-all -DBUILD_BENCHMARKS=ON --verbose; - sccache -s; + # conda-cpp-tests: + # needs: [conda-cpp-build, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + # with: + # build_type: pull-request + # conda-python-build: + # needs: conda-cpp-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@nvks-runners + # with: + # build_type: pull-request + # conda-python-tests: + # needs: [conda-python-build, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # with: + # build_type: pull-request + # docs-build: + # needs: conda-python-build + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners + # with: + # build_type: pull-request + # node_type: "gpu-l4-latest-1" + # arch: "amd64" + # container_image: "rapidsai/ci-conda:latest" + # run_script: "ci/build_docs.sh" + # wheel-build-cpp: + # needs: checks + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners + # with: + # matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + # build_type: pull-request + # script: ci/build_wheel_cpp.sh + # wheel-build-python: + # needs: wheel-build-cpp + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@nvks-runners + # with: + # build_type: pull-request + # script: ci/build_wheel_python.sh + # wheel-tests: + # needs: [wheel-build-python, changed-files] + # secrets: inherit + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@nvks-runners + # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + # with: + # build_type: pull-request + # script: ci/test_wheel.sh + # devcontainer: + # secrets: inherit + # needs: + # - telemetry-setup + # uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@nvks-runners + # with: + # arch: '["amd64"]' + # cuda: '["12.8"]' + # build_command: | + # sccache -z; + # build-all -DBUILD_BENCHMARKS=ON --verbose; + # sccache -s; - telemetry-summarize: - # This job must use a self-hosted runner to record telemetry traces. - runs-on: linux-amd64-cpu4 - needs: pr-builder - if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} - continue-on-error: true - steps: - - name: Telemetry summarize - uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main + # telemetry-summarize: + # # This job must use a self-hosted runner to record telemetry traces. + # runs-on: linux-amd64-cpu4 + # needs: pr-builder + # if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} + # continue-on-error: true + # steps: + # - name: Telemetry summarize + # uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main From 199df338fbb0e3f35038695baa7a812b233bb803 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Mon, 3 Feb 2025 15:22:08 -0700 Subject: [PATCH 3/7] commented out test steps from pr-builder --- .github/workflows/pr.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3a76aa539..67f4ff951 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,15 +16,15 @@ jobs: - changed-files - checks - conda-cpp-build - - conda-cpp-tests - - conda-python-build - - conda-python-tests - - docs-build - - wheel-build-cpp - - wheel-build-python - - wheel-tests - - devcontainer - - telemetry-setup + # - conda-cpp-tests + # - conda-python-build + # - conda-python-tests + # - docs-build + # - wheel-build-cpp + # - wheel-build-python + # - wheel-tests + # - devcontainer + # - telemetry-setup secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@nvks-runners if: always() From 6e71cba015d34fa1c76925ac9a520263d7900160 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Mon, 3 Feb 2025 15:25:12 -0700 Subject: [PATCH 4/7] added telemetry setup to pr-builder --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 67f4ff951..9380b6706 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -24,7 +24,7 @@ jobs: # - wheel-build-python # - wheel-tests # - devcontainer - # - telemetry-setup + - telemetry-setup secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@nvks-runners if: always() From 7cc8e2cd65c8330047f515ce72c5d8f2ee62ce6b Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Tue, 4 Feb 2025 17:26:14 -0700 Subject: [PATCH 5/7] added github download for cpp tests --- ci/test_cpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index e0612c94b..505266377 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -23,7 +23,7 @@ set +u conda activate test set -u -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" From 51ab1adcf66799992b6c3978bd89116101d3dd01 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Wed, 5 Feb 2025 17:27:33 -0700 Subject: [PATCH 6/7] added downloads from github --- .github/workflows/pr.yaml | 68 +++++++++++++++++++-------------------- ci/build_docs.sh | 4 +-- ci/build_python.sh | 2 +- ci/test_python.sh | 4 +-- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9380b6706..9ccad9dd0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,10 +16,10 @@ jobs: - changed-files - checks - conda-cpp-build - # - conda-cpp-tests - # - conda-python-build - # - conda-python-tests - # - docs-build + - conda-cpp-tests + - conda-python-build + - conda-python-tests + - docs-build # - wheel-build-cpp # - wheel-build-python # - wheel-tests @@ -89,36 +89,36 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@test-github-artifacts with: build_type: pull-request - # conda-cpp-tests: - # needs: [conda-cpp-build, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@nvks-runners - # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - # with: - # build_type: pull-request - # conda-python-build: - # needs: conda-cpp-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@nvks-runners - # with: - # build_type: pull-request - # conda-python-tests: - # needs: [conda-python-build, changed-files] - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@nvks-runners - # if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - # with: - # build_type: pull-request - # docs-build: - # needs: conda-python-build - # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@nvks-runners - # with: - # build_type: pull-request - # node_type: "gpu-l4-latest-1" - # arch: "amd64" - # container_image: "rapidsai/ci-conda:latest" - # run_script: "ci/build_docs.sh" + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@test-github-artifacts + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@test-github-artifacts + with: + build_type: pull-request + conda-python-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@test-github-artifacts + with: + build_type: pull-request + node_type: "gpu-l4-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" # wheel-build-cpp: # needs: checks # secrets: inherit diff --git a/ci/build_docs.sh b/ci/build_docs.sh index a3562d973..82e62d9db 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -22,8 +22,8 @@ conda activate docs rapids-print-env rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-github python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ diff --git a/ci/build_python.sh b/ci/build_python.sh index 98f57c34b..309f21387 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -16,7 +16,7 @@ rapids-generate-version > ./VERSION rapids-logger "Begin py build" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) sccache --zero-stats diff --git a/ci/test_python.sh b/ci/test_python.sh index 377882c59..53a298192 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -25,8 +25,8 @@ set -u rapids-print-env rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-github python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ From c38201f8bd88b942f90fe66f04d9472a190c4030 Mon Sep 17 00:00:00 2001 From: VenkateshJaya Date: Wed, 5 Feb 2025 17:36:23 -0700 Subject: [PATCH 7/7] fixed error in pr workflow --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9ccad9dd0..d4111f400 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -105,7 +105,7 @@ jobs: conda-python-tests: needs: [conda-python-build, changed-files] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@test-github-artifacts if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python with: build_type: pull-request