From 3934addb7e7a2fbac15f180f25f0068a8e001bef Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 20 Sep 2024 12:25:49 -0700 Subject: [PATCH] test c# and python on container Signed-off-by: Yury-Fridlyand --- .../supported-language-versions.json | 2 +- .github/workflows/csharp.yml | 98 ++++++++++-- .github/workflows/java.yml | 141 +++++------------- .github/workflows/nightly.yml | 26 ++-- .github/workflows/python.yml | 3 +- 5 files changed, 137 insertions(+), 133 deletions(-) diff --git a/.github/json_matrices/supported-language-versions.json b/.github/json_matrices/supported-language-versions.json index 75fa12a20b..788634f0f4 100644 --- a/.github/json_matrices/supported-language-versions.json +++ b/.github/json_matrices/supported-language-versions.json @@ -25,7 +25,7 @@ "always-run-versions": ["16.x"] }, { - "language": "csharp", + "language": "dotnet", "versions": ["8.0", "6.0"], "always-run-versions": ["8.0"] }, diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index da5adb13d5..babccba8c5 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -58,8 +58,6 @@ run-name: jobs: get-matrices: runs-on: ubuntu-latest - # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') || github.event_name == 'push' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} @@ -69,7 +67,7 @@ jobs: - id: get-matrices uses: ./.github/workflows/create-test-matrices with: - language-name: csharp + language-name: dotnet # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow run-full-matrix: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} @@ -102,17 +100,14 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} - - name: Format - working-directory: ./csharp - run: dotnet format --verify-no-changes --verbosity diagnostic - - name: Test dotnet ${{ matrix.dotnet }} - working-directory: ./csharp - run: dotnet test --framework net${{ matrix.dotnet }} "-l:html;LogFileName=TestReport.html" --results-directory . -warnaserror + working-directory: csharp +# run: dotnet test --framework net${{ matrix.dotnet }} "-l:html;LogFileName=TestReport.html" --results-directory . -warnaserror + run: dotnet build --framework net${{ matrix.dotnet }} - - uses: ./.github/workflows/test-benchmark - with: - language-flag: -csharp -dotnet-framework net${{ matrix.dotnet }} +# - uses: ./.github/workflows/test-benchmark +# with: +# language-flag: -csharp -dotnet-framework net${{ matrix.dotnet }} - name: Upload test reports if: always() @@ -125,9 +120,72 @@ jobs: benchmarks/results/* utils/clusters/** - # TODO Add amazonlinux + get-containers: + runs-on: ubuntu-latest + # using `needs` instead of `if` to ensure that same condition applied + needs: [get-matrices] + if: ${{ github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' }} + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: dotnet + run-full-matrix: true + containers: true + + test-csharp-container: + runs-on: ${{ matrix.host.RUNNER }} + needs: [get-containers] + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }} + dotnet: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }} + container: + image: ${{ matrix.host.IMAGE }} + options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options + steps: + - name: Install git + run: | + yum update + yum install -y git tar + + - uses: actions/checkout@v4 + with: + submodules: recursive - lint-rust: + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test dotnet ${{ matrix.dotnet }} + working-directory: csharp + run: dotnet test --framework net${{ matrix.dotnet }} "-l:html;LogFileName=TestReport.html" --results-directory . -warnaserror + + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: test-reports-dotnet-${{ matrix.dotnet }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.IMAGE }}-${{ matrix.host.ARCH }} + path: | + csharp/TestReport.html + benchmarks/results/* + utils/clusters/** + + lint: timeout-minutes: 10 runs-on: ubuntu-latest steps: @@ -135,7 +193,17 @@ jobs: with: submodules: recursive - - uses: ./.github/workflows/lint-rust + - name: lint rust + uses: ./.github/workflows/lint-rust with: cargo-toml-folder: csharp/lib github-token: ${{ secrets.GITHUB_TOKEN }} + +# - name: Set up dotnet +# uses: actions/setup-dotnet@v4 +# with: +# dotnet-version: 6.8 + + - name: Format + working-directory: csharp + run: dotnet format --verify-no-changes --verbosity diagnostic \ No newline at end of file diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 8628890088..842b778fb2 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -92,31 +92,31 @@ jobs: distribution: "temurin" java-version: ${{ matrix.java }} -# - name: Install shared software dependencies -# uses: ./.github/workflows/install-shared-dependencies -# with: -# os: ${{ matrix.host.OS }} -# target: ${{ matrix.host.TARGET }} -# github-token: ${{ secrets.GITHUB_TOKEN }} -# engine-version: ${{ matrix.engine.version }} -# -# - name: Install protoc (protobuf) -# uses: arduino/setup-protoc@v3 -# with: -# version: "26.1" -# repo-token: ${{ secrets.GITHUB_TOKEN }} -# -# - name: Build java client -# working-directory: java -# run: ./gradlew --continue build -x javadoc -# -# - name: Ensure no skipped files by linter -# working-directory: java -# run: ./gradlew spotlessDiagnose | grep 'All formatters are well behaved for all files' -# -# - uses: ./.github/workflows/test-benchmark -# with: -# language-flag: -java + - name: Install shared software dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Install protoc (protobuf) + uses: arduino/setup-protoc@v3 + with: + version: "26.1" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build java client + working-directory: java + run: ./gradlew --continue build -x javadoc + + - name: Ensure no skipped files by linter + working-directory: java + run: ./gradlew spotlessDiagnose | grep 'All formatters are well behaved for all files' + + - uses: ./.github/workflows/test-benchmark + with: + language-flag: -java - name: Upload test & spotbugs reports if: always() @@ -189,8 +189,7 @@ jobs: - name: Build java wrapper working-directory: java - #run: ./gradlew --continue build -x javadoc - run: ./gradlew jar + run: ./gradlew --continue build -x javadoc - name: Upload test & spotbugs reports if: always() @@ -203,80 +202,16 @@ jobs: java/integTest/build/reports/** java/client/build/reports/spotbugs/** -# build-amazonlinux-latest: -# if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true')) -# needs: get-matrices -# strategy: -# # Run all jobs -# fail-fast: false -# matrix: -# java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} -# engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} -# -# runs-on: ubuntu-latest -# container: amazonlinux:latest -# timeout-minutes: 35 -# steps: -# - name: Install git -# run: | -# yum -y remove git -# yum -y remove git-* -# yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm -# yum update -# yum install -y git -# git --version -# -# - uses: actions/checkout@v4 -# -# - name: Checkout submodules -# run: | -# git config --global --add safe.directory "$GITHUB_WORKSPACE" -# git submodule update --init --recursive -# -# - name: Install shared software dependencies -# uses: ./.github/workflows/install-shared-dependencies -# with: -# os: "amazon-linux" -# target: "x86_64-unknown-linux-gnu" -# github-token: ${{ secrets.GITHUB_TOKEN }} -# engine-version: ${{ matrix.engine.version }} -# -# - name: Install protoc (protobuf) -# uses: arduino/setup-protoc@v3 -# with: -# version: "26.1" -# repo-token: ${{ secrets.GITHUB_TOKEN }} -# -# - name: Install Java -# run: | -# yum install -y java-${{ matrix.java }}-amazon-corretto-devel.x86_64 -# -# - name: Build java wrapper -# working-directory: java -# run: ./gradlew --continue build -x javadoc -# -# - name: Upload test & spotbugs reports -# if: always() -# continue-on-error: true -# uses: actions/upload-artifact@v4 -# with: -# name: test-reports-${{ matrix.java }}-${{ matrix.engine }}-amazon-linux -# path: | -# java/client/build/reports/** -# java/integTest/build/reports/** -# java/client/build/reports/spotbugs/** -###################### + lint-rust: + timeout-minutes: 15 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive -# lint-rust: -# timeout-minutes: 15 -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# with: -# submodules: recursive -# -# - name: lint java rust -# uses: ./.github/workflows/lint-rust -# with: -# cargo-toml-folder: java -# github-token: ${{ secrets.GITHUB_TOKEN }} + - name: lint java rust + uses: ./.github/workflows/lint-rust + with: + cargo-toml-folder: java + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 7a390ba14e..79f84f52c6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -66,16 +66,18 @@ jobs: name: Run CI for node client secrets: inherit -# run-full-tests-for-csharp: -# # C# deactivated in cron, uncomment condition to activate -# if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.csharp == true) -# uses: ./.github/workflows/csharp.yml -# name: Run CI for csharp client -# secrets: inherit + run-full-tests-for-csharp: + # C# deactivated in cron, uncomment condition to activate + #if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.csharp == true) + if: (github.event_name == 'workflow_dispatch' && inputs.csharp == true) + uses: ./.github/workflows/csharp.yml + name: Run CI for c# client + secrets: inherit -# run-full-tests-for-go: -# # Go deactivated in cron, uncomment condition to activate -# if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.go == true) -# uses: ./.github/workflows/go.yml -# name: Run CI for go client -# secrets: inherit + run-full-tests-for-go: + # Go deactivated in cron, uncomment condition to activate + #if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') || (github.event_name == 'workflow_dispatch' && inputs.go == true) + if: (github.event_name == 'workflow_dispatch' && inputs.go == true) + uses: ./.github/workflows/go.yml + name: Run CI for go client + secrets: inherit diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 7018acb356..d4c60a1fef 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -269,8 +269,7 @@ jobs: - name: Install git run: | yum update - yum install -y git tar -# yum install -y git tar python3 + yum install -y git tar python3 - uses: actions/checkout@v4 with: