ci: Add delta to main to bench table
#4282
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: QNS | |
| on: | |
| push: | |
| branches: ["main"] | |
| paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"] | |
| pull_request: | |
| branches: ["main"] | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"] | |
| merge_group: | |
| workflow_dispatch: | |
| schedule: | |
| # Run at 1 AM each day, so there is a `main`-branch baseline in the cache. | |
| - cron: '0 1 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| env: | |
| LATEST: neqo-latest | |
| DELIM: ' vs. ' | |
| TIMEOUT: 20 | |
| jobs: | |
| docker-image: | |
| name: Build Docker image | |
| runs-on: ubuntu-latest | |
| outputs: | |
| imageID: ${{ steps.docker_build_and_push.outputs.imageID }} | |
| permissions: | |
| packages: write | |
| steps: | |
| - uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3.3.0 | |
| - uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0 | |
| with: | |
| cache-binary: ${{ github.event_name == 'pull_request' }} # zizmor: ignore[cache-poisoning] | |
| - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ github.token }} | |
| - uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 | |
| id: meta | |
| with: | |
| images: ghcr.io/${{ github.repository }}-qns | |
| tags: | | |
| # default | |
| type=schedule | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=ref,event=pr | |
| # set latest tag for default branch | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| - uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0 | |
| if: ${{ github.event_name != 'pull_request' }} | |
| with: | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| file: qns/Dockerfile | |
| build-args: RUST_VERSION=stable | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| # FIXME: gcc for arm64 currently segmentation faults :-( recheck periodically | |
| # platforms: 'linux/amd64, linux/arm64' | |
| platforms: 'linux/amd64' | |
| - uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0 | |
| id: docker_build_and_push | |
| with: | |
| tags: ${{ steps.meta.outputs.tags }} | |
| file: qns/Dockerfile | |
| build-args: RUST_VERSION=stable | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| platforms: 'linux/amd64' | |
| outputs: type=docker,dest=/tmp/${{ env.LATEST }}.tar | |
| - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 | |
| with: | |
| name: '${{ env.LATEST }} Docker image' | |
| path: /tmp/${{ env.LATEST }}.tar | |
| implementations: | |
| name: Determine interop pairs | |
| if: ${{ github.event_name != 'push' }} | |
| needs: docker-image | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pairs: ${{ steps.config.outputs.pairs }} | |
| implementations: ${{ steps.config.outputs.implementations }} | |
| env: | |
| URL: https://github.com/mozilla/neqo | |
| ROLE: both | |
| IMAGE: ${{ needs.docker-image.outputs.imageID }} | |
| steps: | |
| - id: config | |
| run: | | |
| # Add neqo-latest and some non-default implementations. | |
| # tquic: https://github.com/quic-interop/quic-interop-runner/pull/385/files | |
| # openssl: https://github.com/quic-interop/quic-interop-runner/pull/402/files | |
| cat <<EOF > additional.json | |
| { | |
| "neqo-latest": { | |
| "image": "$IMAGE", | |
| "url": "$URL", | |
| "role": "$ROLE" | |
| }, | |
| "tquic": { | |
| "image": "tquicgroup/tquic:latest", | |
| "url": "https://github.com/Tencent/tquic/", | |
| "role": "both" | |
| }, | |
| "openssl": { | |
| "image": "quay.io/openssl-ci/openssl-quic-interop", | |
| "url": "https://github.com/openssl/openssl", | |
| "role": "client" | |
| } | |
| } | |
| EOF | |
| curl -o runner.json https://raw.githubusercontent.com/quic-interop/quic-interop-runner/master/implementations.json | |
| jq '. += input' runner.json additional.json > implementations.json | |
| { | |
| echo "implementations<<EOF" | |
| cat implementations.json | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| # Determine valid interop pairs that contain $LATEST | |
| jq < implementations.json "[ | |
| [to_entries[] | select(.value.role==\"server\" or .value.role==\"both\").key] as \$servers | | |
| [to_entries[] | select(.value.role==\"client\" or .value.role==\"both\").key] as \$clients | | |
| \$clients[] as \$client | | |
| \$servers[] as \$server | | |
| \$client + \"$DELIM\" + \$server | | |
| select(contains(\"$LATEST\")) | |
| ]" > pairs.json | |
| { | |
| echo "pairs<<EOF" | |
| cat pairs.json | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| run-qns: | |
| name: Run QNS | |
| if: ${{ github.event_name != 'push' }} | |
| needs: implementations | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| pair: ${{ fromJson(needs.implementations.outputs.pairs) }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: '${{ env.LATEST }} Docker image' | |
| path: /tmp | |
| - run: docker load --input /tmp/${{ env.LATEST }}.tar | |
| - id: depair | |
| env: | |
| PAIR: ${{ matrix.pair }} | |
| run: | | |
| PAIR="${PAIR/$DELIM/%}" | |
| echo "client=$(echo "$PAIR" | cut -d% -f1)" >> "$GITHUB_OUTPUT" | |
| echo "server=$(echo "$PAIR" | cut -d% -f2)" >> "$GITHUB_OUTPUT" | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| persist-credentials: false | |
| # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged. | |
| - uses: ./.github/actions/quic-interop-runner | |
| timeout-minutes: ${{ fromJSON(env.TIMEOUT) }} | |
| with: | |
| client: ${{ steps.depair.outputs.client }} | |
| server: ${{ steps.depair.outputs.server }} | |
| implementations: ${{ needs.implementations.outputs.implementations }} | |
| report: | |
| name: Report results | |
| if: ${{ !cancelled() && github.event_name != 'push' }} | |
| needs: [run-qns, implementations] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| pattern: '*results' | |
| path: results | |
| - uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 | |
| with: | |
| path: results-main | |
| key: qns-${{ github.sha }} | |
| restore-keys: qns- | |
| - run: sudo apt-get install -y --no-install-recommends wdiff | |
| - env: | |
| PAIRS: ${{ needs.implementations.outputs.pairs }} | |
| run: | | |
| # shellcheck disable=SC2153 | |
| mapfile -t LIST < <(echo "$PAIRS" | jq '.[]' | sort) | |
| for PREFIX in "${LIST[@]}"; do | |
| PREFIX=$(echo "$PREFIX" | tr -d '"') | |
| CLIENT=$(echo "$PREFIX" | cut -f1 -d " ") | |
| SERVER=$(echo "$PREFIX" | cut -f3 -d " ") | |
| if [ "$CLIENT" == "$LATEST" ]; then | |
| ROLE=client | |
| else | |
| ROLE=server | |
| fi | |
| RUN="results/${PREFIX} results" | |
| PAIR="$CLIENT $DELIM $SERVER" | |
| if [ ! -e "$RUN/result.json" ]; then | |
| echo "* $PAIR: run cancelled after $TIMEOUT min" >> "$ROLE.failed.md" | |
| continue | |
| fi | |
| jq < "$RUN/result.json" ' | |
| . as $data | | |
| .results[][].result //= "failed" | | |
| { | |
| results: [.results[] | group_by(.result)[] | {(.[0].result): [.[] | .abbr]}] | | |
| add | |
| } | | |
| . + {log_url: $data.log_url} | |
| ' > "$RUN/$ROLE.grouped.json" | |
| for ROLE in client server; do | |
| [ ! -e "$RUN/$ROLE.grouped.json" ] && continue | |
| for GROUP in $(jq -r < "$RUN/$ROLE.grouped.json" '.results | keys[]'); do | |
| RESULT=$(jq < "$RUN/$ROLE.grouped.json" -r '.results.'"$GROUP"'[]' | fmt -w 1000) | |
| LOG=$(jq -r < "$RUN/$ROLE.grouped.json" -r '.log_url') | |
| BASELINE=$(mktemp) | |
| if [ -e "results-main/${PREFIX} results/$ROLE.grouped.json" ]; then | |
| jq < "results-main/${PREFIX} results/$ROLE.grouped.json" -r '.results.'"$GROUP"'[]' | fmt -w 1000 > "$BASELINE" | |
| else | |
| touch "$BASELINE" | |
| fi | |
| [ -n "$RESULT" ] || continue | |
| DIFF=$(wdiff -n "$BASELINE" - <<< "$RESULT" || true) | |
| if [ "$GROUP" == "failed" ]; then | |
| ADD=":warning:" | |
| ADD_DELIM="\*\*" | |
| DEL=":rocket:" | |
| DEL_DELIM="~~" | |
| elif [ "$GROUP" == "succeeded" ]; then | |
| ADD=":rocket:" | |
| ADD_DELIM="~~" | |
| DEL=":warning:" | |
| DEL_DELIM="\*\*" | |
| else | |
| ADD="" | |
| ADD_DELIM="" | |
| DEL="" | |
| DEL_DELIM="" | |
| fi | |
| RESULT=$(echo "$DIFF" | sed -E "s/\[-/ $DEL$DEL_DELIM/g; s/-\]/$DEL_DELIM /g; s/\{\+/ $ADD$ADD_DELIM/g; s/\+\}/$ADD_DELIM /g") | |
| echo "* [$PAIR]($LOG): $RESULT" >> "$ROLE.$GROUP.md" | |
| done | |
| done | |
| done | |
| { | |
| echo "### Failed Interop Tests" | |
| if [ -e client.failed.md ] || [ -e server.failed.md ]; then | |
| echo -n "[QUIC Interop Runner](https://github.com/quic-interop/quic-interop-runner), *client* vs. *server*" | |
| SHA=$(cat results-main/baseline-sha.txt || true) | |
| if [ -n "$SHA" ]; then | |
| echo ", differences relative to $SHA." | |
| fi | |
| echo | |
| echo "#### $LATEST as client" | |
| cat client.failed.md || echo | |
| echo "#### $LATEST as server" | |
| cat server.failed.md || echo | |
| else | |
| echo -n "None " | |
| if [ -e "client.succeeded.md" ] || [ -e "server.succeeded.md" ] || [ -e "client.unsupported.md" ] || [ -e "server.unsupported.md" ]; then | |
| echo ":tada:" | |
| else | |
| echo ":question:" | |
| fi | |
| fi | |
| echo "<details><summary>All results</summary>" | |
| echo | |
| for GROUP in succeeded unsupported; do | |
| echo "### ${GROUP^} Interop Tests" | |
| if [ -e "client.$GROUP.md" ] || [ -e "server.$GROUP.md" ]; then | |
| echo "[QUIC Interop Runner](https://github.com/quic-interop/quic-interop-runner), *client* vs. *server*" | |
| echo "#### $LATEST as client" | |
| cat "client.$GROUP.md" | |
| echo "#### $LATEST as server" | |
| cat "server.$GROUP.md" | |
| else | |
| echo "None :question:" | |
| fi | |
| done | |
| echo | |
| echo "</details>" | |
| } >> comment.md | |
| - if: ${{ github.ref == 'refs/heads/main' }} | |
| run: | | |
| rm -rf results-main || true | |
| mv results results-main | |
| echo "${{ github.sha }}" > results-main/baseline-sha.txt | |
| - if: ${{ github.ref == 'refs/heads/main' }} | |
| uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 | |
| with: | |
| path: results-main | |
| key: qns-${{ github.sha }} | |
| - uses: ./.github/actions/pr-comment-data-export | |
| with: | |
| name: ${{ github.workflow }} | |
| contents: comment.md |