From af38601843dd20f282a412082a90b2fc811d92b1 Mon Sep 17 00:00:00 2001
From: Marcin <marcin.radecki@cardinals.cc>
Date: Wed, 5 Jun 2024 06:20:42 +0000
Subject: [PATCH] L1-258: Make all e2e tests run on a chain with up to speed
 finalization (#1756)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

# Description

Recently, on main, there have been spurious nightly e2e logic test
failures. While the exact root cause is unknown, they happen when the
brand new chain is not yet up to speed with finalization. The scope of
this task is to wait for finalization to spin off before starting an e2e
case.

Currently, there’s an optional check called follow-up-finalization-check
that is run after some e2e tests. It’s very cheap regarding time
consumed, so we can always run before and after each e2e test. That
fulfills the requirements of this task. Note that some e2e expects
finalization to break after they run.

When testing this PR, a bug in the previous change was accidentally
found, causing 2 e2e cases to not run. A safeguard was added to prevent
such situations in the future.

## Type of change

Please delete options that are not relevant.

- Bug fix (non-breaking change which fixes an issue)
- New feature (non-breaking change which adds functionality)

# Checklist:

[Nightly e2e logic
tests](https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/9365029928)
- OK
[Safeguard against empty e2e CI
run](https://github.com/Cardinal-Cryptography/aleph-node/actions/runs/9364653772/job/25778573449)
- OK

---------

Co-authored-by: Grzegorz Gawryał <31205678+ggawryal@users.noreply.github.com>
---
 .github/actions/run-e2e-test/action.yml | 17 ++++++--
 .github/scripts/run_e2e_test.sh         |  8 +++-
 .github/workflows/_run-e2e-tests.yml    | 53 ++++++-------------------
 3 files changed, 32 insertions(+), 46 deletions(-)

diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml
index 6ab8fccc..512e70c1 100644
--- a/.github/actions/run-e2e-test/action.yml
+++ b/.github/actions/run-e2e-test/action.yml
@@ -25,9 +25,10 @@ inputs:
   non-reserved-seats:
     description: 'Number of non-reserved seats available to validators.'
     required: false
-  follow-up-finalization-check:
-    description: 'Whether to run a follow-up finalization check.'
+  check-finalization-after-test:
+    description: 'Whether to check finalization after e2e case run.'
     required: false
+    default: 'true'
   deploy-adder:
     description: 'Whether to deploy the adder sample contract to the node.'
     required: false
@@ -89,6 +90,13 @@ runs:
       shell: bash
       run: docker load -i aleph-e2e-client.tar
 
+    - name: Wait for the finalization before e2e test
+      shell: bash
+      run: |
+        export TIMEOUT_MINUTES="5m"
+        ./.github/scripts/run_e2e_test.sh -t finalization::finalization \
+          -a aleph-e2e-client:latest
+
     - name: Run single e2e test
       id: run-single-e2e-test
       shell: bash
@@ -144,9 +152,10 @@ runs:
         if-no-files-found: error
         retention-days: 7
 
-    - name: Run finalization e2e test
-      if: inputs.follow-up-finalization-check == 'true'
+    - name: Check finalization after e2e test
+      if: inputs.check-finalization-after-test == 'true'
       shell: bash
       run: |
+        export TIMEOUT_MINUTES="5m"
         ./.github/scripts/run_e2e_test.sh -t finalization::finalization \
           -a aleph-e2e-client:latest
diff --git a/.github/scripts/run_e2e_test.sh b/.github/scripts/run_e2e_test.sh
index b0d071c5..da56388a 100755
--- a/.github/scripts/run_e2e_test.sh
+++ b/.github/scripts/run_e2e_test.sh
@@ -85,7 +85,8 @@ if [[ -n "${OUT_LATENCY:-}" ]]; then
     ARGS+=(-e OUT_LATENCY)
 fi
 timeout_duration="${TIMEOUT_MINUTES:-20m}"
-echo "Running test, logs will be shown when tests finishes or after ${timeout_duration} timeout."
+echo "Running e2e test ${TEST_CASES}"
+echo "Logs will be shown when tests finishes or after ${timeout_duration} timeout."
 # a hack to set global timeout on a e2e testcase run
 # we can't do that on GH yaml level due to https://github.com/actions/runner/issues/1979
 docker_service=$(docker run -v "$(pwd)/contracts:/contracts" -v "$(pwd)/docker/data:/data" -v "$(pwd)/e2e-tests:/e2e-tests" -d "${ARGS[@]}" \
@@ -101,4 +102,9 @@ fi
 echo "Test exited with exit code ${docker_exit_code}"
 echo "Logs from test:"
 docker logs "${docker_service}"
+
+if docker logs "${docker_service}" 2>&1 | grep -q "^running 0 tests"; then
+  echo "No e2e testcase run. This is most likely an error."
+  exit 1
+fi
 exit "${docker_exit_code}"
diff --git a/.github/workflows/_run-e2e-tests.yml b/.github/workflows/_run-e2e-tests.yml
index e0dfab26..09442b87 100644
--- a/.github/workflows/_run-e2e-tests.yml
+++ b/.github/workflows/_run-e2e-tests.yml
@@ -45,7 +45,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: rewards::disable_node
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -142,7 +141,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: staking_era_payouts
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -159,7 +157,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: staking_new_validator
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -176,7 +173,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: change_validators
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -193,7 +189,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: fail_changing_validators
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -210,7 +205,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: validators_rotate
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -227,7 +221,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: era_payout::era_payout
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -244,7 +237,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: era_validators
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -261,7 +253,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: rewards::force_new_era
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -278,7 +269,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: rewards::points_stake_change
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -295,7 +285,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: rewards::change_stake_and_force_new_era
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -312,7 +301,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: points_basic
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -332,7 +320,6 @@ jobs:
           node-count: 6
           reserved-seats: 3
           non-reserved-seats: 3
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -349,7 +336,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: ban_automatic
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -366,7 +352,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: ban_manual
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -383,7 +368,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: clearing_session_count
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -400,7 +384,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: ban_threshold
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -417,7 +400,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: permissionless_ban
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -477,13 +459,12 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: finality_version::finality_version_change
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
 
-  run-e2e-committee-split-reserved-01:
-    name: Run committee split test with node-0 and node-1 dead
+  run-e2e-committee-split-reserved-12:
+    name: Run committee split test with node-1 and node-2 dead
     needs: [run-e2e-finalization-test]
     runs-on: ubuntu-20.04
     steps:
@@ -495,14 +476,15 @@ jobs:
           VALIDATOR_COUNT: 7
         uses: ./.github/actions/run-e2e-test
         with:
-          test-case: committee_split::split_test_reserved_01
+          test-case: committee_split::split_test_reserved_12
           node-count: 7
+          check-finalization-after-test: 'false'
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
 
-  run-e2e-committee-split-reserved-12:
-    name: Run committee split test with node-1 and node-2 dead
+  run-e2e-committee-split-reserved-23:
+    name: Run committee split test with node-2 and node-3 dead
     needs: [run-e2e-finalization-test]
     runs-on: ubuntu-20.04
     steps:
@@ -514,14 +496,15 @@ jobs:
           VALIDATOR_COUNT: 7
         uses: ./.github/actions/run-e2e-test
         with:
-          test-case: committee_split::split_test_reserved_12
+          test-case: committee_split::split_test_reserved_23
           node-count: 7
+          check-finalization-after-test: 'false'
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
 
-  run-e2e-committee-split-reserved-02:
-    name: Run committee split test with node-0 and node-2 dead
+  run-e2e-committee-split-reserved-13:
+    name: Run committee split test with node-1 and node-3 dead
     needs: [run-e2e-finalization-test]
     runs-on: ubuntu-20.04
     steps:
@@ -533,8 +516,9 @@ jobs:
           VALIDATOR_COUNT: 7
         uses: ./.github/actions/run-e2e-test
         with:
-          test-case: committee_split::split_test_reserved_02
+          test-case: committee_split::split_test_reserved_13
           node-count: 7
+          check-finalization-after-test: 'false'
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -553,7 +537,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: committee_split::split_test_success_without_any_deads
-          follow-up-finalization-check: true
           node-count: 7
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
@@ -573,7 +556,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: committee_split::split_test_success_with_one_dead
-          follow-up-finalization-check: true
           node-count: 7
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
@@ -591,7 +573,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: set_emergency_finalizer_test
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -608,7 +589,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: set_lenient_threshold_test
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -627,7 +607,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: chain_dead_scenario
-          follow-up-finalization-check: true
           node-count: 6
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
@@ -647,7 +626,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: committee_split::split_test_success_with_all_non_reserved_dead
-          follow-up-finalization-check: true
           node-count: 7
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
@@ -665,7 +643,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: any_proxy_works
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -682,7 +659,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: staking_proxy_works
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -699,7 +675,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: non_transfer_proxy_works
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -716,7 +691,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: fresh_chain_has_verifier_enabled
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -733,7 +707,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: verifier_can_be_disabled
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -750,7 +723,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: safe_mode_operations_are_disabled_for_users
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}
@@ -767,7 +739,6 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: safe_mode_is_configured_correctly
-          follow-up-finalization-check: true
           artifact-aleph-e2e-client-image: ${{ inputs.artifact-aleph-e2e-client-image }}
           artifact-aleph-node-image: ${{ inputs.artifact-aleph-node-image }}
           artifact-chain-bootstrapper-image: ${{ inputs.artifact-chain-bootstrapper-image }}