From 63a8b6c786642b0d4069cff326634f560b5aa603 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Mon, 20 Jan 2025 15:32:20 +0200 Subject: [PATCH 01/11] CI: Update a8c-ci-toolkit to 3.9.1 Release Notes: https://github.com/Automattic/ a8c-ci-toolkit-buildkite-plugin/releases/tag/3.9.1 FYI: This change points to that version of the 'A8C CI Toolkit' where both, the #135 and #138 PRs, are merged into 'trunk', and, the new dependency cache mechanism, per project, without 'GRADLE_RO_DEP_CACHE', is fully operational. A8C CI Toolkit #135 PR: [Dependency Cache] Dependency Cache on CI per Project [without GRADLE_RO_DEP_CACHE] #135 - Automattic/a8c-ci-toolkit-buildkite-plugin#135 A8C CI Toolkit #138 PR: [Dependency Cache] Fix Dependency Cache #138 - Automattic/a8c-ci-toolkit-buildkite-plugin#138 --- .buildkite/shared-pipeline-vars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/shared-pipeline-vars b/.buildkite/shared-pipeline-vars index 707a5b0b575..0740bc5b33b 100644 --- a/.buildkite/shared-pipeline-vars +++ b/.buildkite/shared-pipeline-vars @@ -3,4 +3,4 @@ # This file is `source`'d before calling `buildkite-agent pipeline upload`, and can be used # to set up some variables that will be interpolated in the `.yml` pipeline before uploading it. -export CI_TOOLKIT="automattic/a8c-ci-toolkit#3.7.1" +export CI_TOOLKIT="automattic/a8c-ci-toolkit#3.9.1" From 7e0580591c1359b62beeb87d8c4b9595beb4ea46 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Mon, 20 Jan 2025 15:46:09 +0200 Subject: [PATCH 02/11] CI: Add a scheduled dependency cache job (save cache) FYI: This job will be then used by 'buildkite-ci' and configured as a 'buildkite_pipeline_schedule' with a weekly frequency. PS: The targeted 'pipeline' related jobs are: - App - Lint - Unit Tests - Android tests --- .buildkite/commands/save-cache.sh | 35 +++++++++++++++++++++++ .buildkite/schedules/dependency-cache.yml | 24 ++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100755 .buildkite/commands/save-cache.sh create mode 100644 .buildkite/schedules/dependency-cache.yml diff --git a/.buildkite/commands/save-cache.sh b/.buildkite/commands/save-cache.sh new file mode 100755 index 00000000000..4f944c13e63 --- /dev/null +++ b/.buildkite/commands/save-cache.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -euo pipefail + +echo "--- :rubygems: Setting up Gems" +install_gems + +echo "--- :closed_lock_with_key: Installing Secrets" +bundle exec fastlane run configure_apply + +# .buildkite/pipeline.yml -> ./gradlew assembleRelease +# .buildkite/commands/prototype-build.sh -> build_and_upload_prototype_build +# -> prototype_build_type = 'debugProd' +echo "--- 🛠 Download Mobile App Dependencies [Assemble Apps]" +./gradlew assembleDebug +echo "" + +# .buildkite/commands/lint.sh -> ./gradlew :app:lintRelease + ./gradlew :automotive:lintRelease :wear:lintRelease +echo "--- 🧹 Download Lint Dependencies [Lint Apps]" +./gradlew lintDebug +echo "" + +# .buildkite/pipeline.yml -> ./gradlew testDebugUnitTest +echo "--- 🧪 Download Unit Test Dependencies [Assemble Unit Tests]" +./gradlew testDebugUnitTest +echo "" + +# .buildkite/pipeline.yml -> build_and_instrumented_test +# -> gradle(tasks: %w[assembleDebug assembleDebugAndroidTest]) +echo "--- 🧪 Download Android Test Dependencies [Assemble Android Tests]" +./gradlew assembleDebugAndroidTest +echo "" + +echo "--- 💾 Save Cache" +save_gradle_dependency_cache diff --git a/.buildkite/schedules/dependency-cache.yml b/.buildkite/schedules/dependency-cache.yml new file mode 100644 index 00000000000..ad18b3e4ef1 --- /dev/null +++ b/.buildkite/schedules/dependency-cache.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +--- + +agents: + queue: "android" + +steps: + - label: "dependency cache" + command: | + echo "--- 💾 Download and Cache Dependencies" + .buildkite/commands/save-cache.sh + plugins: [$CI_TOOLKIT] + +notify: + - slack: + channels: + - "#android-core-notifs" + message: "The dependency cache has been updated." + if: build.state == "passed" + - slack: + channels: + - "#android-core-notifs" + message: "Failure to update the dependency cache." + if: build.state == "failed" From 4c8e0d19a8f2d28fb2095280580b03fbc133e59e Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Mon, 20 Jan 2025 15:48:37 +0200 Subject: [PATCH 03/11] CI: Add restore cache and run on targeted pipeline related jobs only FYI: The targeted 'pipeline' related jobs are: - App - Lint - Unit Tests - Android Tests --- .buildkite/commands/lint.sh | 2 ++ .buildkite/commands/prototype-build.sh | 2 ++ .buildkite/commands/restore-cache.sh | 6 ++++++ .buildkite/pipeline.yml | 3 +++ 4 files changed, 13 insertions(+) create mode 100755 .buildkite/commands/restore-cache.sh diff --git a/.buildkite/commands/lint.sh b/.buildkite/commands/lint.sh index 229a305f26a..9984167896c 100755 --- a/.buildkite/commands/lint.sh +++ b/.buildkite/commands/lint.sh @@ -1,5 +1,7 @@ #!/bin/bash -u +.buildkite/commands/restore-cache.sh + echo "--- :rubygems: Setting up Gems" install_gems diff --git a/.buildkite/commands/prototype-build.sh b/.buildkite/commands/prototype-build.sh index dceeea803b9..007921f4945 100755 --- a/.buildkite/commands/prototype-build.sh +++ b/.buildkite/commands/prototype-build.sh @@ -1,5 +1,7 @@ #!/bin/bash -eu +.buildkite/commands/restore-cache.sh + echo "--- :rubygems: Setting up Gems" install_gems diff --git a/.buildkite/commands/restore-cache.sh b/.buildkite/commands/restore-cache.sh new file mode 100755 index 00000000000..480ae698790 --- /dev/null +++ b/.buildkite/commands/restore-cache.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +echo "--- 💾 Restore Cache" +restore_gradle_dependency_cache || true diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 79a1af1ecd8..b239abfe005 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -33,6 +33,7 @@ steps: - label: 'Unit tests' command: | + .buildkite/commands/restore-cache.sh echo "--- 🧪 Testing" ./gradlew testDebugUnitTest plugins: [$CI_TOOLKIT] @@ -52,6 +53,7 @@ steps: - label: "Instrumented tests" command: | + .buildkite/commands/restore-cache.sh echo "--- :rubygems: Setting up Gems" install_gems echo "--- :closed_lock_with_key: Installing Secrets" @@ -64,6 +66,7 @@ steps: - label: "Assemble release APK" command: | + .buildkite/commands/restore-cache.sh echo "--- :rubygems: Setting up Gems" install_gems echo "--- :closed_lock_with_key: Installing Secrets" From 097d194d6e4d50c52e28c54e434244c6483fd91d Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Mon, 20 Jan 2025 15:51:50 +0200 Subject: [PATCH 04/11] CI: Extract assemble release apk into a commands script --- .buildkite/commands/assemble-release-apk.sh | 15 +++++++++++++++ .buildkite/pipeline.yml | 9 +-------- 2 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 .buildkite/commands/assemble-release-apk.sh diff --git a/.buildkite/commands/assemble-release-apk.sh b/.buildkite/commands/assemble-release-apk.sh new file mode 100644 index 00000000000..ee47baa8181 --- /dev/null +++ b/.buildkite/commands/assemble-release-apk.sh @@ -0,0 +1,15 @@ +#!/bin/bash -eu + +.buildkite/commands/restore-cache.sh + +echo "--- :rubygems: Setting up Gems" + +install_gems + +echo "--- :closed_lock_with_key: Installing Secrets" + +bundle exec fastlane run configure_apply + +echo "--- ⚙️ Building release variant" + +./gradlew assembleRelease -PskipSentryProguardMappingUpload=true diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b239abfe005..a40bdf246ea 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -65,14 +65,7 @@ steps: - "**/build/instrumented-tests/**/*" - label: "Assemble release APK" - command: | - .buildkite/commands/restore-cache.sh - echo "--- :rubygems: Setting up Gems" - install_gems - echo "--- :closed_lock_with_key: Installing Secrets" - bundle exec fastlane run configure_apply - echo "--- ⚙️ Building release variant" - ./gradlew assembleRelease -PskipSentryProguardMappingUpload=true + command: ".buildkite/commands/assemble-release-apk.sh" plugins: [ $CI_TOOLKIT ] artifact_paths: - "**/build/outputs/apk/**/*" From 1415267975854b4f31d8bbfd9b9ffa377cbb051b Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Mon, 20 Jan 2025 15:53:30 +0200 Subject: [PATCH 05/11] CI: Extract instrumented tests into a commands script --- .buildkite/commands/run-instrumented-tests.sh | 15 +++++++++++++++ .buildkite/pipeline.yml | 9 +-------- 2 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 .buildkite/commands/run-instrumented-tests.sh diff --git a/.buildkite/commands/run-instrumented-tests.sh b/.buildkite/commands/run-instrumented-tests.sh new file mode 100644 index 00000000000..c98331941db --- /dev/null +++ b/.buildkite/commands/run-instrumented-tests.sh @@ -0,0 +1,15 @@ +#!/bin/bash -eu + +.buildkite/commands/restore-cache.sh + +echo "--- :rubygems: Setting up Gems" + +install_gems + +echo "--- :closed_lock_with_key: Installing Secrets" + +bundle exec fastlane run configure_apply + +echo "--- 🧪 Testing" + +bundle exec fastlane build_and_instrumented_test diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a40bdf246ea..050f52672ba 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -52,14 +52,7 @@ steps: plugins: [$CI_TOOLKIT] - label: "Instrumented tests" - command: | - .buildkite/commands/restore-cache.sh - echo "--- :rubygems: Setting up Gems" - install_gems - echo "--- :closed_lock_with_key: Installing Secrets" - bundle exec fastlane run configure_apply - echo "--- 🧪 Testing" - bundle exec fastlane build_and_instrumented_test + command: ".buildkite/commands/run-instrumented-tests.sh" plugins: [$CI_TOOLKIT] artifact_paths: - "**/build/instrumented-tests/**/*" From 2bb7fa4c3136981caba61781ddf41d361e2c3256 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Mon, 20 Jan 2025 15:54:56 +0200 Subject: [PATCH 06/11] CI: Extract unit tests into a commands script --- .buildkite/commands/run-unit-tests.sh | 7 +++++++ .buildkite/pipeline.yml | 5 +---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100755 .buildkite/commands/run-unit-tests.sh diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh new file mode 100755 index 00000000000..542d2391476 --- /dev/null +++ b/.buildkite/commands/run-unit-tests.sh @@ -0,0 +1,7 @@ +#!/bin/bash -eu + +.buildkite/commands/restore-cache.sh + +echo "--- 🧪 Testing" + +./gradlew testDebugUnitTest diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 050f52672ba..5265fdab283 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -32,10 +32,7 @@ steps: - "**/build/reports/lint-results*.*" - label: 'Unit tests' - command: | - .buildkite/commands/restore-cache.sh - echo "--- 🧪 Testing" - ./gradlew testDebugUnitTest + command: ".buildkite/commands/run-unit-tests.sh" plugins: [$CI_TOOLKIT] - label: 'Dependency diff' From f98afbd14d7cfce7a85aaed1595428ffd0af3a07 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Mon, 20 Jan 2025 16:54:13 +0200 Subject: [PATCH 07/11] CI: Make lint debug run only on specific app related modules Otherwise, running 'lintDebug', which triggers Lint on all modules will fail. This is due to the fact that a corresponding 'lint-baseline.xml' file only exists on these 'app', 'automotive' and 'wear' app related modules. --- .buildkite/commands/save-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/commands/save-cache.sh b/.buildkite/commands/save-cache.sh index 4f944c13e63..ccdf37b5cf8 100755 --- a/.buildkite/commands/save-cache.sh +++ b/.buildkite/commands/save-cache.sh @@ -17,7 +17,7 @@ echo "" # .buildkite/commands/lint.sh -> ./gradlew :app:lintRelease + ./gradlew :automotive:lintRelease :wear:lintRelease echo "--- 🧹 Download Lint Dependencies [Lint Apps]" -./gradlew lintDebug +./gradlew :app:lintDebug :automotive:lintDebug :wear:lintDebug echo "" # .buildkite/pipeline.yml -> ./gradlew testDebugUnitTest From f97c320b04dce9fee7b440e78ab0bad2fc66f10c Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Tue, 21 Jan 2025 17:01:08 +0200 Subject: [PATCH 08/11] CI: Replace test with assemble debug unit test on save cache script Having 'testDebugUnitTest' instead of 'assembleDebugUnitTest' was actually a copy-pasting oversight. There is no need to run any tests for the unit tests dependencies to be downloaded. --- .buildkite/commands/save-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/commands/save-cache.sh b/.buildkite/commands/save-cache.sh index ccdf37b5cf8..a01fa8e3746 100755 --- a/.buildkite/commands/save-cache.sh +++ b/.buildkite/commands/save-cache.sh @@ -22,7 +22,7 @@ echo "" # .buildkite/pipeline.yml -> ./gradlew testDebugUnitTest echo "--- 🧪 Download Unit Test Dependencies [Assemble Unit Tests]" -./gradlew testDebugUnitTest +./gradlew assembleDebugUnitTest echo "" # .buildkite/pipeline.yml -> build_and_instrumented_test From e2f5907a2819809766a84e7838e1e21834981f41 Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Wed, 22 Jan 2025 12:55:24 +0200 Subject: [PATCH 09/11] CI: Use a more compact form to define the bash behavior of this script --- .buildkite/commands/restore-cache.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.buildkite/commands/restore-cache.sh b/.buildkite/commands/restore-cache.sh index 480ae698790..e0807c3eff3 100755 --- a/.buildkite/commands/restore-cache.sh +++ b/.buildkite/commands/restore-cache.sh @@ -1,6 +1,4 @@ -#!/bin/bash - -set -e +#!/bin/bash -e echo "--- 💾 Restore Cache" restore_gradle_dependency_cache || true From b3a3726d00f875462774484fea269bec2dd2eccf Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Wed, 22 Jan 2025 12:56:31 +0200 Subject: [PATCH 10/11] CI: Use a syntax that makes the script file path relative --- .buildkite/commands/assemble-release-apk.sh | 2 +- .buildkite/commands/lint.sh | 2 +- .buildkite/commands/prototype-build.sh | 2 +- .buildkite/commands/run-instrumented-tests.sh | 2 +- .buildkite/commands/run-unit-tests.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/commands/assemble-release-apk.sh b/.buildkite/commands/assemble-release-apk.sh index ee47baa8181..ee11d661c48 100644 --- a/.buildkite/commands/assemble-release-apk.sh +++ b/.buildkite/commands/assemble-release-apk.sh @@ -1,6 +1,6 @@ #!/bin/bash -eu -.buildkite/commands/restore-cache.sh +"$(dirname "${BASH_SOURCE[0]}")/restore-cache.sh" echo "--- :rubygems: Setting up Gems" diff --git a/.buildkite/commands/lint.sh b/.buildkite/commands/lint.sh index 9984167896c..413c8bef09b 100755 --- a/.buildkite/commands/lint.sh +++ b/.buildkite/commands/lint.sh @@ -1,6 +1,6 @@ #!/bin/bash -u -.buildkite/commands/restore-cache.sh +"$(dirname "${BASH_SOURCE[0]}")/restore-cache.sh" echo "--- :rubygems: Setting up Gems" install_gems diff --git a/.buildkite/commands/prototype-build.sh b/.buildkite/commands/prototype-build.sh index 007921f4945..2770879ec84 100755 --- a/.buildkite/commands/prototype-build.sh +++ b/.buildkite/commands/prototype-build.sh @@ -1,6 +1,6 @@ #!/bin/bash -eu -.buildkite/commands/restore-cache.sh +"$(dirname "${BASH_SOURCE[0]}")/restore-cache.sh" echo "--- :rubygems: Setting up Gems" install_gems diff --git a/.buildkite/commands/run-instrumented-tests.sh b/.buildkite/commands/run-instrumented-tests.sh index c98331941db..3ce0be386e7 100644 --- a/.buildkite/commands/run-instrumented-tests.sh +++ b/.buildkite/commands/run-instrumented-tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -eu -.buildkite/commands/restore-cache.sh +"$(dirname "${BASH_SOURCE[0]}")/restore-cache.sh" echo "--- :rubygems: Setting up Gems" diff --git a/.buildkite/commands/run-unit-tests.sh b/.buildkite/commands/run-unit-tests.sh index 542d2391476..947e1508a9a 100755 --- a/.buildkite/commands/run-unit-tests.sh +++ b/.buildkite/commands/run-unit-tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -eu -.buildkite/commands/restore-cache.sh +"$(dirname "${BASH_SOURCE[0]}")/restore-cache.sh" echo "--- 🧪 Testing" From afedd3fac1bd98e1f87ce860f74d107cddd9872d Mon Sep 17 00:00:00 2001 From: Petros Paraskevopoulos Date: Wed, 22 Jan 2025 12:57:08 +0200 Subject: [PATCH 11/11] CI: Use the echo command with its emoji and name as the step label --- .buildkite/schedules/dependency-cache.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/schedules/dependency-cache.yml b/.buildkite/schedules/dependency-cache.yml index ad18b3e4ef1..1e61ab6a5f1 100644 --- a/.buildkite/schedules/dependency-cache.yml +++ b/.buildkite/schedules/dependency-cache.yml @@ -5,10 +5,8 @@ agents: queue: "android" steps: - - label: "dependency cache" - command: | - echo "--- 💾 Download and Cache Dependencies" - .buildkite/commands/save-cache.sh + - label: "💾 Download and Cache Dependencies" + command: .buildkite/commands/save-cache.sh plugins: [$CI_TOOLKIT] notify: