From daf2a4f676fa52b0ca9b0d3884184adb294e33e4 Mon Sep 17 00:00:00 2001
From: Petros Paraskevopoulos <9729923+ParaskP7@users.noreply.github.com>
Date: Wed, 15 Jan 2025 10:26:12 +0200
Subject: [PATCH] [Dependency Cache] Dependency Cache on CI per Project
 [without GRADLE_RO_DEP_CACHE] (#135)

* Gradle: Add project prefix with the standard buildkite pipeline slug

This is done in order to avoid having each client project adding its own
project prefix to distinguish the Gradle dependency cache key.

Test Only PRs:
- WCAndroid: https://github.com/woocommerce/woocommerce-android/
pull/13122
- JP/WPAndroid: https://github.com/wordpress-mobile/WordPress-Android/
pull/21540

For example, for WCAndroid, this would mean that instead of using this
'WOOCOMMERCE' value via this 'GRADLE_DEPENDENCY_CACHE_PROJECT_PREFIX'
environment variable, which ultimately is creating the below
'WOOCOMMERCE_GRADLE_DEPENDENCY_CACHE' key, instead, with this change the
'woocommerce-android_GRADLE_DEPENDENCY_CACHE' key will be
automatically created, taking the 'woocommerce-android' value from the
standard 'BUILDKITE_PIPELINE_SLUG' environment variable.

For more info see: p1734350857095859-slack-C020Q0Z579V

* Gradle: Measure how long it takes to save/restore the cache

* Gradle: Measure the (un)compressed cache size during save/restore

* Gradle: Add optional cache file to restore cache script

This is done in order for us to be able to output the uncompressed cache
size during cache restoration, just as it is being done during the cache
saving process, for consistency purposes.

But, given the fact that are also other clients of this 'restore_cache'
script, keeping this parameter optional was the preferred approach in
order to not affect, or need to update, any of those other clients of
this specific script.

* Gradle: Replace gradle ro dependency cache with gradle dependency cache

1. On saving, the script will create a new folder, within the
'GRADLE_HOME' space, named 'dependency-cache', where all the contents
from 'caches/modules-2' will be copied after the build finishes
(minus '*.lock' and 'gc.properties' files, as per the documentation).
2. On restoring, the script will create the 'caches/modules-2' folder,
within the 'GRADLE_HOME' space, and extract the saved contents into it
and before the build starts.
3. On restoring, the script will also check if 'modules-2' directory
exists prior to placing cache. This check is necessary because during
'restore_cache' it might happen that the dependency cache based on this
key is not available for download. Thus, no 'dependency-cache/module-2'
folder will exist to be copied into the 'caches/modules-2' folder, which
will break the script.

* Gradle: Add v2 suffix on the gradle dependency cache key

This is done in order to differentiate between the 'GRADLE_RO_DEP_CACHE'
and the 'GRADLE_DEP_CACHE' caches, and thus be able to test both caching
strategies in-parallel, without one overwriting the other during save,
or one reusing the other's cache during restore.

Test Only PRs:
- WCAndroid: https://github.com/woocommerce/woocommerce-android/
pull/13170
- JP/WPAndroid: https://github.com/wordpress-mobile/WordPress-Android/
pull/21550

* Gradle: Remove uncompressed cache size measurement

With this change only the compressed cache size is being measured and
echoed simply as 'cache size'. There is no real need to overcomplicate
things by echoing both, the compresses and uncompressed cache size.
---
 bin/restore_cache                   | 12 ++++++++++++
 bin/restore_gradle_dependency_cache | 14 ++++++++++++--
 bin/save_cache                      | 12 ++++++++++++
 bin/save_gradle_dependency_cache    | 17 ++++++++++-------
 4 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/bin/restore_cache b/bin/restore_cache
index f88fe687..c09314d9 100755
--- a/bin/restore_cache
+++ b/bin/restore_cache
@@ -2,6 +2,11 @@
 
 CACHE_KEY=$1
 
+bytes_to_mb() {
+	local bytes=$1
+	printf "%.2f" "$(awk "BEGIN {print $bytes / (1024 * 1024)}")"
+}
+
 S3_BUCKET_NAME=${CACHE_BUCKET_NAME-}
 if [ -z "$S3_BUCKET_NAME" ]; then
 	if [ -z "$BUILDKITE_PLUGIN_A8C_CI_TOOLKIT_BUCKET" ]; then
@@ -16,6 +21,7 @@ fi
 echo "Using $S3_BUCKET_NAME as cache bucket"
 
 if aws s3api head-object --bucket "$S3_BUCKET_NAME" --key "$CACHE_KEY" > /dev/null 2>&1; then
+	SECONDS=0
 	echo "Restoring cache entry $CACHE_KEY"
 
 	echo "	Downloading"
@@ -29,11 +35,17 @@ if aws s3api head-object --bucket "$S3_BUCKET_NAME" --key "$CACHE_KEY" > /dev/nu
 		aws s3 cp "s3://$S3_BUCKET_NAME/$CACHE_KEY" "$CACHE_KEY" --quiet
 	fi
 
+	CACHE_SIZE=$(wc -c < "$CACHE_KEY")
 	echo "	Decompressing"
 	tar -xf "$CACHE_KEY"
 
 	echo "	Cleaning Up"
 	rm "$CACHE_KEY"
+
+	duration=$SECONDS
+	echo "Cache entry successfully restored"
+	echo "	Duration: $((duration / 60)) minutes and $((duration % 60)) seconds"
+	echo "	Cache Size: $(bytes_to_mb "$CACHE_SIZE") MB"
 else
 	echo "No cache entry found for '$CACHE_KEY'"
 fi
diff --git a/bin/restore_gradle_dependency_cache b/bin/restore_gradle_dependency_cache
index d5a3f746..27e7bd10 100755
--- a/bin/restore_gradle_dependency_cache
+++ b/bin/restore_gradle_dependency_cache
@@ -1,15 +1,25 @@
 #!/bin/bash -eu
 
 # The key is shared with `bin/save_gradle_dependency_cache`
-GRADLE_DEPENDENCY_CACHE_KEY="GRADLE_DEPENDENCY_CACHE"
+GRADLE_DEPENDENCY_CACHE_KEY="${BUILDKITE_PIPELINE_SLUG}_GRADLE_DEPENDENCY_CACHE_V2"
 
 echo "Restoring Gradle dependency cache..."
 
-DEP_CACHE_BASE_FOLDER=$(dirname "$GRADLE_RO_DEP_CACHE")
+# The directory is shared with `bin/save_gradle_dependency_cache`
+GRADLE_DEP_CACHE="$GRADLE_HOME/dependency-cache"
+
+DEP_CACHE_BASE_FOLDER=$(dirname "$GRADLE_DEP_CACHE")
 
 # `save_cache` & `restore_cache` scripts only work if they are called from the same directory
 pushd "$DEP_CACHE_BASE_FOLDER"
 restore_cache "$GRADLE_DEPENDENCY_CACHE_KEY"
+
+if [ -d "$DEP_CACHE_FOLDER_NAME/modules-2/" ]; then
+	echo "Placing Gradle dependency cache..."
+	mv "$DEP_CACHE_FOLDER_NAME/modules-2/"* caches/modules-2/
+	rm -r "$DEP_CACHE_FOLDER_NAME"
+fi
+
 popd
 
 echo "---"
diff --git a/bin/save_cache b/bin/save_cache
index cb9b4a4b..045dc338 100755
--- a/bin/save_cache
+++ b/bin/save_cache
@@ -3,6 +3,11 @@
 CACHE_FILE=$1
 CACHE_KEY=$2
 
+bytes_to_mb() {
+	local bytes=$1
+	printf "%.2f" "$(awk "BEGIN {print $bytes / (1024 * 1024)}")"
+}
+
 if [ -z "$CACHE_FILE" ]; then
 	echo "You must pass the file or directory you want to be cached"
 	exit 1
@@ -45,6 +50,7 @@ if [[ "$SHOULD_FORCE" == '--force' ]]; then
 fi
 
 if ! aws s3api head-object --bucket "$S3_BUCKET_NAME" --key "$CACHE_KEY" > /dev/null 2>&1; then
+	SECONDS=0
 	echo "No existing cache entry for $CACHE_KEY – storing in cache"
 
 	echo "	Compressing"
@@ -59,6 +65,7 @@ if ! aws s3api head-object --bucket "$S3_BUCKET_NAME" --key "$CACHE_KEY" > /dev/
 	else
 		tar -czf "$CACHE_KEY" "$CACHE_FILE"
 	fi
+	CACHE_SIZE=$(wc -c < "$CACHE_KEY")
 
 	echo "	Uploading"
 	# If the bucket has transfer acceleration enabled, use it!
@@ -73,6 +80,11 @@ if ! aws s3api head-object --bucket "$S3_BUCKET_NAME" --key "$CACHE_KEY" > /dev/
 
 	echo "	Cleaning Up"
 	rm "$CACHE_KEY"
+
+	duration=$SECONDS
+	echo "Cache entry successfully saved"
+	echo "	Duration: $((duration / 60)) minutes and $((duration % 60)) seconds"
+	echo "	Cache Size: $(bytes_to_mb "$CACHE_SIZE") MB"
 else
 	echo "This file is already cached – skipping upload"
 fi
diff --git a/bin/save_gradle_dependency_cache b/bin/save_gradle_dependency_cache
index f0a08fff..e0c2546c 100755
--- a/bin/save_gradle_dependency_cache
+++ b/bin/save_gradle_dependency_cache
@@ -1,20 +1,23 @@
 #!/bin/bash -eu
 
 # The key is shared with `bin/restore_gradle_dependency_cache`
-GRADLE_DEPENDENCY_CACHE_KEY="GRADLE_DEPENDENCY_CACHE"
+GRADLE_DEPENDENCY_CACHE_KEY="${BUILDKITE_PIPELINE_SLUG}_GRADLE_DEPENDENCY_CACHE_V2"
 
 echo "Saving Gradle dependency cache..."
 
-mkdir -p "$GRADLE_RO_DEP_CACHE"
+# The directory is shared with `bin/restore_gradle_dependency_cache`
+GRADLE_DEP_CACHE="$GRADLE_HOME/dependency-cache"
+
+mkdir -p "$GRADLE_DEP_CACHE"
 
 # https://docs.gradle.org/current/userguide/dependency_resolution.html#sub:cache_copy
 # Gradle suggests removing the `*.lock` files and the `gc.properties` file before saving cache
-cp -r ~/.gradle/caches/modules-2 "$GRADLE_RO_DEP_CACHE" \
-    && find "$GRADLE_RO_DEP_CACHE" -name "*.lock" -type f -delete \
-    && find "$GRADLE_RO_DEP_CACHE" -name "gc.properties" -type f -delete
+cp -r ~/.gradle/caches/modules-2 "$GRADLE_DEP_CACHE" \
+    && find "$GRADLE_DEP_CACHE" -name "*.lock" -type f -delete \
+    && find "$GRADLE_DEP_CACHE" -name "gc.properties" -type f -delete
 
-DEP_CACHE_BASE_FOLDER=$(dirname "$GRADLE_RO_DEP_CACHE")
-DEP_CACHE_FOLDER_NAME=$(basename "$GRADLE_RO_DEP_CACHE")
+DEP_CACHE_BASE_FOLDER=$(dirname "$GRADLE_DEP_CACHE")
+DEP_CACHE_FOLDER_NAME=$(basename "$GRADLE_DEP_CACHE")
 
 # `save_cache` & `restore_cache` scripts only work if they are called from the same directory
 pushd "$DEP_CACHE_BASE_FOLDER"