From ac2f5c489351673de1adc381a37fff4a722a40e6 Mon Sep 17 00:00:00 2001 From: marboledacci Date: Tue, 10 Sep 2024 15:06:35 -0500 Subject: [PATCH] Add cache_to option in the build and publish commands (#195) * Add cache_to option to docker build and publish * Fix trailing spaces * Remove macos old tests * Remove old resource class for macos * Use docker buildx * Add platfrom option to credentials helper * Fix arch parameter name * Use load when generating cache * Use a bigger image to test the cache * Add bigger image * Use a different tag instead of repo * use machine executor for cache test * Use cache from on save cache step * Add more debug to build script * Use arm to cache test * Use amd64 * update cache parameters description --- .circleci/test-deploy.yml | 66 +++++++++++++++---- src/commands/build.yml | 14 +++- src/commands/check.yml | 12 +++- .../install-docker-credential-helper.yml | 10 ++- src/examples/with-cache-to.yml | 15 +++++ src/jobs/publish.yml | 13 +++- src/scripts/build.sh | 8 ++- .../install-docker-credential-helper.sh | 12 ++-- test4.Dockerfile | 54 +++++++++++++++ 9 files changed, 179 insertions(+), 25 deletions(-) create mode 100644 src/examples/with-cache-to.yml create mode 100644 test4.Dockerfile diff --git a/.circleci/test-deploy.yml b/.circleci/test-deploy.yml index 0236a2d..f3e8fd5 100644 --- a/.circleci/test-deploy.yml +++ b/.circleci/test-deploy.yml @@ -8,9 +8,11 @@ promotion_requires: &promotion_requires test-check-command-docker, test-check-command-machine, test-check-command-macos, + test-check-command-machine-arm, test-credentials-store-docker, test-credentials-store-machine, test-credentials-store-macos, + test-credentials-store-machine-arm, test-credentials-store-docker-custom-tag, test-credentials-store-machine-custom-tag, test-credentials-store-macos-custom-tag, @@ -25,7 +27,6 @@ promotion_requires: &promotion_requires test-install-docker-tools-docker-latest, test-install-docker-tools-docker-old, test-install-docker-tools-macos-latest, - test-install-docker-tools-macos-old, test-install-docker-tools-machine-latest, test-install-docker-tools-machine-old, test-install-docker-tools-machine-arm, @@ -35,7 +36,6 @@ promotion_requires: &promotion_requires test-docker-latest, test-docker-old, test-macos-latest, - test-macos-old, test-machine-latest, test-machine-old, test-machine-arm, @@ -124,12 +124,17 @@ jobs: type: env_var_name use-docker-credentials-store: type: boolean + arch: + default: amd64 + type: enum + enum: [amd64, arm64] executor: <> steps: - docker/check: docker-username: <> docker-password: <> use-docker-credentials-store: <> + arch: <> test-credentials-store: parameters: executor: @@ -145,11 +150,16 @@ jobs: release-tag: type: string default: "" + arch: + default: amd64 + type: enum + enum: [amd64, arm64] executor: <> steps: - docker/install-docker-credential-helper: helper-name: <> release-tag: <> + arch: <> - docker/configure-docker-credentials-store: helper-name: <> - run: @@ -360,6 +370,15 @@ workflows: docker-password: DOCKER_TOKEN use-docker-credentials-store: true filters: *filters + - test-check-command: + name: test-check-command-machine-arm + executor: machine-arm + context: cimg-docker-image-building + docker-username: DOCKER_USER + docker-password: DOCKER_TOKEN + arch: arm64 + use-docker-credentials-store: true + filters: *filters - test-check-command: name: test-check-command-macos executor: macos-latest @@ -388,6 +407,14 @@ workflows: docker-username: DOCKER_USER docker-password: DOCKER_TOKEN filters: *filters + - test-credentials-store: + name: test-credentials-store-machine-arm + executor: machine-arm + context: cimg-docker-image-building + docker-username: DOCKER_USER + docker-password: DOCKER_TOKEN + arch: arm64 + filters: *filters - test-credentials-store: name: test-credentials-store-macos executor: macos-latest @@ -467,17 +494,33 @@ workflows: docker-password: DOCKER_TOKEN use-docker-credentials-store: true filters: *filters + - docker/publish: + name: publish-docker-save-cache + executor: machine-latest + context: cimg-docker-image-building + dockerfile: test4.Dockerfile + image: ccitest/docker-orb-test + tag: $CIRCLE_SHA1 + cache_to: ccitest/docker-orb-test:cache-big,ccitest/docker-orb-test:cache2-big + cache_from: ccitest/docker-orb-test:cache2-big + docker-username: DOCKER_USER + docker-password: DOCKER_TOKEN + pre-steps: + - docker/install-docker-credential-helper: + # release-tag: v0.6.3 + - docker/configure-docker-credentials-store: + helper-name: pass + filters: *filters - docker/publish: name: publish-docker-cache requires: - - publish-docker - executor: docker-latest + - publish-docker-save-cache + executor: machine-latest context: cimg-docker-image-building - use-remote-docker: true - dockerfile: test.Dockerfile + dockerfile: test4.Dockerfile image: ccitest/docker-orb-test tag: $CIRCLE_SHA1 - cache_from: ccitest/docker-orb-test:$CIRCLE_SHA1 + cache_from: ccitest/docker-orb-test:cache-big docker-username: DOCKER_USER docker-password: DOCKER_TOKEN pre-steps: @@ -580,7 +623,7 @@ workflows: name: test-install-docker-tools-<< matrix.executor >> matrix: parameters: - executor: [macos-latest, macos-old] + executor: [macos-latest] install-goss: false filters: *filters # end test-install-docker-tools @@ -596,7 +639,7 @@ workflows: name: test-<< matrix.executor >> matrix: parameters: - executor: [macos-latest, macos-old, machine-arm] + executor: [macos-latest, machine-arm] install-goss: false filters: *filters # end test @@ -617,14 +660,9 @@ workflows: only: /^v[0-9]+\.[0-9]+\.[0-9]+$/ executors: - macos-old: - macos: - xcode: 14.3.1 - resource_class: macos.x86.medium.gen2 macos-latest: macos: xcode: 15.2.0 - resource_class: macos.x86.medium.gen2 docker-old: docker: - image: cimg/base:2020.08-20.04 diff --git a/src/commands/build.yml b/src/commands/build.yml index 8f1a40c..b03f756 100644 --- a/src/commands/build.yml +++ b/src/commands/build.yml @@ -59,8 +59,17 @@ parameters: default: "" description: > Comma-separated list of images, images will first be pulled, then - passed as the --cache-from build argument - https://docs.docker.com/engine/reference/commandline/build/ + passed as the --cache-from build argument. + This parameter only support registry type, the accepted syntax is: + cache_from: user/app:cache,user/app2:cache2 + + cache_to: + type: string + default: "" + description: > + Comman-separated list of images where cache will be pushed to. + This parameter only support registry type, the accepted syntax is: + cache_to: user/app:cache,user/app2:cache2 debug: type: boolean @@ -128,6 +137,7 @@ steps: no_output_timeout: << parameters.no_output_timeout >> environment: PARAM_CACHE_FROM: <> + PARAM_CACHE_TO: <> PARAM_DOCKER_CONTEXT: <> PARAM_DOCKERFILE_NAME: <> PARAM_DOCKERFILE_PATH: <> diff --git a/src/commands/check.yml b/src/commands/check.yml index 4732cc9..48381be 100644 --- a/src/commands/check.yml +++ b/src/commands/check.yml @@ -9,6 +9,15 @@ parameters: default: docker.io description: Name of registry to use, defaults to docker.io + arch: + type: enum + default: amd64 + enum: [amd64, arm64] + description: | + Which architecture is being used. + Values accepted are amd64 and arm64. Defaults to amd64. + When running on MacOS arm64 will be used. + docker-username: type: env_var_name default: DOCKER_LOGIN @@ -32,7 +41,8 @@ steps: - when: condition: <> steps: - - install-docker-credential-helper + - install-docker-credential-helper: + arch: <> - configure-docker-credentials-store - run: diff --git a/src/commands/install-docker-credential-helper.yml b/src/commands/install-docker-credential-helper.yml index 1fa9749..e3592a4 100644 --- a/src/commands/install-docker-credential-helper.yml +++ b/src/commands/install-docker-credential-helper.yml @@ -19,12 +19,20 @@ parameters: Note: Pre or alpha releases cannot be specified. type: string default: "" - + arch: + type: enum + default: amd64 + enum: [amd64, arm64] + description: | + Which architecture is being used. + Values accepted are amd64 and arm64. Defaults to amd64. + When running on MacOS arm64 will be used. steps: - run: name: Install Docker credential helper environment: PARAM_HELPER_NAME: << parameters.helper-name >> PARAM_RELEASE_TAG: << parameters.release-tag >> + PARAM_ARCH: << parameters.arch >> SCRIPT_UTILS: <> command: << include(scripts/install-docker-credential-helper.sh) >> diff --git a/src/examples/with-cache-to.yml b/src/examples/with-cache-to.yml new file mode 100644 index 0000000..20dad97 --- /dev/null +++ b/src/examples/with-cache-to.yml @@ -0,0 +1,15 @@ +description: > + Build/publish a Docker image using --cache-to +usage: + version: 2.1 + + orbs: + docker: circleci/docker@x.y.z + + workflows: + build-docker-image-only: + jobs: + - docker/publish: + image: $CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME + tag: latest + cache_to: $CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME:cache diff --git a/src/jobs/publish.yml b/src/jobs/publish.yml index bc2e0a4..9fec9be 100644 --- a/src/jobs/publish.yml +++ b/src/jobs/publish.yml @@ -96,7 +96,16 @@ parameters: default: "" description: > Comma-separated list of images to pull before build for --cache-from - https://docs.docker.com/engine/reference/commandline/build/ + This parameter only support registry type, the accepted syntax is: + cache_from: user/app:cache,user/app2:cache2 + + cache_to: + type: string + default: "" + description: > + Comman-separated list of images where cache will be pushed to. + This parameter only support registry type, the accepted syntax is: + cache_to: user/app:cache,user/app2:cache2 after_checkout: description: Optional steps to run after checking out the code @@ -135,7 +144,6 @@ parameters: description: > Name of environment variable storing your Docker password - attach-at: type: string default: "" @@ -200,6 +208,7 @@ steps: image: <> tag: <> cache_from: <> + cache_to: <> extra_build_args: <> lint-dockerfile: <> treat-warnings-as-errors: <> diff --git a/src/scripts/build.sh b/src/scripts/build.sh index b535589..b2daebc 100644 --- a/src/scripts/build.sh +++ b/src/scripts/build.sh @@ -66,6 +66,12 @@ if [ -n "$PARAM_CACHE_FROM" ]; then build_args+=("--cache-from=$PARAM_CACHE_FROM") fi +if [ -n "$PARAM_CACHE_TO" ]; then + docker buildx create --name cache --use + docker buildx use cache + build_args+=("--cache-to=$PARAM_CACHE_TO" --load) +fi + # The context must be the last argument. build_args+=("$PARAM_DOCKER_CONTEXT") @@ -74,7 +80,7 @@ IFS=' ' set -x # shellcheck disable=SC2048 # We want word splitting here. -docker build ${build_args[*]} +docker buildx build ${build_args[*]} set +x IFS="$old_ifs" diff --git a/src/scripts/install-docker-credential-helper.sh b/src/scripts/install-docker-credential-helper.sh index 7039209..96f9238 100644 --- a/src/scripts/install-docker-credential-helper.sh +++ b/src/scripts/install-docker-credential-helper.sh @@ -6,8 +6,12 @@ expand_env_vars_with_prefix "PARAM_" HELPER_NAME="$PARAM_HELPER_NAME" -if uname | grep -q "Darwin"; then platform="darwin" -else platform="linux" +if uname | grep -q "Darwin"; then + platform="darwin" + arch="arm64" +else + platform="linux" + arch="$PARAM_ARCH" fi # Infer helper name from the platform @@ -75,11 +79,11 @@ minor_version="$(echo "$RELEASE_VERSION" | cut -d. -f2)" download_base_url="$base_url/download/${RELEASE_VERSION}/${HELPER_FILENAME}-${RELEASE_VERSION}" if [ "$minor_version" -gt 6 ]; then - DOWNLOAD_URL="$download_base_url.$platform-amd64" + DOWNLOAD_URL="$download_base_url.$platform-$arch" echo "Downloading from url: $DOWNLOAD_URL" curl -L -o "${HELPER_FILENAME}" "$DOWNLOAD_URL" else - DOWNLOAD_URL="$download_base_url-amd64.tar.gz" + DOWNLOAD_URL="$download_base_url-$arch.tar.gz" echo "Downloading from url: $DOWNLOAD_URL" curl -L -o "${HELPER_FILENAME}_archive" "$DOWNLOAD_URL" tar xvf "./${HELPER_FILENAME}_archive" diff --git a/test4.Dockerfile b/test4.Dockerfile new file mode 100644 index 0000000..5800b25 --- /dev/null +++ b/test4.Dockerfile @@ -0,0 +1,54 @@ +FROM ubuntu:22.04 + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive + +# Update and install common packages (same as base) +RUN apt-get update && apt-get install -y \ + python3.10 \ + python3-pip \ + curl \ + wget \ + git \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install common Python packages (same as base) +RUN pip3 install --no-cache-dir --upgrade pip && \ + pip3 install --no-cache-dir \ + numpy \ + pandas \ + scikit-learn \ + matplotlib + +# Install additional packages +RUN apt-get update && apt-get install -y \ + nodejs \ + npm \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install TensorFlow and Keras +RUN pip3 install --no-cache-dir tensorflow keras + +# Install PyTorch +RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + +# Install some Node.js packages globally +RUN npm install -g \ + express \ + lodash \ + moment + +# Create large files to simulate longer build time +RUN dd if=/dev/urandom of=/large_file_1 bs=1M count=500 && \ + dd if=/dev/urandom of=/large_file_2 bs=1M count=500 && \ + rm /large_file_1 /large_file_2 + +# Set working directory +WORKDIR /app + +# Create a simple Python script that uses one of the new packages +RUN echo 'import tensorflow as tf; print(f"TensorFlow version: {tf.__version__}")' > tf_version.py + +CMD ["python3", "tf_version.py"] \ No newline at end of file