From 566348107a829f3cf7b3601ff77dc255a5ac9751 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 26 Feb 2025 12:51:58 -0500 Subject: [PATCH 01/14] fix(go/base): set GOTOOLCHAIN=local Disable the automatic downloads of the Go toolchain. For the goal of reproducible builds using golang-crossbuild, we want all of the binaries used to produce our artifacts to be self-contained in the image. Adding GOTOOLCHAIN=local disable downloads of newer toolchains. This changes the entrypoint of the images to document that GOTOOLCHAIN is set during builds. And it sets GOTOOLCHAIN=local in the Dockerfile so that when building entrypoint.go it uses the toolchain that was previously installed. The two Dockerfiles (base and base-arm) had different 'go build' commands. This unifies them to both have CGO_ENABLED=0 and to explicitly set GOARCH to avoid accidentally producing a binary for the wrong target architecture (which we've seen happen for unknown reasons). https://go.dev/doc/toolchain --- go/base-arm/Dockerfile.tmpl | 3 ++- go/base-arm/rootfs/entrypoint.go | 5 +++-- go/base/Dockerfile.tmpl | 3 ++- go/base/rootfs/entrypoint.go | 5 +++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/go/base-arm/Dockerfile.tmpl b/go/base-arm/Dockerfile.tmpl index 97375004..81b7b2d5 100644 --- a/go/base-arm/Dockerfile.tmpl +++ b/go/base-arm/Dockerfile.tmpl @@ -69,11 +69,12 @@ RUN ldd --version WORKDIR / RUN mkdir -p /root/.config/go/telemetry && echo "off 2024-08-23" > /root/.config/go/telemetry/mode +ENV GOTOOLCHAIN local RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION-arm \ && go get . \ && go env \ && echo "toolcompile=$(go tool compile -V)" \ - && go build -o /crossbuild /entrypoint.go \ + && CGO_ENABLED=0 GOARCH=arm64 go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go COPY sdks/libpcap-1.8.1.tar.gz . diff --git a/go/base-arm/rootfs/entrypoint.go b/go/base-arm/rootfs/entrypoint.go index 0e5c2985..159fcb11 100644 --- a/go/base-arm/rootfs/entrypoint.go +++ b/go/base-arm/rootfs/entrypoint.go @@ -25,8 +25,8 @@ by mounting the project inside of a container equipped with cross-compilers. The root of your project's repo should be mounted at the appropriate location on the GOPATH which is set to /go. -While executing the build command the following variables with be added to the -environment: GOOS, GOARCH, GOARM, PLATFORM_ID, CC, and CXX. +While executing the build command the following variables will be added to the +environment: GOOS, GOARCH, GOARM, GOTOOLCHAIN=local, PLATFORM_ID, CC, and CXX. `, RunE: doBuild, SilenceUsage: true, @@ -104,6 +104,7 @@ func buildEnvironment(platform string) (map[string]string, error) { "GOOS": goos, "GOARCH": goarch, "GOARM": goarm, + "GOTOOLCHAIN": "local", // Disable automatic downloads of toolchains for reproducible builds. "PLATFORM_ID": platformID, } diff --git a/go/base/Dockerfile.tmpl b/go/base/Dockerfile.tmpl index bb4acab6..c8a63fda 100644 --- a/go/base/Dockerfile.tmpl +++ b/go/base/Dockerfile.tmpl @@ -62,11 +62,12 @@ RUN ldd --version WORKDIR / RUN mkdir -p /root/.config/go/telemetry && echo "off 2024-08-23" > /root/.config/go/telemetry/mode +ENV GOTOOLCHAIN=local RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION \ && go get . \ && go env \ && echo "toolcompile=$(go tool compile -V)" \ - && CGO_ENABLED=0 go build -o /crossbuild /entrypoint.go \ + && CGO_ENABLED=0 GOARCH=amd64 go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go COPY sdks/libpcap-1.8.1.tar.gz . diff --git a/go/base/rootfs/entrypoint.go b/go/base/rootfs/entrypoint.go index 0e5c2985..159fcb11 100644 --- a/go/base/rootfs/entrypoint.go +++ b/go/base/rootfs/entrypoint.go @@ -25,8 +25,8 @@ by mounting the project inside of a container equipped with cross-compilers. The root of your project's repo should be mounted at the appropriate location on the GOPATH which is set to /go. -While executing the build command the following variables with be added to the -environment: GOOS, GOARCH, GOARM, PLATFORM_ID, CC, and CXX. +While executing the build command the following variables will be added to the +environment: GOOS, GOARCH, GOARM, GOTOOLCHAIN=local, PLATFORM_ID, CC, and CXX. `, RunE: doBuild, SilenceUsage: true, @@ -104,6 +104,7 @@ func buildEnvironment(platform string) (map[string]string, error) { "GOOS": goos, "GOARCH": goarch, "GOARM": goarm, + "GOTOOLCHAIN": "local", // Disable automatic downloads of toolchains for reproducible builds. "PLATFORM_ID": platformID, } From f06635b24da37900b1c48d03ef38211f915243d5 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 27 Feb 2025 17:51:10 +0200 Subject: [PATCH 02/14] Don't build arm64 images on amd64 workers --- .buildkite/pipeline.yml | 15 ++++++++++----- go/Makefile.debian10 | 14 +++++++++++--- go/Makefile.debian11 | 13 ++++++++++--- go/Makefile.debian12 | 13 ++++++++++--- go/Makefile.debian9 | 4 ++-- go/base-arm/Dockerfile.tmpl | 2 +- 6 files changed, 44 insertions(+), 17 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 393b1028..4d7998f8 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -42,24 +42,29 @@ steps: - "false" - "true" - - label: ":linux: Staging / Ubuntu ARM - Makefile.debian9 - fips: {{matrix.fips}}" - key: "build-ubuntu-arm" + - label: ":linux: Staging / Ubuntu ARM64 - {{matrix.makefile}} - fips: {{matrix.fips}}" + key: "build-ubuntu-arm64" command: - - ".buildkite/scripts/build.sh Makefile.debian9" - - ".buildkite/scripts/publish.sh Makefile.debian9" + - ".buildkite/scripts/build.sh {{matrix.makefile}}" + - ".buildkite/scripts/publish.sh {{matrix.makefile}}" env: REPOSITORY: "${STAGING_IMAGE}" FIPS: "{{matrix.fips}}" if: build.env("BUILDKITE_PULL_REQUEST") != "false" notify: - github_commit_status: - context: "Staging / Ubuntu ARM" + context: "Staging / Ubuntu {{matrix.makefile}}" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.large" matrix: setup: + makefile: + - "Makefile.debian9" + - "Makefile.debian10" + - "Makefile.debian11" + - "Makefile.debian12" fips: - "false" - "true" diff --git a/go/Makefile.debian10 b/go/Makefile.debian10 index 0ae5d8f4..cbe685c8 100644 --- a/go/Makefile.debian10 +++ b/go/Makefile.debian10 @@ -1,4 +1,5 @@ -IMAGES ?= base main darwin arm armhf darwin-arm64 npcap +IMAGES := base main darwin armhf darwin-arm64 npcap +ARM_IMAGES := base-arm DEBIAN_VERSION := 10 TAG_EXTENSION := -debian10 @@ -7,8 +8,15 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) -# Requires login at https://docker.elastic.co:7000/. +build-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) + +# Requires login at https://docker-auth.elastic.co push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -.PHONY: build push +push-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) + +.PHONY: build build-arm push push-arm + diff --git a/go/Makefile.debian11 b/go/Makefile.debian11 index 24da642c..e37f4a95 100644 --- a/go/Makefile.debian11 +++ b/go/Makefile.debian11 @@ -1,4 +1,5 @@ -IMAGES := base main darwin arm armhf darwin-arm64 npcap +IMAGES := base main darwin armhf darwin-arm64 npcap +ARM_IMAGES := base-arm DEBIAN_VERSION := 11 TAG_EXTENSION := -debian11 @@ -7,8 +8,14 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) -# Requires login at https://docker.elastic.co:7000/. +build-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) + +# Requires login at https://docker-auth.elastic.co push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -.PHONY: build push +push-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) + +.PHONY: build build-arm push push-arm diff --git a/go/Makefile.debian12 b/go/Makefile.debian12 index 04ecdcdf..5b0b7b50 100644 --- a/go/Makefile.debian12 +++ b/go/Makefile.debian12 @@ -1,4 +1,5 @@ -IMAGES := base main darwin arm armhf armel mips s390x darwin-arm64 npcap +IMAGES := base main darwin armhf armel mips s390x darwin-arm64 npcap +ARM_IMAGES := base-arm DEBIAN_VERSION := 12 TAG_EXTENSION := -debian12 @@ -7,8 +8,14 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) -# Requires login at https://docker.elastic.co:7000/. +build-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) + +# Requires login at https://docker-auth.elastic.co push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -.PHONY: build push +push-arm: + @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) + +.PHONY: build build-arm push push-arm diff --git a/go/Makefile.debian9 b/go/Makefile.debian9 index 2c4dee9f..b1493ea3 100644 --- a/go/Makefile.debian9 +++ b/go/Makefile.debian9 @@ -1,4 +1,4 @@ -IMAGES := base main darwin arm armhf npcap +IMAGES := base main darwin armhf npcap ARM_IMAGES := base-arm DEBIAN_VERSION := 9 TAG_EXTENSION := -debian9 @@ -11,7 +11,7 @@ build: build-arm: @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) -# Requires login at https://docker.elastic.co:7000/. +# Requires login at https://docker-auth.elastic.co push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) diff --git a/go/base-arm/Dockerfile.tmpl b/go/base-arm/Dockerfile.tmpl index 81b7b2d5..19b350b3 100644 --- a/go/base-arm/Dockerfile.tmpl +++ b/go/base-arm/Dockerfile.tmpl @@ -77,7 +77,7 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION-arm \ && CGO_ENABLED=0 GOARCH=arm64 go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -COPY sdks/libpcap-1.8.1.tar.gz . +COPY sdks/libpcap-1.8.1.tar.gz . RUN mkdir /libpcap \ && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ && cd /libpcap/libpcap-1.8.1 \ From 85cdd7e1d4753a1da236383610cd6e9c65741f69 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 28 Feb 2025 12:26:13 +0200 Subject: [PATCH 03/14] Revert "Don't build arm64 images on amd64 workers" This reverts commit f06635b24da37900b1c48d03ef38211f915243d5. --- .buildkite/pipeline.yml | 15 +++++---------- go/Makefile.debian10 | 14 +++----------- go/Makefile.debian11 | 13 +++---------- go/Makefile.debian12 | 13 +++---------- go/Makefile.debian9 | 4 ++-- go/base-arm/Dockerfile.tmpl | 2 +- 6 files changed, 17 insertions(+), 44 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 4d7998f8..393b1028 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -42,29 +42,24 @@ steps: - "false" - "true" - - label: ":linux: Staging / Ubuntu ARM64 - {{matrix.makefile}} - fips: {{matrix.fips}}" - key: "build-ubuntu-arm64" + - label: ":linux: Staging / Ubuntu ARM - Makefile.debian9 - fips: {{matrix.fips}}" + key: "build-ubuntu-arm" command: - - ".buildkite/scripts/build.sh {{matrix.makefile}}" - - ".buildkite/scripts/publish.sh {{matrix.makefile}}" + - ".buildkite/scripts/build.sh Makefile.debian9" + - ".buildkite/scripts/publish.sh Makefile.debian9" env: REPOSITORY: "${STAGING_IMAGE}" FIPS: "{{matrix.fips}}" if: build.env("BUILDKITE_PULL_REQUEST") != "false" notify: - github_commit_status: - context: "Staging / Ubuntu {{matrix.makefile}}" + context: "Staging / Ubuntu ARM" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.large" matrix: setup: - makefile: - - "Makefile.debian9" - - "Makefile.debian10" - - "Makefile.debian11" - - "Makefile.debian12" fips: - "false" - "true" diff --git a/go/Makefile.debian10 b/go/Makefile.debian10 index cbe685c8..0ae5d8f4 100644 --- a/go/Makefile.debian10 +++ b/go/Makefile.debian10 @@ -1,5 +1,4 @@ -IMAGES := base main darwin armhf darwin-arm64 npcap -ARM_IMAGES := base-arm +IMAGES ?= base main darwin arm armhf darwin-arm64 npcap DEBIAN_VERSION := 10 TAG_EXTENSION := -debian10 @@ -8,15 +7,8 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) -build-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) - -# Requires login at https://docker-auth.elastic.co +# Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -push-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) - -.PHONY: build build-arm push push-arm - +.PHONY: build push diff --git a/go/Makefile.debian11 b/go/Makefile.debian11 index e37f4a95..24da642c 100644 --- a/go/Makefile.debian11 +++ b/go/Makefile.debian11 @@ -1,5 +1,4 @@ -IMAGES := base main darwin armhf darwin-arm64 npcap -ARM_IMAGES := base-arm +IMAGES := base main darwin arm armhf darwin-arm64 npcap DEBIAN_VERSION := 11 TAG_EXTENSION := -debian11 @@ -8,14 +7,8 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) -build-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) - -# Requires login at https://docker-auth.elastic.co +# Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -push-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) - -.PHONY: build build-arm push push-arm +.PHONY: build push diff --git a/go/Makefile.debian12 b/go/Makefile.debian12 index 5b0b7b50..04ecdcdf 100644 --- a/go/Makefile.debian12 +++ b/go/Makefile.debian12 @@ -1,5 +1,4 @@ -IMAGES := base main darwin armhf armel mips s390x darwin-arm64 npcap -ARM_IMAGES := base-arm +IMAGES := base main darwin arm armhf armel mips s390x darwin-arm64 npcap DEBIAN_VERSION := 12 TAG_EXTENSION := -debian12 @@ -8,14 +7,8 @@ export DEBIAN_VERSION TAG_EXTENSION build: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) build || exit 1;) -build-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) - -# Requires login at https://docker-auth.elastic.co +# Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) -push-arm: - @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) push-arm || exit 1;) - -.PHONY: build build-arm push push-arm +.PHONY: build push diff --git a/go/Makefile.debian9 b/go/Makefile.debian9 index b1493ea3..2c4dee9f 100644 --- a/go/Makefile.debian9 +++ b/go/Makefile.debian9 @@ -1,4 +1,4 @@ -IMAGES := base main darwin armhf npcap +IMAGES := base main darwin arm armhf npcap ARM_IMAGES := base-arm DEBIAN_VERSION := 9 TAG_EXTENSION := -debian9 @@ -11,7 +11,7 @@ build: build-arm: @$(foreach var,$(ARM_IMAGES),$(MAKE) -C $(var) build-arm || exit 1;) -# Requires login at https://docker-auth.elastic.co +# Requires login at https://docker.elastic.co:7000/. push: @$(foreach var,$(IMAGES),$(MAKE) -C $(var) push || exit 1;) diff --git a/go/base-arm/Dockerfile.tmpl b/go/base-arm/Dockerfile.tmpl index 19b350b3..81b7b2d5 100644 --- a/go/base-arm/Dockerfile.tmpl +++ b/go/base-arm/Dockerfile.tmpl @@ -77,7 +77,7 @@ RUN go mod init github.com/elastic/golang-crossbuild-$GOLANG_VERSION-arm \ && CGO_ENABLED=0 GOARCH=arm64 go build -o /crossbuild /entrypoint.go \ && rm -rf /go/* /root/.cache/* /entrypoint.go -COPY sdks/libpcap-1.8.1.tar.gz . +COPY sdks/libpcap-1.8.1.tar.gz . RUN mkdir /libpcap \ && tar -xzf libpcap-1.8.1.tar.gz -C /libpcap \ && cd /libpcap/libpcap-1.8.1 \ From d875fcae85987c000bb893d54c3c30cd7b7ed975 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 28 Feb 2025 12:26:59 +0200 Subject: [PATCH 04/14] Don't build arm64 docker images on amd64 workers for Debian 9 --- go/Makefile.debian9 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/Makefile.debian9 b/go/Makefile.debian9 index 2c4dee9f..8c4e3b17 100644 --- a/go/Makefile.debian9 +++ b/go/Makefile.debian9 @@ -1,4 +1,4 @@ -IMAGES := base main darwin arm armhf npcap +IMAGES := base main darwin armhf npcap ARM_IMAGES := base-arm DEBIAN_VERSION := 9 TAG_EXTENSION := -debian9 From 7cae477295a5b08c674f05b821d040e16edc8df4 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 15:58:49 +0200 Subject: [PATCH 05/14] trigger fpm builds -- to be removed --- fpm/toberemoved.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 fpm/toberemoved.sh diff --git a/fpm/toberemoved.sh b/fpm/toberemoved.sh new file mode 100644 index 00000000..b5295be7 --- /dev/null +++ b/fpm/toberemoved.sh @@ -0,0 +1,2 @@ +# throwaway file used to trigger fpm build on PR +# should be removed before merging From a497c76686bcf8cc755206b3612f0a94d4f098fb Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 16:01:00 +0200 Subject: [PATCH 06/14] Revert "trigger fpm builds -- to be removed" This reverts commit 7cae477295a5b08c674f05b821d040e16edc8df4. --- fpm/toberemoved.sh | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 fpm/toberemoved.sh diff --git a/fpm/toberemoved.sh b/fpm/toberemoved.sh deleted file mode 100644 index b5295be7..00000000 --- a/fpm/toberemoved.sh +++ /dev/null @@ -1,2 +0,0 @@ -# throwaway file used to trigger fpm build on PR -# should be removed before merging From e665ff18181fc7bd9575378b1e14cab8a6675665 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 16:03:17 +0200 Subject: [PATCH 07/14] temporarily trigger fpm builds -- to be removed --- .buildkite/scripts/llvm-fpm/build.sh | 6 +++--- .buildkite/scripts/llvm-fpm/publish.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/llvm-fpm/build.sh b/.buildkite/scripts/llvm-fpm/build.sh index 69b101fe..24615d6d 100755 --- a/.buildkite/scripts/llvm-fpm/build.sh +++ b/.buildkite/scripts/llvm-fpm/build.sh @@ -8,9 +8,9 @@ makefile=${1} patterns=${2} docker_filter_ref=${3} -if ! are_files_changed "$patterns" ; then - exit 0 -fi +#if ! are_files_changed "$patterns" ; then +# exit 0 +#fi add_bin_path with_go "${GOLANG_VERSION}" diff --git a/.buildkite/scripts/llvm-fpm/publish.sh b/.buildkite/scripts/llvm-fpm/publish.sh index 371d324b..60c3ed6c 100755 --- a/.buildkite/scripts/llvm-fpm/publish.sh +++ b/.buildkite/scripts/llvm-fpm/publish.sh @@ -7,9 +7,9 @@ source .buildkite/scripts/common.sh makefile=${1} patterns=${2} -if ! are_files_changed "$patterns" ; then - exit 0 -fi +#if ! are_files_changed "$patterns" ; then +# exit 0 +#fi add_bin_path retry 3 make -C ${makefile} push From 4822107b18d21af224368477ef5cc1d584fe7c61 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 16:31:06 +0200 Subject: [PATCH 08/14] build arm64 image for fpm --- .buildkite/fpm-pipeline.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.buildkite/fpm-pipeline.yml b/.buildkite/fpm-pipeline.yml index 0a15324a..f9900f65 100644 --- a/.buildkite/fpm-pipeline.yml +++ b/.buildkite/fpm-pipeline.yml @@ -3,6 +3,7 @@ env: SETUP_GVM_VERSION: "v0.5.1" IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" DOCKER_REGISTRY: "docker.elastic.co" STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci" MAKEFILE: "fpm" @@ -25,3 +26,19 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":linux: Build FPM / Ubuntu ARM64" + key: "build-ubuntu-arm64" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/ + command: + - ".buildkite/scripts/llvm-fpm/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}" + - ".buildkite/scripts/llvm-fpm/publish.sh ${MAKEFILE} ${CHANGESET_FILE}" + notify: + - github_commit_status: + context: "Build FPM / Ubuntu arm64" + env: + REPOSITORY: "${STAGING_IMAGE}" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" From 817b0f27edae8bfd8f83463b61882af69576692a Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 16:31:22 +0200 Subject: [PATCH 09/14] attempt to fix debian 11 build failures --- .buildkite/scripts/buildx.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/buildx.sh b/.buildkite/scripts/buildx.sh index c2b76345..7388949e 100755 --- a/.buildkite/scripts/buildx.sh +++ b/.buildkite/scripts/buildx.sh @@ -6,7 +6,8 @@ BUILDPLATFORM=${BUILDPLATFORM:-"linux/amd64,linux/arm64"} BUILDER_NAME="multibuilder${RANDOM}" echo "Add support for multiarch" -docker run --privileged --rm tonistiigi/binfmt:master --install all +#docker run --privileged --rm tonistiigi/binfmt:master --install all +docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes docker buildx ls echo 'Create builder' From c6370b7323bbfa0536452a2a441b66adf1dda9cb Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 16:39:13 +0200 Subject: [PATCH 10/14] Revert "attempt to fix debian 11 build failures" This reverts commit 817b0f27edae8bfd8f83463b61882af69576692a. --- .buildkite/scripts/buildx.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/scripts/buildx.sh b/.buildkite/scripts/buildx.sh index 7388949e..c2b76345 100755 --- a/.buildkite/scripts/buildx.sh +++ b/.buildkite/scripts/buildx.sh @@ -6,8 +6,7 @@ BUILDPLATFORM=${BUILDPLATFORM:-"linux/amd64,linux/arm64"} BUILDER_NAME="multibuilder${RANDOM}" echo "Add support for multiarch" -#docker run --privileged --rm tonistiigi/binfmt:master --install all -docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes +docker run --privileged --rm tonistiigi/binfmt:master --install all docker buildx ls echo 'Create builder' From 76313af6759c53e363ba881419dffbb27afb36b1 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 17:51:06 +0200 Subject: [PATCH 11/14] Revert "temporarily trigger fpm builds -- to be removed" This reverts commit e665ff18181fc7bd9575378b1e14cab8a6675665. --- .buildkite/scripts/llvm-fpm/build.sh | 6 +++--- .buildkite/scripts/llvm-fpm/publish.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/llvm-fpm/build.sh b/.buildkite/scripts/llvm-fpm/build.sh index 24615d6d..69b101fe 100755 --- a/.buildkite/scripts/llvm-fpm/build.sh +++ b/.buildkite/scripts/llvm-fpm/build.sh @@ -8,9 +8,9 @@ makefile=${1} patterns=${2} docker_filter_ref=${3} -#if ! are_files_changed "$patterns" ; then -# exit 0 -#fi +if ! are_files_changed "$patterns" ; then + exit 0 +fi add_bin_path with_go "${GOLANG_VERSION}" diff --git a/.buildkite/scripts/llvm-fpm/publish.sh b/.buildkite/scripts/llvm-fpm/publish.sh index 60c3ed6c..371d324b 100755 --- a/.buildkite/scripts/llvm-fpm/publish.sh +++ b/.buildkite/scripts/llvm-fpm/publish.sh @@ -7,9 +7,9 @@ source .buildkite/scripts/common.sh makefile=${1} patterns=${2} -#if ! are_files_changed "$patterns" ; then -# exit 0 -#fi +if ! are_files_changed "$patterns" ; then + exit 0 +fi add_bin_path retry 3 make -C ${makefile} push From 12d53c24e87cdb20ad6d79eaec19d6745ae1a1e0 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 19:59:18 +0200 Subject: [PATCH 12/14] Revert "build arm64 image for fpm" This reverts commit 4822107b18d21af224368477ef5cc1d584fe7c61. --- .buildkite/fpm-pipeline.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.buildkite/fpm-pipeline.yml b/.buildkite/fpm-pipeline.yml index f9900f65..0a15324a 100644 --- a/.buildkite/fpm-pipeline.yml +++ b/.buildkite/fpm-pipeline.yml @@ -3,7 +3,6 @@ env: SETUP_GVM_VERSION: "v0.5.1" IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" DOCKER_REGISTRY: "docker.elastic.co" STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci" MAKEFILE: "fpm" @@ -26,19 +25,3 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - - - label: ":linux: Build FPM / Ubuntu ARM64" - key: "build-ubuntu-arm64" - if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/ - command: - - ".buildkite/scripts/llvm-fpm/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}" - - ".buildkite/scripts/llvm-fpm/publish.sh ${MAKEFILE} ${CHANGESET_FILE}" - notify: - - github_commit_status: - context: "Build FPM / Ubuntu arm64" - env: - REPOSITORY: "${STAGING_IMAGE}" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.large" From ea548d9901e74263c429e4cb16bdb5afc0dcbd0c Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 20:05:54 +0200 Subject: [PATCH 13/14] trigger fpm build -- to be removed --- .buildkite/scripts/llvm-fpm/build.sh | 6 +++--- .buildkite/scripts/llvm-fpm/publish.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/llvm-fpm/build.sh b/.buildkite/scripts/llvm-fpm/build.sh index 69b101fe..24615d6d 100755 --- a/.buildkite/scripts/llvm-fpm/build.sh +++ b/.buildkite/scripts/llvm-fpm/build.sh @@ -8,9 +8,9 @@ makefile=${1} patterns=${2} docker_filter_ref=${3} -if ! are_files_changed "$patterns" ; then - exit 0 -fi +#if ! are_files_changed "$patterns" ; then +# exit 0 +#fi add_bin_path with_go "${GOLANG_VERSION}" diff --git a/.buildkite/scripts/llvm-fpm/publish.sh b/.buildkite/scripts/llvm-fpm/publish.sh index 371d324b..60c3ed6c 100755 --- a/.buildkite/scripts/llvm-fpm/publish.sh +++ b/.buildkite/scripts/llvm-fpm/publish.sh @@ -7,9 +7,9 @@ source .buildkite/scripts/common.sh makefile=${1} patterns=${2} -if ! are_files_changed "$patterns" ; then - exit 0 -fi +#if ! are_files_changed "$patterns" ; then +# exit 0 +#fi add_bin_path retry 3 make -C ${makefile} push From 6426ed106a8061b1fbb3ac406307f148baa8cc31 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 3 Mar 2025 20:10:46 +0200 Subject: [PATCH 14/14] Revert "trigger fpm build -- to be removed" This reverts commit ea548d9901e74263c429e4cb16bdb5afc0dcbd0c. --- .buildkite/scripts/llvm-fpm/build.sh | 6 +++--- .buildkite/scripts/llvm-fpm/publish.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/llvm-fpm/build.sh b/.buildkite/scripts/llvm-fpm/build.sh index 24615d6d..69b101fe 100755 --- a/.buildkite/scripts/llvm-fpm/build.sh +++ b/.buildkite/scripts/llvm-fpm/build.sh @@ -8,9 +8,9 @@ makefile=${1} patterns=${2} docker_filter_ref=${3} -#if ! are_files_changed "$patterns" ; then -# exit 0 -#fi +if ! are_files_changed "$patterns" ; then + exit 0 +fi add_bin_path with_go "${GOLANG_VERSION}" diff --git a/.buildkite/scripts/llvm-fpm/publish.sh b/.buildkite/scripts/llvm-fpm/publish.sh index 60c3ed6c..371d324b 100755 --- a/.buildkite/scripts/llvm-fpm/publish.sh +++ b/.buildkite/scripts/llvm-fpm/publish.sh @@ -7,9 +7,9 @@ source .buildkite/scripts/common.sh makefile=${1} patterns=${2} -#if ! are_files_changed "$patterns" ; then -# exit 0 -#fi +if ! are_files_changed "$patterns" ; then + exit 0 +fi add_bin_path retry 3 make -C ${makefile} push