From e9a99345ecbd3f2730a3649c635440599eaf53dc Mon Sep 17 00:00:00 2001 From: kruskal <99559985+kruskall@users.noreply.github.com> Date: Fri, 14 Feb 2025 01:27:15 +0100 Subject: [PATCH 01/12] feat: bump go version to 1.23 --- .go-version | 2 +- .golangci.yml | 12 ------------ CHANGELOG.next.asciidoc | 2 +- auditbeat/Dockerfile | 2 +- dev-tools/kubernetes/filebeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/heartbeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/metricbeat/Dockerfile.debug | 2 +- go.mod | 2 +- heartbeat/Dockerfile | 2 +- libbeat/docs/version.asciidoc | 2 +- metricbeat/Dockerfile | 2 +- metricbeat/module/http/_meta/Dockerfile | 2 +- metricbeat/module/nats/_meta/Dockerfile | 2 +- metricbeat/module/vsphere/_meta/Dockerfile | 2 +- packetbeat/Dockerfile | 2 +- x-pack/metricbeat/module/stan/_meta/Dockerfile | 2 +- 16 files changed, 15 insertions(+), 27 deletions(-) diff --git a/.go-version b/.go-version index 2560439f071b..d8c40e539ce6 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.22.12 +1.23.6 diff --git a/.golangci.yml b/.golangci.yml index 3dcc780986f1..d84c748a94dc 100755 --- a/.golangci.yml +++ b/.golangci.yml @@ -159,10 +159,6 @@ linters-settings: - github.com/meraki/dashboard-api-go/v3 - github.com/snowflakedb/gosnowflake - gosimple: - # Select the Go version to target. The default is '1.13'. - go: "1.22.12" - nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 max-func-lines: 0 @@ -178,21 +174,13 @@ linters-settings: require-specific: false staticcheck: - # Select the Go version to target. The default is '1.13'. - go: "1.22.12" checks: ["all"] stylecheck: - # Select the Go version to target. The default is '1.13'. - go: "1.22.12" # Disabled: # ST1005: error strings should not be capitalized checks: ["all", "-ST1005"] - unused: - # Select the Go version to target. The default is '1.13'. - go: "1.22.12" - gosec: excludes: - G306 # Expect WriteFile permissions to be 0600 or less diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3eac8fe94637..d74234a08ade 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -295,7 +295,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - The environment variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS` overrides configured/default `add_cloud_metadata` providers {pull}38669[38669] - When running under Elastic-Agent Kafka output allows dynamic topic in `topic` field {pull}40415[40415] - The script processor has a new configuration option that only uses the cached javascript sessions and prevents the creation of new javascript sessions. -- Update to Go 1.22.12. {pull}42681[42681] +- Update to Go 1.23.6. {pull}42681[42681] - Replace Ubuntu 20.04 with 24.04 for Docker base images {issue}40743[40743] {pull}40942[40942] - Reduce memory consumption of k8s autodiscovery and the add_kubernetes_metadata processor when Deployment metadata is enabled - Add `lowercase` processor. {issue}22254[22254] {pull}41424[41424] diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 3401b07804a9..9e872194fa7a 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.12 +FROM golang:1.23.6 RUN \ apt-get update \ diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug index 51f66c3b4329..d9a277fed271 100644 --- a/dev-tools/kubernetes/filebeat/Dockerfile.debug +++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.22.12 as builder +FROM golang:1.23.6 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/heartbeat/Dockerfile.debug b/dev-tools/kubernetes/heartbeat/Dockerfile.debug index f5ce33be30a1..242ef82b4d60 100644 --- a/dev-tools/kubernetes/heartbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/heartbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.22.12 as builder +FROM golang:1.23.6 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug index f7e1f080068d..7ac6de957f8a 100644 --- a/dev-tools/kubernetes/metricbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.22.12 as builder +FROM golang:1.23.6 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/go.mod b/go.mod index f295ab111110..b1f87d19e549 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.22.12 +go 1.23.6 require ( cloud.google.com/go/bigquery v1.65.0 diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index 19cca621a04b..095925fdede0 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.12 +FROM golang:1.23.6 RUN \ apt-get update \ diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index d374efed893f..4b5861cee137 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 9.1.0 :doc-branch: main -:go-version: 1.22.10 +:go-version: 1.23.6 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index f2c12cf7083b..f5d474cd1ac3 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.12 +FROM golang:1.23.6 COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ RUN \ diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 1e933b381919..22a59e9c2758 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.12 +FROM golang:1.23.6 COPY test/main.go main.go diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index eaff16e36c86..bace3e9c6b8b 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION # build stage -FROM golang:1.22.12 AS build-env +FROM golang:1.23.6 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/nats.go.git /nats-go RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build . diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile index 2fd564cfebec..6f67b4e5011d 100644 --- a/metricbeat/module/vsphere/_meta/Dockerfile +++ b/metricbeat/module/vsphere/_meta/Dockerfile @@ -1,5 +1,5 @@ ARG VSPHERE_GOLANG_VERSION -FROM golang:1.22.12 +FROM golang:1.23.6 RUN apt-get install curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index f1967a08a609..e904ade2ba1c 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.12 +FROM golang:1.23.6 RUN \ apt-get update \ diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile index f4be8577fce5..e8d13f72d7a6 100644 --- a/x-pack/metricbeat/module/stan/_meta/Dockerfile +++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1 FROM nats-streaming:$STAN_VERSION # build stage -FROM golang:1.22.12 AS build-env +FROM golang:1.23.6 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/stan.go.git /stan-go RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build . From b71ef7b9785528443b88e6c736b020437257a484 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Fri, 14 Feb 2025 01:28:50 +0100 Subject: [PATCH 02/12] Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d74234a08ade..28564073d886 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -295,7 +295,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - The environment variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS` overrides configured/default `add_cloud_metadata` providers {pull}38669[38669] - When running under Elastic-Agent Kafka output allows dynamic topic in `topic` field {pull}40415[40415] - The script processor has a new configuration option that only uses the cached javascript sessions and prevents the creation of new javascript sessions. -- Update to Go 1.23.6. {pull}42681[42681] +- Update to Go 1.23.6. {pull}42705[42705] - Replace Ubuntu 20.04 with 24.04 for Docker base images {issue}40743[40743] {pull}40942[40942] - Reduce memory consumption of k8s autodiscovery and the add_kubernetes_metadata processor when Deployment metadata is enabled - Add `lowercase` processor. {issue}22254[22254] {pull}41424[41424] From f501c58554877ccfee9d39b6faab0f1a98f508da Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Tue, 25 Feb 2025 17:43:08 +0100 Subject: [PATCH 03/12] Update packaging.sh --- .buildkite/scripts/packaging/packaging.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/packaging/packaging.sh b/.buildkite/scripts/packaging/packaging.sh index 43d3ae19bfa5..1f84260fd94a 100755 --- a/.buildkite/scripts/packaging/packaging.sh +++ b/.buildkite/scripts/packaging/packaging.sh @@ -12,8 +12,10 @@ set -ueo pipefail BEAT_DIR=${1:?-"Error: Beat directory must be specified."} -#Use newer multiarch support for packaging -docker run --privileged --rm tonistiigi/binfmt:master --install all +BINFMT_IMAGE="tonistiigi/binfmt:qemu-v8.1.5" +# Make sure to uninstall first to avoid conflicts +docker run --privileged --rm "$BINFMT_IMAGE" --uninstall qemu-* +docker run --privileged --rm "$BINFMT_IMAGE" --install all cd $BEAT_DIR mage package From 747f315a3b485f520d7b7dd5d0212e4ce8ac188a Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Tue, 25 Feb 2025 18:19:29 +0100 Subject: [PATCH 04/12] Update packaging.sh --- .buildkite/scripts/packaging/packaging.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/packaging/packaging.sh b/.buildkite/scripts/packaging/packaging.sh index 1f84260fd94a..ddf1932847fd 100755 --- a/.buildkite/scripts/packaging/packaging.sh +++ b/.buildkite/scripts/packaging/packaging.sh @@ -12,6 +12,8 @@ set -ueo pipefail BEAT_DIR=${1:?-"Error: Beat directory must be specified."} +# NOTE: Pinned to QEMU v8.x since v9.x breaks compilation of amd64 binaries in arm64. +# See: https://gitlab.com/qemu-project/qemu/-/issues/2560 BINFMT_IMAGE="tonistiigi/binfmt:qemu-v8.1.5" # Make sure to uninstall first to avoid conflicts docker run --privileged --rm "$BINFMT_IMAGE" --uninstall qemu-* From def7f306fc35a0d0d6ca168e27af99bd3ef378d1 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 28 Feb 2025 11:32:22 +0100 Subject: [PATCH 05/12] test https://github.com/elastic/golang-crossbuild/pull/527 --- dev-tools/mage/settings.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 22c081351e8b..9ede5804d289 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -41,9 +41,9 @@ const ( fpmVersion = "1.13.1" // Docker images. See https://github.com/elastic/golang-crossbuild. - beatsFPMImage = "docker.elastic.co/beats-dev/fpm" + beatsFPMImage = "docker.elastic.co/observability-ci/fpm" // BeatsCrossBuildImage is the image used for crossbuilding Beats. - BeatsCrossBuildImage = "docker.elastic.co/beats-dev/golang-crossbuild" + BeatsCrossBuildImage = "docker.elastic.co/observability-ci/golang-crossbuild" elasticBeatsImportPath = "github.com/elastic/beats" From fa109dcf2c354c105d9490d3a5dd14fe41194ec2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 28 Feb 2025 12:05:51 +0100 Subject: [PATCH 06/12] use the aarch image instead of the alias --- dev-tools/mage/crossbuild.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index ede35e08d8a3..0aabf0032118 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -227,7 +227,7 @@ func CrossBuildImage(platform string) (string, error) { case platform == "darwin/universal": tagSuffix = "darwin-arm64-debian11" case platform == "linux/arm64": - tagSuffix = "arm" + tagSuffix = "base-arm-debian9" case platform == "linux/armv5": tagSuffix = "armel" case platform == "linux/armv6": From 9b5e2df663664e321f817cd1ac7b05f26a773cbf Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 28 Feb 2025 12:08:00 +0100 Subject: [PATCH 07/12] ci: speed up --- .buildkite/auditbeat/auditbeat-pipeline.yml | 407 -------------------- 1 file changed, 407 deletions(-) diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 370ca6096bba..35eef1bc8e58 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -38,413 +38,6 @@ env: CONCURRENCY_METHOD: eager steps: - - group: "Check/Update" - key: "auditbeat-check-update" - - steps: - - label: "Auditbeat: Run check/update" - command: | - set -eo pipefail - make -C auditbeat check update - make check-no-changes - retry: - automatic: - - limit: 1 - agents: - image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" - cpu: "4000m" - memory: "8Gi" - useCustomGlobalHooks: true - notify: - - github_commit_status: - context: "auditbeat: check/update" - - - label: "Auditbeat: Run pre-commit" - command: "pre-commit run --all-files" - agents: - image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" - memory: "2Gi" - useCustomGlobalHooks: true - notify: - - github_commit_status: - context: "auditbeat: pre-commit" - - - wait: ~ - # with PRs, we want to run mandatory tests only if check/update step succeed - # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests - # this allows building DRA artifacts even if there is flakiness in check/update step - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: "auditbeat-check-update" - - - group: "Mandatory Testing" - key: "auditbeat-mandatory-tests" - - steps: - - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Unit Tests" - command: | - set -euo pipefail - cd auditbeat - mage build unitTest - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_2204_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: Ubuntu x86_64 Unit Tests" - - - label: ":rhel: Auditbeat: RHEL9 Unit Tests" - command: | - set -euo pipefail - cd auditbeat - mage build unitTest - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "${IMAGE_RHEL9}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: RHEL9 Unit Tests" - - - label: ":windows: Auditbeat: Win 2016 Unit Tests" - command: | - Set-Location -Path auditbeat - mage build unitTest - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: Win 2016 Unit Tests" - - - label: ":windows: Auditbeat: Win 2022 Unit Tests" - command: | - Set-Location -Path auditbeat - mage build unitTest - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: Win 2022 Unit Tests" - - - label: ":ubuntu: Auditbeat: Crosscompile" - command: | - make -C auditbeat crosscompile - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_2204_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - notify: - - github_commit_status: - context: "auditbeat: Crosscompile" - - - group: "Extended Tests" - key: "auditbeat-extended-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|integrations).*/ - - steps: - - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Integration Tests -- {{matrix.image}}" - key: "auditbeat-extended-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ - command: | - set -euo pipefail - cd auditbeat - mage build integTest - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "{{matrix.image}}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: Ubuntu x86_64 Integration Tests" - matrix: - setup: - image: - - "${IMAGE_UBUNTU_2004_X86_64}" - - "${IMAGE_UBUNTU_2204_X86_64}" - - "${IMAGE_UBUNTU_2404_X86_64}" - - - label: ":ubuntu: Auditbeat: Ubuntu arm64 Integration Tests -- {{matrix.image}}" - key: "auditbeat-extended-arm64-integ-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ - command: | - set -euo pipefail - cd auditbeat - mage build integTest - retry: - automatic: - - limit: 1 - agents: - provider: "aws" - imagePrefix: "{{matrix.image}}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: Ubuntu arm64 Integration Tests" - matrix: - setup: - image: - - "${IMAGE_UBUNTU_2004_ARM64}" - - "${IMAGE_UBUNTU_2204_ARM64}" - - "${IMAGE_UBUNTU_2404_ARM64}" - - - label: ":ubuntu: Auditbeat: Ubuntu arm64 Unit Tests" - key: "auditbeat-extended-arm64-unit-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: | - set -euo pipefail - cd auditbeat - mage build unitTest - retry: - automatic: - - limit: 1 - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_2204_ARM64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: Ubuntu arm64 Unit Tests" - - - label: ":mac: Auditbeat: macOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd auditbeat - mage build unitTest - concurrency_group: "${CONCURRENCY_GROUP}" - concurrency: "${CONCURRENCY_COUNT}" - concurrency_method: "${CONCURRENCY_METHOD}" - retry: - automatic: - - limit: 3 # using higher retries for now due to lack of custom vm images and vendor instability - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: macOS x86_64 Unit Tests" - - - label: ":mac: Auditbeat: macOS arm64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd auditbeat - mage build unitTest - concurrency_group: "${CONCURRENCY_GROUP}" - concurrency: "${CONCURRENCY_COUNT}" - concurrency_method: "${CONCURRENCY_METHOD}" - retry: - automatic: - - limit: 3 # using higher retries for now due to lack of custom vm images and vendor instability - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: macOS arm64 Unit Tests" - - - group: "Extended Windows Tests" - key: "auditbeat-extended-tests-win" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ - - steps: - - label: ":windows: Auditbeat: Win 2019 Unit Tests" - key: "auditbeat-extended-win-2019-unit-tests" - command: | - Set-Location -Path auditbeat - mage build unitTest - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: Win 2019 Unit Tests" - - - label: ":windows: Auditbeat: Win 10 Unit Tests" - key: "auditbeat-extended-win-10-unit-tests" - command: | - Set-Location -Path auditbeat - mage build unitTest - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: Win 10 Unit Tests" - - - label: ":windows: Auditbeat: Win 11 Unit Tests" - key: "auditbeat-extended-win-11-unit-tests" - command: | - Set-Location -Path auditbeat - mage build unitTest - retry: - automatic: - - limit: 1 - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - plugins: - - test-collector#v1.10.2: - files: "auditbeat/build/TEST-*.xml" - format: "junit" - branches: "main" - debug: true - notify: - - github_commit_status: - context: "auditbeat: Win 11 Unit Tests" - - - wait: ~ - # with PRs, we want to run packaging only if mandatory tests succeed - # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests - # this allows building DRA artifacts even if there is flakiness in mandatory tests - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: "auditbeat-mandatory-tests" - - group: "Packaging" key: "auditbeat-packaging" From 45cbfd3b78ddc85520a00df41762bac188eb3868 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 28 Feb 2025 12:16:33 +0100 Subject: [PATCH 08/12] remove qemu for now use the native support instead --- .buildkite/scripts/packaging/packaging.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.buildkite/scripts/packaging/packaging.sh b/.buildkite/scripts/packaging/packaging.sh index ddf1932847fd..28153c129820 100755 --- a/.buildkite/scripts/packaging/packaging.sh +++ b/.buildkite/scripts/packaging/packaging.sh @@ -9,15 +9,7 @@ set -ueo pipefail - BEAT_DIR=${1:?-"Error: Beat directory must be specified."} -# NOTE: Pinned to QEMU v8.x since v9.x breaks compilation of amd64 binaries in arm64. -# See: https://gitlab.com/qemu-project/qemu/-/issues/2560 -BINFMT_IMAGE="tonistiigi/binfmt:qemu-v8.1.5" -# Make sure to uninstall first to avoid conflicts -docker run --privileged --rm "$BINFMT_IMAGE" --uninstall qemu-* -docker run --privileged --rm "$BINFMT_IMAGE" --install all - cd $BEAT_DIR mage package From 81b304e67e65d3df258c26f76e8e540b95ed160e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 28 Feb 2025 12:20:17 +0100 Subject: [PATCH 09/12] Revert "remove qemu for now" This reverts commit 45cbfd3b78ddc85520a00df41762bac188eb3868. --- .buildkite/scripts/packaging/packaging.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.buildkite/scripts/packaging/packaging.sh b/.buildkite/scripts/packaging/packaging.sh index 28153c129820..ddf1932847fd 100755 --- a/.buildkite/scripts/packaging/packaging.sh +++ b/.buildkite/scripts/packaging/packaging.sh @@ -9,7 +9,15 @@ set -ueo pipefail + BEAT_DIR=${1:?-"Error: Beat directory must be specified."} +# NOTE: Pinned to QEMU v8.x since v9.x breaks compilation of amd64 binaries in arm64. +# See: https://gitlab.com/qemu-project/qemu/-/issues/2560 +BINFMT_IMAGE="tonistiigi/binfmt:qemu-v8.1.5" +# Make sure to uninstall first to avoid conflicts +docker run --privileged --rm "$BINFMT_IMAGE" --uninstall qemu-* +docker run --privileged --rm "$BINFMT_IMAGE" --install all + cd $BEAT_DIR mage package From a40e52552777d4f1b272a279eedddaf7357ab180 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 28 Feb 2025 12:29:26 +0100 Subject: [PATCH 10/12] we don't use qemu on ARM64 --- .buildkite/scripts/packaging/packaging.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/packaging/packaging.sh b/.buildkite/scripts/packaging/packaging.sh index ddf1932847fd..98258068cee1 100755 --- a/.buildkite/scripts/packaging/packaging.sh +++ b/.buildkite/scripts/packaging/packaging.sh @@ -3,21 +3,24 @@ # Centralise the mage package for a given beat in Buildkite. # It enables multi-arch builds to avoid the exec format errors when # attempting to build arm64 inside arm64 workers. +# # For further details, see https://github.com/elastic/elastic-agent/pull/6948 # and https://github.com/elastic/golang-crossbuild/pull/507 # set -ueo pipefail - BEAT_DIR=${1:?-"Error: Beat directory must be specified."} -# NOTE: Pinned to QEMU v8.x since v9.x breaks compilation of amd64 binaries in arm64. -# See: https://gitlab.com/qemu-project/qemu/-/issues/2560 -BINFMT_IMAGE="tonistiigi/binfmt:qemu-v8.1.5" -# Make sure to uninstall first to avoid conflicts -docker run --privileged --rm "$BINFMT_IMAGE" --uninstall qemu-* -docker run --privileged --rm "$BINFMT_IMAGE" --install all +if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then + echo "Skipping qemu installation on arm64 worker" +else + BINFMT_IMAGE="tonistiigi/binfmt:qemu-v9.2.2" + + # Make sure to uninstall first to avoid conflicts + docker run --privileged --rm "$BINFMT_IMAGE" --uninstall qemu-* + docker run --privileged --rm "$BINFMT_IMAGE" --install all +fi cd $BEAT_DIR mage package From aa3466dc06b7a338175a65bdaf7122e5026578e7 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 28 Feb 2025 13:05:32 +0100 Subject: [PATCH 11/12] Revert "ci: speed up" This reverts commit 9b5e2df663664e321f817cd1ac7b05f26a773cbf. --- .buildkite/auditbeat/auditbeat-pipeline.yml | 407 ++++++++++++++++++++ 1 file changed, 407 insertions(+) diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 35eef1bc8e58..370ca6096bba 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -38,6 +38,413 @@ env: CONCURRENCY_METHOD: eager steps: + - group: "Check/Update" + key: "auditbeat-check-update" + + steps: + - label: "Auditbeat: Run check/update" + command: | + set -eo pipefail + make -C auditbeat check update + make check-no-changes + retry: + automatic: + - limit: 1 + agents: + image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "auditbeat: check/update" + + - label: "Auditbeat: Run pre-commit" + command: "pre-commit run --all-files" + agents: + image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}" + memory: "2Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "auditbeat: pre-commit" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "auditbeat-check-update" + + - group: "Mandatory Testing" + key: "auditbeat-mandatory-tests" + + steps: + - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Unit Tests" + command: | + set -euo pipefail + cd auditbeat + mage build unitTest + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_2204_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: Ubuntu x86_64 Unit Tests" + + - label: ":rhel: Auditbeat: RHEL9 Unit Tests" + command: | + set -euo pipefail + cd auditbeat + mage build unitTest + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: RHEL9 Unit Tests" + + - label: ":windows: Auditbeat: Win 2016 Unit Tests" + command: | + Set-Location -Path auditbeat + mage build unitTest + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: Win 2016 Unit Tests" + + - label: ":windows: Auditbeat: Win 2022 Unit Tests" + command: | + Set-Location -Path auditbeat + mage build unitTest + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: Win 2022 Unit Tests" + + - label: ":ubuntu: Auditbeat: Crosscompile" + command: | + make -C auditbeat crosscompile + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_2204_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + notify: + - github_commit_status: + context: "auditbeat: Crosscompile" + + - group: "Extended Tests" + key: "auditbeat-extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|integrations).*/ + + steps: + - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Integration Tests -- {{matrix.image}}" + key: "auditbeat-extended-integ-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + command: | + set -euo pipefail + cd auditbeat + mage build integTest + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: Ubuntu x86_64 Integration Tests" + matrix: + setup: + image: + - "${IMAGE_UBUNTU_2004_X86_64}" + - "${IMAGE_UBUNTU_2204_X86_64}" + - "${IMAGE_UBUNTU_2404_X86_64}" + + - label: ":ubuntu: Auditbeat: Ubuntu arm64 Integration Tests -- {{matrix.image}}" + key: "auditbeat-extended-arm64-integ-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ + command: | + set -euo pipefail + cd auditbeat + mage build integTest + retry: + automatic: + - limit: 1 + agents: + provider: "aws" + imagePrefix: "{{matrix.image}}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: Ubuntu arm64 Integration Tests" + matrix: + setup: + image: + - "${IMAGE_UBUNTU_2004_ARM64}" + - "${IMAGE_UBUNTU_2204_ARM64}" + - "${IMAGE_UBUNTU_2404_ARM64}" + + - label: ":ubuntu: Auditbeat: Ubuntu arm64 Unit Tests" + key: "auditbeat-extended-arm64-unit-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: | + set -euo pipefail + cd auditbeat + mage build unitTest + retry: + automatic: + - limit: 1 + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_2204_ARM64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: Ubuntu arm64 Unit Tests" + + - label: ":mac: Auditbeat: macOS x86_64 Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd auditbeat + mage build unitTest + concurrency_group: "${CONCURRENCY_GROUP}" + concurrency: "${CONCURRENCY_COUNT}" + concurrency_method: "${CONCURRENCY_METHOD}" + retry: + automatic: + - limit: 3 # using higher retries for now due to lack of custom vm images and vendor instability + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: macOS x86_64 Unit Tests" + + - label: ":mac: Auditbeat: macOS arm64 Unit Tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd auditbeat + mage build unitTest + concurrency_group: "${CONCURRENCY_GROUP}" + concurrency: "${CONCURRENCY_COUNT}" + concurrency_method: "${CONCURRENCY_METHOD}" + retry: + automatic: + - limit: 3 # using higher retries for now due to lack of custom vm images and vendor instability + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: macOS arm64 Unit Tests" + + - group: "Extended Windows Tests" + key: "auditbeat-extended-tests-win" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + + steps: + - label: ":windows: Auditbeat: Win 2019 Unit Tests" + key: "auditbeat-extended-win-2019-unit-tests" + command: | + Set-Location -Path auditbeat + mage build unitTest + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: Win 2019 Unit Tests" + + - label: ":windows: Auditbeat: Win 10 Unit Tests" + key: "auditbeat-extended-win-10-unit-tests" + command: | + Set-Location -Path auditbeat + mage build unitTest + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: Win 10 Unit Tests" + + - label: ":windows: Auditbeat: Win 11 Unit Tests" + key: "auditbeat-extended-win-11-unit-tests" + command: | + Set-Location -Path auditbeat + mage build unitTest + retry: + automatic: + - limit: 1 + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "auditbeat/build/*.xml" + - "auditbeat/build/*.json" + plugins: + - test-collector#v1.10.2: + files: "auditbeat/build/TEST-*.xml" + format: "junit" + branches: "main" + debug: true + notify: + - github_commit_status: + context: "auditbeat: Win 11 Unit Tests" + + - wait: ~ + # with PRs, we want to run packaging only if mandatory tests succeed + # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in mandatory tests + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "auditbeat-mandatory-tests" + - group: "Packaging" key: "auditbeat-packaging" From e1e84f9873ad6a80548ae8d8c33e3b4e35e8c452 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 28 Feb 2025 13:03:34 +0100 Subject: [PATCH 12/12] fix packaging --- .buildkite/metricbeat/pipeline.yml | 4 +++- .buildkite/scripts/packaging/packaging.sh | 16 ++-------------- .buildkite/scripts/qemu.sh | 19 +++++++++++++++++++ .../x-pack/pipeline.xpack.dockerlogbeat.yml | 1 + 4 files changed, 25 insertions(+), 15 deletions(-) create mode 100755 .buildkite/scripts/qemu.sh diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 7082869ba665..8be300f55806 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -164,7 +164,9 @@ steps: - label: ":ubuntu: Metricbeat: Crosscompile" key: "mandatory-cross-compile" - command: "make -C metricbeat crosscompile" + command: | + source .buildkite/scripts/qemu.sh + make -C metricbeat crosscompile retry: automatic: - limit: 1 diff --git a/.buildkite/scripts/packaging/packaging.sh b/.buildkite/scripts/packaging/packaging.sh index 98258068cee1..aeaa97db0388 100755 --- a/.buildkite/scripts/packaging/packaging.sh +++ b/.buildkite/scripts/packaging/packaging.sh @@ -1,26 +1,14 @@ #!/usr/bin/env bash # # Centralise the mage package for a given beat in Buildkite. -# It enables multi-arch builds to avoid the exec format errors when -# attempting to build arm64 inside arm64 workers. -# -# For further details, see https://github.com/elastic/elastic-agent/pull/6948 -# and https://github.com/elastic/golang-crossbuild/pull/507 # set -ueo pipefail BEAT_DIR=${1:?-"Error: Beat directory must be specified."} -if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then - echo "Skipping qemu installation on arm64 worker" -else - BINFMT_IMAGE="tonistiigi/binfmt:qemu-v9.2.2" - - # Make sure to uninstall first to avoid conflicts - docker run --privileged --rm "$BINFMT_IMAGE" --uninstall qemu-* - docker run --privileged --rm "$BINFMT_IMAGE" --install all -fi +# shellcheck source=/dev/null +source .buildkite/scripts/qemu.sh cd $BEAT_DIR mage package diff --git a/.buildkite/scripts/qemu.sh b/.buildkite/scripts/qemu.sh new file mode 100755 index 000000000000..65488931d2e1 --- /dev/null +++ b/.buildkite/scripts/qemu.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# It enables multi-arch builds to avoid the exec format errors when +# attempting to build arm64 inside arm64 workers. +# +# For further details, see https://github.com/elastic/elastic-agent/pull/6948 +# and https://github.com/elastic/golang-crossbuild/pull/507 +# +set -euo pipefail + +if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then + echo "Skipping qemu installation on arm64 worker" +else + BINFMT_IMAGE="tonistiigi/binfmt:qemu-v9.2.2" + + # Make sure to uninstall first to avoid conflicts + docker run --privileged --rm "$BINFMT_IMAGE" --uninstall qemu-* + docker run --privileged --rm "$BINFMT_IMAGE" --install all +fi diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 85b8634b90f0..4f74f99ae0b6 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -64,6 +64,7 @@ steps: - label: ":ubuntu: x-pack/dockerlogbeat: Ubuntu x86_64 Unit Tests" key: "mandatory-linux-unit-test" command: | + source .buildkite/scripts/qemu.sh cd x-pack/dockerlogbeat mage build unitTest retry: