Merge branch 'main' of github.com:elastic/beats into queue-byte-limits

Author: faec

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -27,6 +27,33 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "Auditbeat Check/Update"
+    key: "auditbeat-check-update"
+
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C auditbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "auditbeat: check/update"
+      
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "auditbeat-check-update"
+
   - group: "Auditbeat Mandatory Testing"
     key: "auditbeat-mandatory-tests"
 

.buildkite/filebeat/filebeat-pipeline.yml

@@ -27,6 +27,32 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "Filebeat Check/Update"
+    key: "filebeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C filebeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "filebeat: check/update"
+      
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "filebeat-check-update"
+
   - group: "Filebeat Mandatory Tests"
     key: "filebeat-mandatory-tests"
     steps:

.buildkite/libbeat/pipeline.libbeat.yml

@@ -16,6 +16,32 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "libbeat Check/Update"
+    key: "libbeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C libbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "libbeat: check/update"
+      
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "libbeat-check-update"
+
   - group: "Mandatory Tests"
     key: "mandatory-tests"
     steps:

.buildkite/metricbeat/pipeline.yml

@@ -2,15 +2,15 @@
 name: "beats-metricbeat"
 
 env:
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
 
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
@@ -32,10 +32,13 @@ env:
 steps:
   - group: "Metricbeat Mandatory Tests"
     key: "metricbeat-mandatory-tests"
+
     steps:
-      - label: ":linux: Ubuntu Unit Tests"
+      - label: ":ubuntu: Metricbeat Unit Tests"
         key: "mandatory-linux-unit-test"
-        command: "cd metricbeat && mage build unitTest"
+        command: |
+          cd metricbeat
+          mage build unitTest
         retry:
           automatic:
             - limit: 3
@@ -48,27 +51,20 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Ubuntu Unit Tests"
+              context: "metricbeat: Linux x86_64 Unit Tests"
 
-      - label: ":go: Go Integration Tests"
+      - label: ":ubuntu: Metricbeat Go Integration Tests"
         key: "mandatory-int-test"
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Running tests"
-
-          # TODO move this section to base image / pre-command hook
-          echo "~~~ Installing kind"
-          asdf plugin add kind
-          asdf install kind $ASDF_KIND_VERSION
-
-          .buildkite/deploy/kubernetes/scripts/kind-setup.sh
 
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
-          cd metricbeat && mage goIntegTest
+          cd metricbeat
+          mage goIntegTest
         retry:
           automatic:
             - limit: 3
@@ -83,25 +79,18 @@ steps:
           - github_commit_status:
               context: "metricbeat: Go Integration Tests"
 
-      - label: ":python: Python Integration Tests"
+      - label: ":ubuntu: Metricbeat Python Integration Tests"
         key: "mandatory-python-int-test"
         command: |
           set -euo pipefail
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Running tests"
-
-          # TODO move this section to base image / pre-command hook
-          echo "~~~ Installing kind"
-          asdf plugin add kind
-          asdf install kind $ASDF_KIND_VERSION
-
-          .buildkite/deploy/kubernetes/scripts/kind-setup.sh
 
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
-          cd metricbeat && mage pythonIntegTest
+          cd metricbeat
+          mage pythonIntegTest
         retry:
           automatic:
             - limit: 3
@@ -116,7 +105,7 @@ steps:
           - github_commit_status:
               context: "metricbeat: Python Integration Tests"
 
-      - label: ":negative_squared_cross_mark: Cross compile"
+      - label: ":ubuntu: Metricbeat Crosscompile"
         key: "mandatory-cross-compile"
         command: "make -C metricbeat crosscompile"
         retry:
@@ -131,9 +120,9 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Cross compile"
+              context: "metricbeat: Crosscompile"
 
-      - label: ":windows: Windows 2016 Unit Tests"
+      - label: ":windows: Metricbeat Win-2016 Unit Tests"
         command: |
           Set-Location -Path metricbeat
           mage build unitTest
@@ -152,9 +141,9 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Windows 2016 Unit Tests"
+              context: "metricbeat: Win-2016 Unit Tests"
 
-      - label: ":windows: Windows 2022 Unit Tests"
+      - label: ":windows: Metricbeat Win-2022 Unit Tests"
         command: |
           Set-Location -Path metricbeat
           mage build unitTest
@@ -173,13 +162,14 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Windows 2022 Unit Tests"
+              context: "metricbeat: Win-2022 Unit Tests"
 
   - group: "Metricbeat Extended Windows Tests"
     key: "metricbeat-extended-win-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
     steps:
-      - label: ":windows: Windows 10 Unit Tests"
+      - label: ":windows: Metricbeat Win 10 Unit Tests"
         command: |
           Set-Location -Path metricbeat
           mage build unitTest
@@ -198,9 +188,9 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Extended Windows 10 Unit Tests"
+              context: "metricbeat: Win 10 Unit Tests"
 
-      - label: ":windows: Windows 11 Unit Tests"
+      - label: ":windows: Metricbeat Win 11 Unit Tests"
         command: |
           Set-Location -Path metricbeat
           mage build unitTest
@@ -219,9 +209,9 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Extended Windows 11 Unit Tests"
+              context: "metricbeat: Win 11 Unit Tests"
 
-      - label: ":windows: Windows 2019 Unit Tests"
+      - label: ":windows: Metricbeat Win-2019 Unit Tests"
         command: |
           Set-Location -Path metricbeat
           mage build unitTest
@@ -240,18 +230,20 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Extended Windows 2019 Unit Tests"
+              context: "metricbeat: Win-2019 Unit Tests"
 
-  - group: "Metricbeat Extended MacOS Tests"
+  - group: "Metricbeat Extended macOS Tests"
     key: "metricbeat-extended-macos-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+
     steps:
-      - label: ":mac: MacOS x64_64 Unit Tests"
+      - label: ":mac: Metricbeat macOS x64_64 Unit Tests"
         key: "extended-macos-x64-64-unit-tests"
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
-          cd metricbeat && mage build unitTest
+          cd metricbeat
+          mage build unitTest
         retry:
           automatic:
             - limit: 3
@@ -263,15 +255,17 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Extended MacOS x86_64 Unit Tests"
+              context: "metricbeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: MacOS arm64 Unit Tests"
+      - label: ":mac: Metricbeat macOS arm64 Unit Tests"
         key: "extended-macos-arm64-unit-tests"
-        skip: "due to https://github.com/elastic/beats/issues/33035"
+        skip: "Skipping due to elastic/beats#33035"
+        # https://github.com/elastic/beats/issues/33035
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
-          cd metricbeat && mage build unitTest
+          cd metricbeat
+          mage build unitTest
         retry:
           automatic:
             - limit: 3
@@ -283,7 +277,7 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Extended MacOS arm64 Unit Tests"
+              context: "metricbeat: macOS arm64 Unit Tests"
 
   - wait: ~
     # with PRs, we want to run packaging only if mandatory tests succeed
@@ -295,9 +289,11 @@ steps:
   - group: "Metricbeat Packaging"
     key: "metricbeat-packaging"
     steps:
-      - label: ":linux: Packaging Linux"
+      - label: ":ubuntu: Metricbeat Packaging Linux"
         key: "packaging-linux"
-        command: "cd metricbeat && mage package"
+        command: |
+          cd metricbeat
+          mage package
         retry:
           automatic:
             - limit: 3
@@ -314,20 +310,22 @@ steps:
           - github_commit_status:
               context: "metricbeat: Packaging Linux"
 
-      - label: ":linux: Packaging ARM"
+      - label: ":ubuntu: Metricbeat Packaging Linux arm64"
         key: "packaging-arm"
-        command: "cd metricbeat && mage package"
+        command: |
+          cd metricbeat
+          mage package
         retry:
           automatic:
             - limit: 3
         timeout_in_minutes: 20
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         notify:
           - github_commit_status:
-              context: "metricbeat: Packaging Linux ARM"
+              context: "metricbeat: Packaging Linux arm64"

.buildkite/packetbeat/pipeline.packetbeat.yml

@@ -25,6 +25,32 @@ env:
   TEST_COVERAGE: "true"
 
 steps:
+  - group: "Packetbeat Check/Update"
+    key: "packetbeat-check-update"
+    steps:
+      - label: "Run check/update"
+        command: |
+          make -C packetbeat check update
+          make check-no-changes
+        retry:
+          automatic:
+            - limit: 3
+        agents:
+          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          cpu: "4000m"
+          memory: "8Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "packetbeat: check/update"
+      
+  - wait: ~
+    # with PRs, we want to run mandatory tests only if check/update step succeed
+    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in check/update step
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "packetbeat-check-update"
+
   - group: "packetbeat Mandatory Tests"
     key: "packetbeat-mandatory-tests"
     steps: