Merge remote-tracking branch 'origin/423225-input-event-report' into …

…423225-input-event-report

.buildkite/metricbeat/pipeline.yml

@@ -164,7 +164,9 @@ steps:
 
       - label: ":ubuntu: Metricbeat: Crosscompile"
         key: "mandatory-cross-compile"
-        command: "make -C metricbeat crosscompile"
+        command: |
+          source .buildkite/scripts/qemu.sh
+          make -C metricbeat crosscompile
         retry:
           automatic:
             - limit: 1

.buildkite/scripts/packaging/packaging.sh

@@ -1,19 +1,14 @@
 #!/usr/bin/env bash
 #
 # Centralise the mage package for a given beat in Buildkite.
-# It enables multi-arch builds to avoid the exec format errors when 
-# attempting to build arm64 inside arm64 workers.
-# For further details, see https://github.com/elastic/elastic-agent/pull/6948
-# and https://github.com/elastic/golang-crossbuild/pull/507
 #
 
 set -ueo pipefail
 
-
 BEAT_DIR=${1:?-"Error: Beat directory must be specified."}
 
-#Use newer multiarch support for packaging
-docker run --privileged --rm tonistiigi/binfmt:master --install all
+# shellcheck source=/dev/null
+source .buildkite/scripts/qemu.sh
 
 cd $BEAT_DIR
 mage package

.buildkite/scripts/qemu.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+# It enables multi-arch builds to avoid the exec format errors when 
+# attempting to build arm64 inside arm64 workers.
+#
+# For further details, see https://github.com/elastic/elastic-agent/pull/6948
+# and https://github.com/elastic/golang-crossbuild/pull/507
+#
+set -euo pipefail
+
+if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then
+    echo "Skipping qemu installation on arm64 worker"
+else
+    BINFMT_IMAGE="tonistiigi/binfmt:qemu-v9.2.2"
+
+    # Make sure to uninstall first to avoid conflicts
+    docker run --privileged --rm "$BINFMT_IMAGE" --uninstall qemu-*
+    docker run --privileged --rm "$BINFMT_IMAGE" --install all
+fi

.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml

@@ -64,6 +64,7 @@ steps:
       - label: ":ubuntu: x-pack/dockerlogbeat: Ubuntu x86_64 Unit Tests"
         key: "mandatory-linux-unit-test"
         command: |
+          source .buildkite/scripts/qemu.sh
           cd x-pack/dockerlogbeat
           mage build unitTest
         retry:

.go-version

@@ -1 +1 @@
-1.22.12
+1.23.6

.golangci.yml

@@ -159,10 +159,6 @@ linters-settings:
       - github.com/meraki/dashboard-api-go/v3
       - github.com/snowflakedb/gosnowflake
 
-  gosimple:
-    # Select the Go version to target. The default is '1.13'.
-    go: "1.22.12"
-
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
     max-func-lines: 0
@@ -178,21 +174,13 @@ linters-settings:
     require-specific: false
 
   staticcheck:
-    # Select the Go version to target. The default is '1.13'.
-    go: "1.22.12"
     checks: ["all"]
 
   stylecheck:
-    # Select the Go version to target. The default is '1.13'.
-    go: "1.22.12"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
-  unused:
-    # Select the Go version to target. The default is '1.13'.
-    go: "1.22.12"
-
   gosec:
     excludes:
     - G306 # Expect WriteFile permissions to be 0600 or less

CHANGELOG.asciidoc

@@ -3,6 +3,160 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-9.0.0-beta1]]
+=== Beats version 9.0.0-beta1
+https://github.com/elastic/beats/compare/v8.17.2\...v9.0.0-beta1[View commits]
+
+==== Breaking changes
+
+*Affecting all Beats*
+
+- Set default Kafka version to 2.1.0 in Kafka output and Filebeat. {pull}41662[41662]
+- Replace default Ubuntu-based images with UBI-minimal-based ones. {pull}42150[42150]
+- removed support for a single `-` to precede multi-letter command line arguments.  Use `--` instead. {issue}42117[42117] {pull}42209[42209]
+
+*Filebeat*
+
+- Filebeat fails to start if there is any input with a duplicated ID. It logs the duplicated IDs and the offending inputs configurations. {pull}41731[41731]
+- Filestream inputs with duplicated IDs will fail to start. An error is logged showing the ID and the full input configuration. {issue}41938[41938] {pull}41954[41954]
+- Filestream inputs can define `allow_deprecated_id_duplication: true` to run keep the previous behaviour of running inputs with duplicated IDs. {issue}41938[41938] {pull}41954[41954]
+- The Filestream input only starts to ingest a file when it is >= 1024 bytes in size. This happens because the fingerprint is the default file identity now. To restore the previous behaviour, set `file_identity.native: ~` and `prospector.scanner.fingerprint.enabled: false`. {issue}40197[40197] {pull}41762[41762]
+- Filebeat fails to start when its configuration contains usage of the deprecated `log` or `container` inputs. However, they can still be used when `allow_deprecated_use: true` is set in their configuration. {pull}42295[42295]
+
+*Osquerybeat*
+
+- Upgrade osquery version to 5.13.1. {pull}40849[40849]
+
+*Packetbeat*
+
+- Use base-16 for reporting `serial_number` value in TLS fields in line with the ECS recommendation. {pull}41542[41542]
+
+*Winlogbeat*
+
+- Default to use raw API and delete older XML implementation. {pull}42275[42275]
+
+==== Bugfixes
+
+*Auditbeat*
+
+- hasher: Add a cached hasher for upcoming backend. {pull}41952[41952]
+- Split common tty definitions. {pull}42004[42004]
+
+*Filebeat*
+
+- Redact authorization headers in HTTPJSON debug logs. {pull}41920[41920]
+- Further rate limiting fix in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
+- The `_id` generation process for S3 events has been updated to incorporate the LastModified field. This enhancement ensures that the `_id` is unique. {pull}42078[42078]
+- Fix truncation of bodies in request tracing by limiting bodies to 10% of the maximum file size. {pull}42327[42327]
+- [Journald] Fixes handling of `journalctl` restart. A known symptom was broken multiline messages when there was a restart of journalctl while aggregating the lines. {issue}41331[41331] {pull}42595[42595]
+
+*Metricbeat*
+
+- Fix bug where Metricbeat unintentionally triggers Windows ASR. {pull}42177[42177]
+- Remove `hostname` field from ZooKeeper's `mntr` data stream. {pull}41887[41887]
+
+*Packetbeat*
+
+- Properly marshal nested structs in ECS fields, fixing issues with mixed cases in field names. {pull}42116[42116]
+
+==== Added
+
+*Auditbeat*
+
+- Improve logging in system/socket. {pull}41571[41571]
+
+*Filebeat*
+
+- Added out of the box support for Amazon EventBridge notifications over SQS to S3 input. {pull}40006[40006]
+- Update CEL mito extensions to v1.16.0. {pull}41727[41727]
+- Filebeat's registry is now added to the Elastic-Agent diagnostics bundle. {issue}33238[33238] {pull}41795[41795]
+- Add `unifiedlogs` input for MacOS. {pull}41791[41791]
+- Add evaluation state dump debugging option to CEL input. {pull}41335[41335]
+- Rate limiting operability improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
+- Rate limiting fault tolerance improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42094[42094]
+- Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804]
+- Journald input now can report its status to Elastic-Agent. {issue}39791[39791] {pull}42462[42462]
+- Publish events progressively in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42567[42567]
+- Journald `include_matches.match` now accepts `+` to represent a logical disjunction (OR). {issue}40185[40185] {pull}42517[42517]
+- The journald input is now generally available. {pull}42107[42107]
+
+*Heartbeat*
+
+- Add support for RFC7231 methods to HTTP monitors. {pull}41975[41975]
+
+*Metricbeat*
+
+- Add `use_kubeadm` config option in kubernetes module in order to toggle kubeadm-config API requests. {pull}40086[40086]
+- Preserve queries for debugging when `merge_results: true` in SQL module. {pull}42271[42271]
+- Collect more fields from ES node/stats metrics and only those that are necessary. {pull}42421[42421]
+
+*Metricbeat*
+- Add benchmark module. {pull}41801[41801]
+
+*Osquerybeat*
+
+- Increase maximum query timeout to 24 hours. {pull}42356[42356]
+
+*Winlogbeat*
+
+- Properly set events `UserData` when experimental API is used. {pull}41525[41525]
+- Include XML is respected for experimental API. {pull}41525[41525]
+- Forwarded events use renderedtext info for experimental API. {pull}41525[41525]
+- Language setting is respected for experimental API. {pull}41525[41525]
+- Language setting also added to decode XML wineventlog processor. {pull}41525[41525]
+- Format embedded messages in the experimental API. {pull}41525[41525]
+- Make the experimental API GA and rename it to winlogbeat-raw. {issue}39580[39580] {pull}41770[41770]
+- Remove 22 clause limitation. {issue}35047[35047] {pull}42187[42187]
+- Add handling for recoverable publisher disabled errors. {issue}35316[35316] {pull}42187[42187]
+
+*Functionbeat*
+
+- Remove Functionbeat binaries from CI pipelines. {issue}40745[40745] {pull}41506[41506]
+
+
+[[release-notes-8.17.3]]
+=== Beats version 8.17.3
+https://github.com/elastic/beats/compare/v8.17.2\...v8.17.3[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Restored event Meta fields in the Elasticsearch output's error logs. {pull}42559[42559]
+
+*Filebeat*
+
+- [Journald] Fixes handling of `journalctl` restart. A known symptom was broken multiline messages when there was a restart of journalctl while aggregating the lines. {issue}41331[41331] {pull}42595[42595]
+- Fix entityanalytics activedirectory provider full sync use before initialization bug. {pull}42682[42682]
+- In the `http_endpoint` input, fix the check for a missing HMAC HTTP header. {pull}42756[42756]
+
+*Metricbeat*
+
+- Fixed panic caused by uninitialized meraki device wifi0 and wifi1 struct pointers in the device WiFi data fetching. {issue}42745[42745] {pull}42746[42746]
+- Only fetch cluster-level index stats summary. {issue}36019[36019] {pull}42901[42901]
+- Fixed an issue in Metricbeat's Windows module where data collection would fail if the data was unavailable. {issue}42802[42802] {pull}42803[42803]
+
+*Winlogbeat*
+
+- Sync missing changes in modules pipelines. {pull}42619[42619]
+
+==== Added
+
+*Affecting all Beats*
+
+- Update Go version to 1.22.12. {pull}42681[42681]
+
+*Filebeat*
+
+- Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804]
+- Publish events progressively in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42567[42567]
+
+*Metricbeat*
+
+- Log every 401 response from Kubernetes API Server. {pull}42714[42714]
+- Collect more fields from ES node/stats metrics and only those that are necessary. {pull}42421[42421]
+
+
 [[release-notes-8.17.2]]
 === Beats version 8.17.2
 https://github.com/elastic/beats/compare/v8.17.1\...v8.17.2[View commits]
@@ -180,6 +334,35 @@ https://github.com/elastic/beats/compare/v8.16.1\...v8.17.0[View commits]
 - Implement exclusion range support for event_id. {issue}38623[38623] {pull}41639[41639]
 
 
+[[release-notes-8.16.5]]
+=== Beats version 8.16.5
+https://github.com/elastic/beats/compare/v8.16.4\...v8.16.5[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- [Journald] Fixes handling of `journalctl` restart. A known symptom was broken multiline messages when there was a restart of journalctl while aggregating the lines. {issue}41331[41331] {pull}42595[42595]
+
+*Winlogbeat*
+
+- Sync missing changes in modules pipelines. {pull}42619[42619]
+
+==== Added
+
+*Affecting all Beats*
+
+- Update Go version to 1.22.12. {pull}42681[42681]
+
+*Filebeat*
+
+- Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804]
+
+*Metricbeat*
+
+- Log every 401 response from Kubernetes API Server. {pull}42714[42714]
+
+
 [[release-notes-8.16.4]]
 === Beats version 8.16.4
 https://github.com/elastic/beats/compare/v8.16.3\...v8.16.4[View commits]

CHANGELOG.next.asciidoc

@@ -128,21 +128,6 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Support Elastic Agent control protocol chunking support {pull}37343[37343]
 - Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments {pull}[37816][37816]
 - Set timeout of 1 minute for FQDN requests {pull}37756[37756]
-- Fix issue where old data could be saved in the memory queue after acknowledgment, increasing memory use {pull}41356[41356]
-- Ensure Elasticsearch output can always recover from network errors {pull}40794[40794]
-- Add `translate_ldap_attribute` processor. {pull}41472[41472]
-- Remove unnecessary debug logs during idle connection teardown {issue}40824[40824]
-- Remove unnecessary reload for Elastic Agent managed beats when apm tracing config changes from nil to nil {pull}41794[41794]
-- Fix incorrect cloud provider identification in add_cloud_metadata processor using provider priority mechanism {pull}41636[41636]
-- Prevent panic if libbeat processors are loaded more than once. {issue}41475[41475] {pull}41857[51857]
-- Allow network condition to handle field values that are arrays of IP addresses. {pull}41918[41918]
-- Fix a bug where log files are rotated on startup when interval is configured and rotateonstartup is disabled {issue}41894[41894] {pull}41895[41895]
-- Fix setting unique registry for non beat receivers {issue}42288[42288] {pull}42292[42292]
-- The Kafka output now drops events when there is an authorisation error {issue}42343[42343] {pull}42401[42401]
-- Fix autodiscovery memory leak related to metadata of start events {pull}41748[41748]
-- All standard queue metrics are now included in metrics monitoring, including: `added.{events, bytes}`, `consumed.{events, bytes}`, `removed.{events, bytes}`, and `filled.{events, bytes, pct}`. {pull}42439[42439]
-- The following output latency metrics are now included in metrics monitoring: `output.latency.{count, max, median, p99}`. {pull}42439[42439]
-- Restored event Meta fields in the Elasticsearch output's error logs. {pull}42559[42559]
 
 *Auditbeat*
 
@@ -230,6 +215,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - [Journald] Fixes handling of `journalctl` restart. A known symptom was broken multiline messages when there was a restart of journalctl while aggregating the lines. {issue}41331[41331] {pull}42595[42595]
 - Fix entityanalytics activedirectory provider full sync use before initialization bug. {pull}42682[42682]
 - In the `http_endpoint` input, fix the check for a missing HMAC HTTP header. {pull}42756[42756]
+- Prevent computer details being returned for user queries by Activedirectory Entity Analytics provider. {issue}11818[11818] {pull}42796[42796]
 
 *Heartbeat*
 
@@ -248,18 +234,12 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627]
 - Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664]
 - Fix behavior of pagetypeinfo metrics {pull}39985[39985]
-- Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787]
-- Set GCP metrics config period to the default (60s) when the value is below the minimum allowed period. {issue}30434[30434] {pull}40020[40020]
-- Fix statistic methods for metrics collected for SQS. {pull}40207[40207]
-- Add GCP 'instance_id' resource label in ECS cloud fields. {issue}40033[40033] {pull}40062[40062]
-- Fix missing metrics from CloudWatch when include_linked_accounts set to false. {issue}40071[40071] {pull}40135[40135]
 - Update beat module with apm-server monitoring metrics fields {pull}40127[40127]
 - Fix Azure Monitor metric timespan to restore Storage Account PT1H metrics {issue}40376[40376] {pull}40367[40367]
 - Remove excessive info-level logs in cgroups setup {pull}40491[40491]
 - Add missing ECS Cloud fields in GCP `metrics` metricset when using `exclude_labels: true` {issue}40437[40437] {pull}40467[40467]
 - Add AWS OwningAccount support for cross account monitoring {issue}40570[40570] {pull}40691[40691]
 - Use namespace for GetListMetrics when exists in AWS {pull}41022[41022]
-- Fix http server helper SSL config. {pull}39405[39405]
 - Fix Kubernetes metadata sometimes not being present after startup {pull}41216[41216]
 - Do not report non-existant 0 values for RSS metrics in docker/memory {pull}41449[41449]
 - Log Cisco Meraki `getDevicePerformanceScores` errors without stopping metrics collection. {pull}41622[41622]
@@ -288,6 +268,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Sync missing changes in modules pipelines. {pull}42619[42619]
 - Reset EventLog if error EOF is encountered. {pull}42826[42826]
 - Implement backoff on error retrial. {pull}42826[42826]
+- Fix boolean key in security pipelines and sync pipelines with integration. {pull}43027[43027]
 
 
 *Elastic Logging Plugin*
@@ -308,7 +289,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - The environment variable `BEATS_ADD_CLOUD_METADATA_PROVIDERS` overrides configured/default `add_cloud_metadata` providers {pull}38669[38669]
 - When running under Elastic-Agent Kafka output allows dynamic topic in `topic` field {pull}40415[40415]
 - The script processor has a new configuration option that only uses the cached javascript sessions and prevents the creation of new javascript sessions.
-- Update to Go 1.22.12. {pull}42681[42681]
+- Update to Go 1.23.6. {pull}42705[42705]
 - Replace Ubuntu 20.04 with 24.04 for Docker base images {issue}40743[40743] {pull}40942[40942]
 - Reduce memory consumption of k8s autodiscovery and the add_kubernetes_metadata processor when Deployment metadata is enabled
 - Add `lowercase` processor. {issue}22254[22254] {pull}41424[41424]
@@ -583,6 +564,9 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 
 
 
+
+
+