Merge branch 'main' into bump-libs

CHANGELOG.asciidoc

@@ -3,6 +3,160 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-9.0.0-beta1]]
+=== Beats version 9.0.0-beta1
+https://github.com/elastic/beats/compare/v8.17.2\...v9.0.0-beta1[View commits]
+
+==== Breaking changes
+
+*Affecting all Beats*
+
+- Set default Kafka version to 2.1.0 in Kafka output and Filebeat. {pull}41662[41662]
+- Replace default Ubuntu-based images with UBI-minimal-based ones. {pull}42150[42150]
+- removed support for a single `-` to precede multi-letter command line arguments.  Use `--` instead. {issue}42117[42117] {pull}42209[42209]
+
+*Filebeat*
+
+- Filebeat fails to start if there is any input with a duplicated ID. It logs the duplicated IDs and the offending inputs configurations. {pull}41731[41731]
+- Filestream inputs with duplicated IDs will fail to start. An error is logged showing the ID and the full input configuration. {issue}41938[41938] {pull}41954[41954]
+- Filestream inputs can define `allow_deprecated_id_duplication: true` to run keep the previous behaviour of running inputs with duplicated IDs. {issue}41938[41938] {pull}41954[41954]
+- The Filestream input only starts to ingest a file when it is >= 1024 bytes in size. This happens because the fingerprint is the default file identity now. To restore the previous behaviour, set `file_identity.native: ~` and `prospector.scanner.fingerprint.enabled: false`. {issue}40197[40197] {pull}41762[41762]
+- Filebeat fails to start when its configuration contains usage of the deprecated `log` or `container` inputs. However, they can still be used when `allow_deprecated_use: true` is set in their configuration. {pull}42295[42295]
+
+*Osquerybeat*
+
+- Upgrade osquery version to 5.13.1. {pull}40849[40849]
+
+*Packetbeat*
+
+- Use base-16 for reporting `serial_number` value in TLS fields in line with the ECS recommendation. {pull}41542[41542]
+
+*Winlogbeat*
+
+- Default to use raw API and delete older XML implementation. {pull}42275[42275]
+
+==== Bugfixes
+
+*Auditbeat*
+
+- hasher: Add a cached hasher for upcoming backend. {pull}41952[41952]
+- Split common tty definitions. {pull}42004[42004]
+
+*Filebeat*
+
+- Redact authorization headers in HTTPJSON debug logs. {pull}41920[41920]
+- Further rate limiting fix in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
+- The `_id` generation process for S3 events has been updated to incorporate the LastModified field. This enhancement ensures that the `_id` is unique. {pull}42078[42078]
+- Fix truncation of bodies in request tracing by limiting bodies to 10% of the maximum file size. {pull}42327[42327]
+- [Journald] Fixes handling of `journalctl` restart. A known symptom was broken multiline messages when there was a restart of journalctl while aggregating the lines. {issue}41331[41331] {pull}42595[42595]
+
+*Metricbeat*
+
+- Fix bug where Metricbeat unintentionally triggers Windows ASR. {pull}42177[42177]
+- Remove `hostname` field from ZooKeeper's `mntr` data stream. {pull}41887[41887]
+
+*Packetbeat*
+
+- Properly marshal nested structs in ECS fields, fixing issues with mixed cases in field names. {pull}42116[42116]
+
+==== Added
+
+*Auditbeat*
+
+- Improve logging in system/socket. {pull}41571[41571]
+
+*Filebeat*
+
+- Added out of the box support for Amazon EventBridge notifications over SQS to S3 input. {pull}40006[40006]
+- Update CEL mito extensions to v1.16.0. {pull}41727[41727]
+- Filebeat's registry is now added to the Elastic-Agent diagnostics bundle. {issue}33238[33238] {pull}41795[41795]
+- Add `unifiedlogs` input for MacOS. {pull}41791[41791]
+- Add evaluation state dump debugging option to CEL input. {pull}41335[41335]
+- Rate limiting operability improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
+- Rate limiting fault tolerance improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42094[42094]
+- Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804]
+- Journald input now can report its status to Elastic-Agent. {issue}39791[39791] {pull}42462[42462]
+- Publish events progressively in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42567[42567]
+- Journald `include_matches.match` now accepts `+` to represent a logical disjunction (OR). {issue}40185[40185] {pull}42517[42517]
+- The journald input is now generally available. {pull}42107[42107]
+
+*Heartbeat*
+
+- Add support for RFC7231 methods to HTTP monitors. {pull}41975[41975]
+
+*Metricbeat*
+
+- Add `use_kubeadm` config option in kubernetes module in order to toggle kubeadm-config API requests. {pull}40086[40086]
+- Preserve queries for debugging when `merge_results: true` in SQL module. {pull}42271[42271]
+- Collect more fields from ES node/stats metrics and only those that are necessary. {pull}42421[42421]
+
+*Metricbeat*
+- Add benchmark module. {pull}41801[41801]
+
+*Osquerybeat*
+
+- Increase maximum query timeout to 24 hours. {pull}42356[42356]
+
+*Winlogbeat*
+
+- Properly set events `UserData` when experimental API is used. {pull}41525[41525]
+- Include XML is respected for experimental API. {pull}41525[41525]
+- Forwarded events use renderedtext info for experimental API. {pull}41525[41525]
+- Language setting is respected for experimental API. {pull}41525[41525]
+- Language setting also added to decode XML wineventlog processor. {pull}41525[41525]
+- Format embedded messages in the experimental API. {pull}41525[41525]
+- Make the experimental API GA and rename it to winlogbeat-raw. {issue}39580[39580] {pull}41770[41770]
+- Remove 22 clause limitation. {issue}35047[35047] {pull}42187[42187]
+- Add handling for recoverable publisher disabled errors. {issue}35316[35316] {pull}42187[42187]
+
+*Functionbeat*
+
+- Remove Functionbeat binaries from CI pipelines. {issue}40745[40745] {pull}41506[41506]
+
+
+[[release-notes-8.17.3]]
+=== Beats version 8.17.3
+https://github.com/elastic/beats/compare/v8.17.2\...v8.17.3[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Restored event Meta fields in the Elasticsearch output's error logs. {pull}42559[42559]
+
+*Filebeat*
+
+- [Journald] Fixes handling of `journalctl` restart. A known symptom was broken multiline messages when there was a restart of journalctl while aggregating the lines. {issue}41331[41331] {pull}42595[42595]
+- Fix entityanalytics activedirectory provider full sync use before initialization bug. {pull}42682[42682]
+- In the `http_endpoint` input, fix the check for a missing HMAC HTTP header. {pull}42756[42756]
+
+*Metricbeat*
+
+- Fixed panic caused by uninitialized meraki device wifi0 and wifi1 struct pointers in the device WiFi data fetching. {issue}42745[42745] {pull}42746[42746]
+- Only fetch cluster-level index stats summary. {issue}36019[36019] {pull}42901[42901]
+- Fixed an issue in Metricbeat's Windows module where data collection would fail if the data was unavailable. {issue}42802[42802] {pull}42803[42803]
+
+*Winlogbeat*
+
+- Sync missing changes in modules pipelines. {pull}42619[42619]
+
+==== Added
+
+*Affecting all Beats*
+
+- Update Go version to 1.22.12. {pull}42681[42681]
+
+*Filebeat*
+
+- Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804]
+- Publish events progressively in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42567[42567]
+
+*Metricbeat*
+
+- Log every 401 response from Kubernetes API Server. {pull}42714[42714]
+- Collect more fields from ES node/stats metrics and only those that are necessary. {pull}42421[42421]
+
+
 [[release-notes-8.17.2]]
 === Beats version 8.17.2
 https://github.com/elastic/beats/compare/v8.17.1\...v8.17.2[View commits]

CHANGELOG.next.asciidoc

@@ -128,21 +128,6 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Support Elastic Agent control protocol chunking support {pull}37343[37343]
 - Lower logging level to debug when attempting to configure beats with unknown fields from autodiscovered events/environments {pull}[37816][37816]
 - Set timeout of 1 minute for FQDN requests {pull}37756[37756]
-- Fix issue where old data could be saved in the memory queue after acknowledgment, increasing memory use {pull}41356[41356]
-- Ensure Elasticsearch output can always recover from network errors {pull}40794[40794]
-- Add `translate_ldap_attribute` processor. {pull}41472[41472]
-- Remove unnecessary debug logs during idle connection teardown {issue}40824[40824]
-- Remove unnecessary reload for Elastic Agent managed beats when apm tracing config changes from nil to nil {pull}41794[41794]
-- Fix incorrect cloud provider identification in add_cloud_metadata processor using provider priority mechanism {pull}41636[41636]
-- Prevent panic if libbeat processors are loaded more than once. {issue}41475[41475] {pull}41857[51857]
-- Allow network condition to handle field values that are arrays of IP addresses. {pull}41918[41918]
-- Fix a bug where log files are rotated on startup when interval is configured and rotateonstartup is disabled {issue}41894[41894] {pull}41895[41895]
-- Fix setting unique registry for non beat receivers {issue}42288[42288] {pull}42292[42292]
-- The Kafka output now drops events when there is an authorisation error {issue}42343[42343] {pull}42401[42401]
-- Fix autodiscovery memory leak related to metadata of start events {pull}41748[41748]
-- All standard queue metrics are now included in metrics monitoring, including: `added.{events, bytes}`, `consumed.{events, bytes}`, `removed.{events, bytes}`, and `filled.{events, bytes, pct}`. {pull}42439[42439]
-- The following output latency metrics are now included in metrics monitoring: `output.latency.{count, max, median, p99}`. {pull}42439[42439]
-- Restored event Meta fields in the Elasticsearch output's error logs. {pull}42559[42559]
 
 *Auditbeat*
 
@@ -283,6 +268,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Sync missing changes in modules pipelines. {pull}42619[42619]
 - Reset EventLog if error EOF is encountered. {pull}42826[42826]
 - Implement backoff on error retrial. {pull}42826[42826]
+- Fix boolean key in security pipelines and sync pipelines with integration. {pull}43027[43027]
 
 
 *Elastic Logging Plugin*

libbeat/docs/release.asciidoc

@@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read
 <<breaking-changes>> for more detail about changes that affect
 upgrade.
 
+* <<release-notes-8.17.3>>
 * <<release-notes-8.17.2>>
 * <<release-notes-8.17.1>>
 * <<release-notes-8.17.0>>

x-pack/filebeat/input/unifiedlogs/input_test.go

@@ -58,6 +58,7 @@ func TestInput(t *testing.T) {
 
 	testCases := []struct {
 		name                 string
+		skip                 func(*testing.T) bool
 		cfg                  config
 		timeUntilClose       time.Duration
 		assertFunc           func(collect *assert.CollectT, events []beat.Event, cursors []*time.Time)
@@ -139,6 +140,19 @@ func TestInput(t *testing.T) {
 		},
 		{
 			name: "With end date",
+			skip: func(t *testing.T) bool {
+				const sequoiaPrefix = "15."
+				version, err := exec.Command("sw_vers", "-productVersion").CombinedOutput()
+				if err != nil {
+					t.Fatalf("failed to get macOS version: %v", err)
+					return true
+				}
+				if strings.HasPrefix(strings.TrimSpace(string(version)), sequoiaPrefix) {
+					t.Skip("macOS 15.x does not support the --end flag correctly")
+					return true
+				}
+				return false
+			},
 			cfg: config{
 				ShowConfig: showConfig{
 					ArchiveFile: archivePath,
@@ -147,7 +161,7 @@ func TestInput(t *testing.T) {
 			},
 			timeUntilClose:     time.Second,
 			expectedLogShowCmd: fmt.Sprintf("/usr/bin/log show --style ndjson --archive %s --end 2024-12-04 13:46:00+0200", archivePath),
-			assertFunc:         eventsAndCursorAssertN(462),
+			assertFunc:         eventsAndCursorAssertN(149),
 		},
 		{
 			name: "With predicate",
@@ -204,6 +218,9 @@ func TestInput(t *testing.T) {
 	for _, tc := range testCases {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
+			if tc.skip != nil && tc.skip(t) {
+				return
+			}
 			_, cursorInput := newCursorInput(tc.cfg)
 			input := cursorInput.(*input)
 

x-pack/winlogbeat/module/powershell/ingest/powershell.yml

@@ -46,7 +46,7 @@ processors:
 
   - set:
       field: ecs.version
-      value: '8.0.0'
+      value: '8.17.0'
   - set:
       field: log.level
       copy_from: winlog.level

x-pack/winlogbeat/module/powershell/ingest/powershell_operational.yml

@@ -26,7 +26,7 @@ processors:
 
   - set:
       field: ecs.version
-      value: '8.0.0'
+      value: '8.17.0'
   - set:
       field: log.level
       copy_from: winlog.level

x-pack/winlogbeat/module/routing/ingest/routing.yml

@@ -16,6 +16,7 @@ processors:
   - pipeline:
       name: '{< IngestPipeline "powershell_operational" >}'
       if: ctx.winlog?.channel instanceof String && ctx.winlog.channel.toLowerCase() == 'microsoft-windows-powershell/operational'
+
   - set:
       field: host.os.type
       value: windows
@@ -25,8 +26,39 @@ processors:
       value: windows
       override: false
 
+  # Get user details from the translate_sid processor enrichment
+  # if they are available and we don't already have them.
+  - rename:
+      field: winlog.event_data._MemberUserName
+      target_field: user.name
+      ignore_failure: true
+      ignore_missing: true
+  - rename:
+      field: winlog.event_data._MemberDomain
+      target_field: user.domain
+      ignore_failure: true
+      ignore_missing: true
+  - append:
+      value: '{{{winlog.event_data._MemberAccountType}}}'
+      field: user.roles
+      ignore_failure: true
+      allow_duplicates: false
+      if: ctx.winlog?.event_data?._MemberAccountType != null
+  - remove:
+      field: winlog.event_data._MemberAccountType
+      ignore_missing: true
+      ignore_failure: true
+      if: ctx.user?.roles != null && ctx.winlog?.event_data?._MemberAccountType != null && ctx.user.roles.contains(ctx.winlog.event_data._MemberAccountType)
+
+  - convert:
+      field: error.code
+      type: string
+      ignore_missing: true
+
 on_failure:
   - set:
+      field: event.kind
+      value: pipeline_error
+  - append:
       field: error.message
-      value: |-
-        Processor "{{ _ingest.on_failure_processor_type }}" with tag "{{ _ingest.on_failure_processor_tag }}" in pipeline "{{ _ingest.on_failure_pipeline }}" failed with message "{{ _ingest.on_failure_message }}"
+      value: "{{{ _ingest.on_failure_message }}}"