From 76d74c2e62dabbf40c2ae74df4a27eea8a8024db Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Sun, 23 Feb 2025 14:20:15 +0530 Subject: [PATCH 01/15] [otelconsumer]: Add support for conversion of type any --- libbeat/otelbeat/otelmap/otelmap.go | 46 ++++++++++++++++++++++++ libbeat/otelbeat/otelmap/otelmap_test.go | 16 +++++++++ 2 files changed, 62 insertions(+) diff --git a/libbeat/otelbeat/otelmap/otelmap.go b/libbeat/otelbeat/otelmap/otelmap.go index 69525cc1051b..a1a66d620fcf 100644 --- a/libbeat/otelbeat/otelmap/otelmap.go +++ b/libbeat/otelbeat/otelmap/otelmap.go @@ -168,9 +168,55 @@ func FromMapstr(m mapstr.M) pcommon.Map { newVal := dest.AppendEmpty() newVal.SetStr(i.UTC().Format("2006-01-02T15:04:05.000Z")) } + case []any: + dest := out.PutEmptySlice(k) + convertValue(v.([]interface{}), dest) default: out.PutStr(k, fmt.Sprintf("unknown type: %T", x)) } } return out } + +// convertValue converts a slice of any[] to pcommon.Value +func convertValue(v []any, dest pcommon.Slice) { + // Handling the most common types without reflect is a small perf win. + for _, i := range v { + newValue := dest.AppendEmpty() + switch val := i.(type) { + case bool: + newValue.SetBool(val) + case string: + newValue.SetStr(val) + case int: + newValue.SetInt(int64(val)) + case int8: + newValue.SetInt(int64(val)) + case int16: + newValue.SetInt(int64(val)) + case int32: + newValue.SetInt(int64(val)) + case int64: + newValue.SetInt(int64(val)) + case uint: + newValue.SetInt(int64(val)) + case uint8: + newValue.SetInt(int64(val)) + case uint16: + newValue.SetInt(int64(val)) + case uint32: + newValue.SetInt(int64(val)) + case uint64: + newValue.SetInt(int64(val)) + case float32: + newValue.SetDouble(float64(val)) + case float64: + newValue.SetDouble(float64(val)) + case time.Time: + newValue.SetStr(val.UTC().Format("2006-01-02T15:04:05.000Z")) + default: + newValue.SetStr(fmt.Sprintf("unknown type: %T", val)) + } + + } +} diff --git a/libbeat/otelbeat/otelmap/otelmap_test.go b/libbeat/otelbeat/otelmap/otelmap_test.go index 9970f48b2875..de28769d816e 100644 --- a/libbeat/otelbeat/otelmap/otelmap_test.go +++ b/libbeat/otelbeat/otelmap/otelmap_test.go @@ -124,6 +124,22 @@ func TestFromMapstrSliceInt(t *testing.T) { assert.Equal(t, want, got) } +func TestFromMapstrSliceAny(t *testing.T) { + inputSlice := []any{42, 43, 44} + inputMap := mapstr.M{ + "slice": inputSlice, + } + want := pcommon.NewMap() + sliceOfInt := want.PutEmptySlice("slice") + for _, i := range inputSlice { + val := sliceOfInt.AppendEmpty() + val.SetInt(int64(i.(int))) + } + + got := FromMapstr(inputMap) + assert.Equal(t, want, got) +} + func TestFromMapstrDouble(t *testing.T) { tests := map[string]struct { mapstr_val interface{} From 863fc7248fa3dc936bd51fa3ee03743e8d22248e Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 24 Feb 2025 12:46:29 +0530 Subject: [PATCH 02/15] fix lint --- libbeat/otelbeat/otelmap/otelmap.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/otelbeat/otelmap/otelmap.go b/libbeat/otelbeat/otelmap/otelmap.go index a1a66d620fcf..0e08d69eed7c 100644 --- a/libbeat/otelbeat/otelmap/otelmap.go +++ b/libbeat/otelbeat/otelmap/otelmap.go @@ -197,7 +197,7 @@ func convertValue(v []any, dest pcommon.Slice) { case int32: newValue.SetInt(int64(val)) case int64: - newValue.SetInt(int64(val)) + newValue.SetInt(val) case uint: newValue.SetInt(int64(val)) case uint8: @@ -211,7 +211,7 @@ func convertValue(v []any, dest pcommon.Slice) { case float32: newValue.SetDouble(float64(val)) case float64: - newValue.SetDouble(float64(val)) + newValue.SetDouble(val) case time.Time: newValue.SetStr(val.UTC().Format("2006-01-02T15:04:05.000Z")) default: From c5ef3efd2c4b43445f7345a4bd5fe441c2b3b0f0 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Wed, 26 Feb 2025 23:04:20 +0530 Subject: [PATCH 03/15] poc --- x-pack/filebeat/fbreceiver/factory.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/filebeat/fbreceiver/factory.go b/x-pack/filebeat/fbreceiver/factory.go index b3c12afb8979..10cae89507f4 100644 --- a/x-pack/filebeat/fbreceiver/factory.go +++ b/x-pack/filebeat/fbreceiver/factory.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/include" inputs "github.com/elastic/beats/v7/x-pack/filebeat/input/default-inputs" "github.com/elastic/elastic-agent-libs/mapstr" + "go.uber.org/zap" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" @@ -45,6 +46,8 @@ func createReceiver(_ context.Context, set receiver.Settings, baseCfg component. settings.ElasticLicensed = true settings.Initialize = append(settings.Initialize, include.InitializeModule) + // correct values are extracted from baseCfg + set.Logger = set.Logger.With(zap.String("component.id", "randomID")) b, err := instance.NewBeatReceiver(settings, cfg.Beatconfig, true, consumer, set.Logger.Core()) if err != nil { return nil, fmt.Errorf("error creating %s: %w", Name, err) From 9e0aefe495c9597cf984b2a53d421549977f31b6 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Thu, 27 Feb 2025 18:36:49 +0530 Subject: [PATCH 04/15] first draft --- libbeat/cmd/instance/beat.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 99bf7cbd2c99..17fe9f8a7bd8 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -346,6 +346,14 @@ func NewBeatReceiver(settings Settings, receiverConfig map[string]interface{}, u return nil, fmt.Errorf("error unpacking beats logging config: %w\n%v", err, b.Config.Logging) } + if logpConfig.WithFields != nil { + var fields = []zapcore.Field{} + for key, value := range logpConfig.WithFields { + fields = append(fields, zap.Any(key, value)) + } + core = core.With(fields) + } + if err := logp.ConfigureWithCore(logpConfig, core); err != nil { return nil, fmt.Errorf("error configuring beats logp: %w", err) } From c144433def8e6418daaf216a51f55bd9da4b4505 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 28 Feb 2025 16:46:20 +0530 Subject: [PATCH 05/15] make update --- NOTICE.txt | 20 ++++++++++---------- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index afc3813e3c76..504f295b46d1 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12488,11 +12488,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.18.1 +Version: v0.18.7 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.18.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.18.7/LICENSE: Apache License Version 2.0, January 2004 @@ -27220,11 +27220,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.31.0 +Version: v0.32.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.31.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.32.0/LICENSE: Copyright 2009 The Go Authors. @@ -27294,11 +27294,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.33.0 +Version: v0.34.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.33.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.34.0/LICENSE: Copyright 2009 The Go Authors. @@ -27405,11 +27405,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.28.0 +Version: v0.29.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.28.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.29.0/LICENSE: Copyright 2009 The Go Authors. @@ -27442,11 +27442,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.27.0 +Version: v0.28.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.27.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.28.0/LICENSE: Copyright 2009 The Go Authors. diff --git a/go.mod b/go.mod index 30f5522581b2..69a32b2e734f 100644 --- a/go.mod +++ b/go.mod @@ -126,12 +126,12 @@ require ( go.etcd.io/bbolt v1.3.10 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.31.0 + golang.org/x/crypto v0.32.0 golang.org/x/mod v0.21.0 - golang.org/x/net v0.33.0 + golang.org/x/net v0.34.0 golang.org/x/oauth2 v0.24.0 golang.org/x/sync v0.10.0 - golang.org/x/sys v0.28.0 + golang.org/x/sys v0.29.0 golang.org/x/text v0.21.0 golang.org/x/time v0.8.0 golang.org/x/tools v0.25.0 @@ -176,7 +176,7 @@ require ( github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.6.0 github.com/elastic/elastic-agent-autodiscover v0.9.0 - github.com/elastic/elastic-agent-libs v0.18.1 + github.com/elastic/elastic-agent-libs v0.18.7 github.com/elastic/elastic-agent-system-metrics v0.11.7 github.com/elastic/go-elasticsearch/v8 v8.17.0 github.com/elastic/go-quark v0.3.0 @@ -229,7 +229,7 @@ require ( go.opentelemetry.io/collector/pdata v1.20.0 go.opentelemetry.io/collector/receiver v0.114.0 go.uber.org/mock v0.5.0 - golang.org/x/term v0.27.0 + golang.org/x/term v0.28.0 google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) diff --git a/go.sum b/go.sum index 5d1702535981..99139a839aef 100644 --- a/go.sum +++ b/go.sum @@ -342,8 +342,8 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4= github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7bMgXoT2DsHfolO2CHE= github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI= -github.com/elastic/elastic-agent-libs v0.18.1 h1:dE6jf/D9bP8eRMQsV7KKpKV/G8zQzwMFBTj1w4e716c= -github.com/elastic/elastic-agent-libs v0.18.1/go.mod h1:rWdyrrAFzZwgNNi41Tsqhlt2c2GdXWhCEwcsnqISJ2U= +github.com/elastic/elastic-agent-libs v0.18.7 h1:C/63JieRiRIKBCOHnusIQ6yGBBmTU9rqcxneOw3zVX4= +github.com/elastic/elastic-agent-libs v0.18.7/go.mod h1:Repx7BMzE1v/gTipPogNIQeEnSGwOWGBC63h7h9c5aM= github.com/elastic/elastic-agent-system-metrics v0.11.7 h1:1xm2okCM0eQZ4jivZgUFSlt6HAn/nPgKB/Fj8eLG6mY= github.com/elastic/elastic-agent-system-metrics v0.11.7/go.mod h1:nzkrGajQA29YNcfP62gfzhxX9an3/xdQ3RmfQNw9YTI= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= @@ -1152,8 +1152,8 @@ golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= @@ -1199,8 +1199,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= @@ -1261,8 +1261,8 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -1272,8 +1272,8 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From c62bf6d2f3371ada5dddde8b67a899a54b3bd51d Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 28 Feb 2025 17:01:34 +0530 Subject: [PATCH 06/15] remove files from previous commit --- libbeat/docs/loggingconfig.asciidoc | 15 ++++++++ libbeat/otelbeat/otelmap/otelmap.go | 46 ------------------------ libbeat/otelbeat/otelmap/otelmap_test.go | 16 --------- x-pack/filebeat/fbreceiver/factory.go | 3 -- 4 files changed, 15 insertions(+), 65 deletions(-) diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index dd3a70811713..f5e85b392fad 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -94,6 +94,21 @@ ifndef::serverless[] When true, writes all logging output to standard error output. This is equivalent to using the `-e` command line option. +ifndef::serverless[] +[float] +==== `logging.with_fields` + +A map of key-value pair added to beat process logs for additional context. + +Usage: +``` +logging.with_fields: + id: 123 + data_stream: + dataset: beat + namespace: default +``` + [float] ==== `logging.to_syslog` diff --git a/libbeat/otelbeat/otelmap/otelmap.go b/libbeat/otelbeat/otelmap/otelmap.go index 0e08d69eed7c..69525cc1051b 100644 --- a/libbeat/otelbeat/otelmap/otelmap.go +++ b/libbeat/otelbeat/otelmap/otelmap.go @@ -168,55 +168,9 @@ func FromMapstr(m mapstr.M) pcommon.Map { newVal := dest.AppendEmpty() newVal.SetStr(i.UTC().Format("2006-01-02T15:04:05.000Z")) } - case []any: - dest := out.PutEmptySlice(k) - convertValue(v.([]interface{}), dest) default: out.PutStr(k, fmt.Sprintf("unknown type: %T", x)) } } return out } - -// convertValue converts a slice of any[] to pcommon.Value -func convertValue(v []any, dest pcommon.Slice) { - // Handling the most common types without reflect is a small perf win. - for _, i := range v { - newValue := dest.AppendEmpty() - switch val := i.(type) { - case bool: - newValue.SetBool(val) - case string: - newValue.SetStr(val) - case int: - newValue.SetInt(int64(val)) - case int8: - newValue.SetInt(int64(val)) - case int16: - newValue.SetInt(int64(val)) - case int32: - newValue.SetInt(int64(val)) - case int64: - newValue.SetInt(val) - case uint: - newValue.SetInt(int64(val)) - case uint8: - newValue.SetInt(int64(val)) - case uint16: - newValue.SetInt(int64(val)) - case uint32: - newValue.SetInt(int64(val)) - case uint64: - newValue.SetInt(int64(val)) - case float32: - newValue.SetDouble(float64(val)) - case float64: - newValue.SetDouble(val) - case time.Time: - newValue.SetStr(val.UTC().Format("2006-01-02T15:04:05.000Z")) - default: - newValue.SetStr(fmt.Sprintf("unknown type: %T", val)) - } - - } -} diff --git a/libbeat/otelbeat/otelmap/otelmap_test.go b/libbeat/otelbeat/otelmap/otelmap_test.go index de28769d816e..9970f48b2875 100644 --- a/libbeat/otelbeat/otelmap/otelmap_test.go +++ b/libbeat/otelbeat/otelmap/otelmap_test.go @@ -124,22 +124,6 @@ func TestFromMapstrSliceInt(t *testing.T) { assert.Equal(t, want, got) } -func TestFromMapstrSliceAny(t *testing.T) { - inputSlice := []any{42, 43, 44} - inputMap := mapstr.M{ - "slice": inputSlice, - } - want := pcommon.NewMap() - sliceOfInt := want.PutEmptySlice("slice") - for _, i := range inputSlice { - val := sliceOfInt.AppendEmpty() - val.SetInt(int64(i.(int))) - } - - got := FromMapstr(inputMap) - assert.Equal(t, want, got) -} - func TestFromMapstrDouble(t *testing.T) { tests := map[string]struct { mapstr_val interface{} diff --git a/x-pack/filebeat/fbreceiver/factory.go b/x-pack/filebeat/fbreceiver/factory.go index 10cae89507f4..b3c12afb8979 100644 --- a/x-pack/filebeat/fbreceiver/factory.go +++ b/x-pack/filebeat/fbreceiver/factory.go @@ -16,7 +16,6 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/include" inputs "github.com/elastic/beats/v7/x-pack/filebeat/input/default-inputs" "github.com/elastic/elastic-agent-libs/mapstr" - "go.uber.org/zap" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" @@ -46,8 +45,6 @@ func createReceiver(_ context.Context, set receiver.Settings, baseCfg component. settings.ElasticLicensed = true settings.Initialize = append(settings.Initialize, include.InitializeModule) - // correct values are extracted from baseCfg - set.Logger = set.Logger.With(zap.String("component.id", "randomID")) b, err := instance.NewBeatReceiver(settings, cfg.Beatconfig, true, consumer, set.Logger.Core()) if err != nil { return nil, fmt.Errorf("error creating %s: %w", Name, err) From 8634ad4a3e3a21eaad60b26300eaecc5ce8c3c8c Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 28 Feb 2025 17:10:58 +0530 Subject: [PATCH 07/15] make docs --- filebeat/filebeat.reference.yml | 3 +++ libbeat/docs/loggingconfig.asciidoc | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 875f3f361cd1..f3b52cbd9165 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2679,6 +2679,9 @@ setup.kibana: # Beats and contains module or input metrics. #logging.metrics.namespaces: [stats] +# A map of key-value pair included in beat process logs for additional context. +#logging.with_fields: + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index f5e85b392fad..761471907d3c 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -98,7 +98,7 @@ ifndef::serverless[] [float] ==== `logging.with_fields` -A map of key-value pair added to beat process logs for additional context. +A map of key-value pair included in beat process logs for additional context. Usage: ``` From 21e25553a3667d9c5c80a9e124c68827fa6516af Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 28 Feb 2025 18:11:05 +0530 Subject: [PATCH 08/15] add changelgo --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 42cf55b071f5..6ea76df48c62 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -313,6 +313,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - Add `uppercase` processor. {issue}22254[22254] {pull}41535[41535] - Replace `compress/gzip` with https://github.com/klauspost/compress/gzip library for gzip compression {pull}41584[41584] - Add regex pattern matching to add_kubernetes_metadata processor {pull}41903[41903] +- Add `logging.with_fields` to the logging configuration. It allows for including additional context to the beat logs *Auditbeat* From f8398348b1eb3f21c4927854f849b66a6bd311d8 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Fri, 28 Feb 2025 18:19:48 +0530 Subject: [PATCH 09/15] update docs --- filebeat/filebeat.reference.yml | 2 +- libbeat/docs/loggingconfig.asciidoc | 2 +- x-pack/filebeat/filebeat.reference.yml | 285 +++++++++++++------------ 3 files changed, 146 insertions(+), 143 deletions(-) diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index f3b52cbd9165..1e4ed1d02fd2 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2679,7 +2679,7 @@ setup.kibana: # Beats and contains module or input metrics. #logging.metrics.namespaces: [stats] -# A map of key-value pair included in beat process logs for additional context. +# A map of key-value pair appended to beat process logs for additional context. #logging.with_fields: # Logging to rotating files. Set logging.to_files to false to disable logging to diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 761471907d3c..747dd20e7c06 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -98,7 +98,7 @@ ifndef::serverless[] [float] ==== `logging.with_fields` -A map of key-value pair included in beat process logs for additional context. +A map of key-value pair appended to beat process logs for additional context. Usage: ``` diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 59c1897634d0..169caa3e6cf3 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1488,147 +1488,147 @@ filebeat.modules: #var.password: #------------------------------ Salesforce Module ------------------------------ -# Configuration file for Salesforce module in Filebeat - -# Common Configurations: -# - enabled: Set to true to enable ingestion of Salesforce module fileset -# - initial_interval: Initial interval for log collection. This setting determines the time period for which the logs will be initially collected when the ingestion process starts, i.e. 1d/h/m/s -# - api_version: API version for Salesforce, version should be greater than 46.0 - -# Authentication Configurations: -# User-Password Authentication: -# - enabled: Set to true to enable user-password authentication -# - client.id: Client ID for user-password authentication -# - client.secret: Client secret for user-password authentication -# - token_url: Token URL for user-password authentication -# - username: Username for user-password authentication -# - password: Password for user-password authentication - -# JWT Authentication: -# - enabled: Set to true to enable JWT authentication -# - client.id: Client ID for JWT authentication -# - client.username: Username for JWT authentication -# - client.key_path: Path to client key for JWT authentication -# - url: Audience URL for JWT authentication - -# Event Monitoring: -# - real_time: Set to true to enable real-time logging using object type data collection -# - real_time_interval: Interval for real-time logging - -# Event Log File: -# - event_log_file: Set to true to enable event log file type data collection -# - elf_interval: Interval for event log file -# - log_file_interval: Interval type for log file collection, either Hourly or Daily - -- module: salesforce - - apex: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.event_log_file: true - var.elf_interval: 1h - var.log_file_interval: "Hourly" - - login: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "client-secret" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.event_log_file: true - var.elf_interval: 1h - var.log_file_interval: "Hourly" - - var.real_time: true - var.real_time_interval: 5m - - logout: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "client-secret" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.event_log_file: true - var.elf_interval: 1h - var.log_file_interval: "Hourly" - - var.real_time: true - var.real_time_interval: 5m - - setupaudittrail: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "client-secret" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.real_time: true +# Configuration file for Salesforce module in Filebeat + +# Common Configurations: +# - enabled: Set to true to enable ingestion of Salesforce module fileset +# - initial_interval: Initial interval for log collection. This setting determines the time period for which the logs will be initially collected when the ingestion process starts, i.e. 1d/h/m/s +# - api_version: API version for Salesforce, version should be greater than 46.0 + +# Authentication Configurations: +# User-Password Authentication: +# - enabled: Set to true to enable user-password authentication +# - client.id: Client ID for user-password authentication +# - client.secret: Client secret for user-password authentication +# - token_url: Token URL for user-password authentication +# - username: Username for user-password authentication +# - password: Password for user-password authentication + +# JWT Authentication: +# - enabled: Set to true to enable JWT authentication +# - client.id: Client ID for JWT authentication +# - client.username: Username for JWT authentication +# - client.key_path: Path to client key for JWT authentication +# - url: Audience URL for JWT authentication + +# Event Monitoring: +# - real_time: Set to true to enable real-time logging using object type data collection +# - real_time_interval: Interval for real-time logging + +# Event Log File: +# - event_log_file: Set to true to enable event log file type data collection +# - elf_interval: Interval for event log file +# - log_file_interval: Interval type for log file collection, either Hourly or Daily + +- module: salesforce + + apex: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.event_log_file: true + var.elf_interval: 1h + var.log_file_interval: "Hourly" + + login: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "client-secret" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.event_log_file: true + var.elf_interval: 1h + var.log_file_interval: "Hourly" + + var.real_time: true + var.real_time_interval: 5m + + logout: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "client-secret" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.event_log_file: true + var.elf_interval: 1h + var.log_file_interval: "Hourly" + + var.real_time: true + var.real_time_interval: 5m + + setupaudittrail: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "client-secret" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.real_time: true var.real_time_interval: 5m #----------------------------- Google Santa Module ----------------------------- - module: santa @@ -4608,6 +4608,9 @@ setup.kibana: # Beats and contains module or input metrics. #logging.metrics.namespaces: [stats] +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: + # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true From 589f641c64f74594e38d36af0ab4d1f2b1d79f1c Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Mon, 3 Mar 2025 11:39:53 +0530 Subject: [PATCH 10/15] make update --- auditbeat/auditbeat.reference.yml | 2 + filebeat/filebeat.reference.yml | 5 +- heartbeat/heartbeat.reference.yml | 2 + .../_meta/config/logging.reference.yml.tmpl | 3 + metricbeat/metricbeat.reference.yml | 2 + packetbeat/packetbeat.reference.yml | 2 + winlogbeat/winlogbeat.reference.yml | 2 + x-pack/auditbeat/auditbeat.reference.yml | 2 + x-pack/filebeat/filebeat.reference.yml | 287 +++++++++--------- x-pack/heartbeat/heartbeat.reference.yml | 2 + x-pack/metricbeat/metricbeat.reference.yml | 2 + x-pack/osquerybeat/osquerybeat.reference.yml | 2 + x-pack/packetbeat/packetbeat.reference.yml | 2 + x-pack/winlogbeat/winlogbeat.reference.yml | 2 + 14 files changed, 170 insertions(+), 147 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 9d4919eb24fa..b8fd1ffdf8ff 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1597,6 +1597,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 1e4ed1d02fd2..0d561b779f98 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2679,9 +2679,6 @@ setup.kibana: # Beats and contains module or input metrics. #logging.metrics.namespaces: [stats] -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: - # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true @@ -2763,6 +2760,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 80272414cbfa..de06c4997248 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1684,6 +1684,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/libbeat/_meta/config/logging.reference.yml.tmpl b/libbeat/_meta/config/logging.reference.yml.tmpl index 612b6d5aa2d0..f520227f3639 100644 --- a/libbeat/_meta/config/logging.reference.yml.tmpl +++ b/libbeat/_meta/config/logging.reference.yml.tmpl @@ -115,3 +115,6 @@ logging.files: # Rotate existing logs on startup rather than appending them to the existing # file. Defaults to false. # rotateonstartup: false + +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: \ No newline at end of file diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 0d7ceb34057c..340b01252f2a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -2545,6 +2545,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 1bbcb7c33253..fc189ddbd62f 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -2063,6 +2063,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 4082b1964e39..c19029a4e034 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1474,6 +1474,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index dd3655c8f720..c66aab60261e 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1661,6 +1661,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 169caa3e6cf3..b5272a7dbeb9 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1488,147 +1488,147 @@ filebeat.modules: #var.password: #------------------------------ Salesforce Module ------------------------------ -# Configuration file for Salesforce module in Filebeat - -# Common Configurations: -# - enabled: Set to true to enable ingestion of Salesforce module fileset -# - initial_interval: Initial interval for log collection. This setting determines the time period for which the logs will be initially collected when the ingestion process starts, i.e. 1d/h/m/s -# - api_version: API version for Salesforce, version should be greater than 46.0 - -# Authentication Configurations: -# User-Password Authentication: -# - enabled: Set to true to enable user-password authentication -# - client.id: Client ID for user-password authentication -# - client.secret: Client secret for user-password authentication -# - token_url: Token URL for user-password authentication -# - username: Username for user-password authentication -# - password: Password for user-password authentication - -# JWT Authentication: -# - enabled: Set to true to enable JWT authentication -# - client.id: Client ID for JWT authentication -# - client.username: Username for JWT authentication -# - client.key_path: Path to client key for JWT authentication -# - url: Audience URL for JWT authentication - -# Event Monitoring: -# - real_time: Set to true to enable real-time logging using object type data collection -# - real_time_interval: Interval for real-time logging - -# Event Log File: -# - event_log_file: Set to true to enable event log file type data collection -# - elf_interval: Interval for event log file -# - log_file_interval: Interval type for log file collection, either Hourly or Daily - -- module: salesforce - - apex: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.event_log_file: true - var.elf_interval: 1h - var.log_file_interval: "Hourly" - - login: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "client-secret" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.event_log_file: true - var.elf_interval: 1h - var.log_file_interval: "Hourly" - - var.real_time: true - var.real_time_interval: 5m - - logout: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "client-secret" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.event_log_file: true - var.elf_interval: 1h - var.log_file_interval: "Hourly" - - var.real_time: true - var.real_time_interval: 5m - - setupaudittrail: - enabled: false - var.initial_interval: 1d - var.api_version: 56 - - var.authentication: - user_password_flow: - enabled: true - client.id: "" - client.secret: "client-secret" - token_url: "" - username: "" - password: "" - jwt_bearer_flow: - enabled: false - client.id: "" - client.username: "" - client.key_path: "" - url: "https://login.salesforce.com" - - var.url: "https://instance_id.my.salesforce.com" - - var.real_time: true +# Configuration file for Salesforce module in Filebeat + +# Common Configurations: +# - enabled: Set to true to enable ingestion of Salesforce module fileset +# - initial_interval: Initial interval for log collection. This setting determines the time period for which the logs will be initially collected when the ingestion process starts, i.e. 1d/h/m/s +# - api_version: API version for Salesforce, version should be greater than 46.0 + +# Authentication Configurations: +# User-Password Authentication: +# - enabled: Set to true to enable user-password authentication +# - client.id: Client ID for user-password authentication +# - client.secret: Client secret for user-password authentication +# - token_url: Token URL for user-password authentication +# - username: Username for user-password authentication +# - password: Password for user-password authentication + +# JWT Authentication: +# - enabled: Set to true to enable JWT authentication +# - client.id: Client ID for JWT authentication +# - client.username: Username for JWT authentication +# - client.key_path: Path to client key for JWT authentication +# - url: Audience URL for JWT authentication + +# Event Monitoring: +# - real_time: Set to true to enable real-time logging using object type data collection +# - real_time_interval: Interval for real-time logging + +# Event Log File: +# - event_log_file: Set to true to enable event log file type data collection +# - elf_interval: Interval for event log file +# - log_file_interval: Interval type for log file collection, either Hourly or Daily + +- module: salesforce + + apex: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.event_log_file: true + var.elf_interval: 1h + var.log_file_interval: "Hourly" + + login: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "client-secret" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.event_log_file: true + var.elf_interval: 1h + var.log_file_interval: "Hourly" + + var.real_time: true + var.real_time_interval: 5m + + logout: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "client-secret" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.event_log_file: true + var.elf_interval: 1h + var.log_file_interval: "Hourly" + + var.real_time: true + var.real_time_interval: 5m + + setupaudittrail: + enabled: false + var.initial_interval: 1d + var.api_version: 56 + + var.authentication: + user_password_flow: + enabled: true + client.id: "" + client.secret: "client-secret" + token_url: "" + username: "" + password: "" + jwt_bearer_flow: + enabled: false + client.id: "" + client.username: "" + client.key_path: "" + url: "https://login.salesforce.com" + + var.url: "https://instance_id.my.salesforce.com" + + var.real_time: true var.real_time_interval: 5m #----------------------------- Google Santa Module ----------------------------- - module: santa @@ -4608,9 +4608,6 @@ setup.kibana: # Beats and contains module or input metrics. #logging.metrics.namespaces: [stats] -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: - # Logging to rotating files. Set logging.to_files to false to disable logging to # files. logging.to_files: true @@ -4692,6 +4689,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index 80272414cbfa..de06c4997248 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1684,6 +1684,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index dfb8246bc5ea..4bd6a223ffb3 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -3176,6 +3176,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 16c879aabc51..5bb7cdc8465d 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -1031,6 +1031,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Osquerybeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index 1bbcb7c33253..fc189ddbd62f 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -2063,6 +2063,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 7fedac688041..7205f66c36a6 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1476,6 +1476,8 @@ logging.files: # file. Defaults to false. # rotateonstartup: false +# A map of key-value pair appended to beat process logs for additional context. +#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The From c9a394d001a77afe858ca2524df7fb235e7425cf Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Tue, 4 Mar 2025 18:35:32 +0530 Subject: [PATCH 11/15] remove it from public facing config --- auditbeat/auditbeat.reference.yml | 2 -- filebeat/filebeat.reference.yml | 2 -- heartbeat/heartbeat.reference.yml | 2 -- libbeat/_meta/config/logging.reference.yml.tmpl | 3 --- libbeat/docs/loggingconfig.asciidoc | 15 --------------- metricbeat/metricbeat.reference.yml | 2 -- packetbeat/packetbeat.reference.yml | 2 -- winlogbeat/winlogbeat.reference.yml | 2 -- x-pack/auditbeat/auditbeat.reference.yml | 2 -- x-pack/filebeat/filebeat-otel.yml | 10 +++++----- x-pack/filebeat/filebeat.reference.yml | 2 -- x-pack/heartbeat/heartbeat.reference.yml | 2 -- x-pack/metricbeat/metricbeat.reference.yml | 2 -- x-pack/osquerybeat/osquerybeat.reference.yml | 2 -- x-pack/packetbeat/packetbeat.reference.yml | 2 -- x-pack/winlogbeat/winlogbeat.reference.yml | 2 -- 16 files changed, 5 insertions(+), 49 deletions(-) diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 8d6465db918a..c54679448e90 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -1601,8 +1601,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 0d561b779f98..875f3f361cd1 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -2760,8 +2760,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index de06c4997248..80272414cbfa 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -1684,8 +1684,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/libbeat/_meta/config/logging.reference.yml.tmpl b/libbeat/_meta/config/logging.reference.yml.tmpl index f520227f3639..612b6d5aa2d0 100644 --- a/libbeat/_meta/config/logging.reference.yml.tmpl +++ b/libbeat/_meta/config/logging.reference.yml.tmpl @@ -115,6 +115,3 @@ logging.files: # Rotate existing logs on startup rather than appending them to the existing # file. Defaults to false. # rotateonstartup: false - -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: \ No newline at end of file diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 747dd20e7c06..dd3a70811713 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -94,21 +94,6 @@ ifndef::serverless[] When true, writes all logging output to standard error output. This is equivalent to using the `-e` command line option. -ifndef::serverless[] -[float] -==== `logging.with_fields` - -A map of key-value pair appended to beat process logs for additional context. - -Usage: -``` -logging.with_fields: - id: 123 - data_stream: - dataset: beat - namespace: default -``` - [float] ==== `logging.to_syslog` diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 340b01252f2a..0d7ceb34057c 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -2545,8 +2545,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index fc189ddbd62f..1bbcb7c33253 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -2063,8 +2063,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index c19029a4e034..4082b1964e39 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -1474,8 +1474,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index be545eb74dd6..6bc035a3efd0 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -1665,8 +1665,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Auditbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/filebeat/filebeat-otel.yml b/x-pack/filebeat/filebeat-otel.yml index aef67af61f52..c3e0d7c1891d 100644 --- a/x-pack/filebeat/filebeat-otel.yml +++ b/x-pack/filebeat/filebeat-otel.yml @@ -3,15 +3,15 @@ filebeat.inputs: id: filestream-input-id enabled: true paths: - - /tmp/flog.log + - ./test.log output: elasticsearch: - hosts: ["https://localhost:9200"] - username: elastic - password: changeme + hosts: ["http://localhost:9200"] + username: admin + password: testing setup.kibana: - host: https://localhost:5601 + host: http://localhost:5601 username: elastic password: changeme diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index b5272a7dbeb9..59c1897634d0 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -4689,8 +4689,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Filebeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml index de06c4997248..80272414cbfa 100644 --- a/x-pack/heartbeat/heartbeat.reference.yml +++ b/x-pack/heartbeat/heartbeat.reference.yml @@ -1684,8 +1684,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Heartbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1833351c76a8..40aeb4742c17 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -3167,8 +3167,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Metricbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml index 5bb7cdc8465d..16c879aabc51 100644 --- a/x-pack/osquerybeat/osquerybeat.reference.yml +++ b/x-pack/osquerybeat/osquerybeat.reference.yml @@ -1031,8 +1031,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Osquerybeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index fc189ddbd62f..1bbcb7c33253 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -2063,8 +2063,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Packetbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml index 7205f66c36a6..7fedac688041 100644 --- a/x-pack/winlogbeat/winlogbeat.reference.yml +++ b/x-pack/winlogbeat/winlogbeat.reference.yml @@ -1476,8 +1476,6 @@ logging.files: # file. Defaults to false. # rotateonstartup: false -# A map of key-value pair appended to beat process logs for additional context. -#logging.with_fields: # ============================= X-Pack Monitoring ============================== # Winlogbeat can export internal metrics to a central Elasticsearch monitoring # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The From 2765f2dffc27805b96dec6f3f4f994b1f802292f Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Tue, 4 Mar 2025 18:36:37 +0530 Subject: [PATCH 12/15] remove changelog --- CHANGELOG.next.asciidoc | 1 - x-pack/filebeat/filebeat-otel.yml | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c0224a6eca51..8234289871a3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -314,7 +314,6 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - Add `uppercase` processor. {issue}22254[22254] {pull}41535[41535] - Replace `compress/gzip` with https://github.com/klauspost/compress/gzip library for gzip compression {pull}41584[41584] - Add regex pattern matching to add_kubernetes_metadata processor {pull}41903[41903] -- Add `logging.with_fields` to the logging configuration. It allows for including additional context to the beat logs *Auditbeat* diff --git a/x-pack/filebeat/filebeat-otel.yml b/x-pack/filebeat/filebeat-otel.yml index c3e0d7c1891d..aef67af61f52 100644 --- a/x-pack/filebeat/filebeat-otel.yml +++ b/x-pack/filebeat/filebeat-otel.yml @@ -3,15 +3,15 @@ filebeat.inputs: id: filestream-input-id enabled: true paths: - - ./test.log + - /tmp/flog.log output: elasticsearch: - hosts: ["http://localhost:9200"] - username: admin - password: testing + hosts: ["https://localhost:9200"] + username: elastic + password: changeme setup.kibana: - host: http://localhost:5601 + host: https://localhost:5601 username: elastic password: changeme From 528ea82e9b4044579713c719239f4c3758c6b81e Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Wed, 5 Mar 2025 13:56:37 +0530 Subject: [PATCH 13/15] update notice --- NOTICE.txt | 170 ++++++++++++++++++++++++++--------------------------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index afcac2cbd02b..d1600a494186 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -20860,27 +20860,27 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/microsoft/wmi@v0.25.1/LICENSE: - MIT License - - Copyright (c) Microsoft Corporation. All rights reserved. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE -------------------------------------------------------------------------------- @@ -32824,28 +32824,28 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/!azure/go-amqp@v1.3.0/LICENSE: - MIT License - - Copyright (C) 2017 Kale Blankenship - Portions Copyright (C) Microsoft Corporation - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE + MIT License + + Copyright (C) 2017 Kale Blankenship + Portions Copyright (C) Microsoft Corporation + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE -------------------------------------------------------------------------------- @@ -34557,27 +34557,27 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/!azure!a!d/microsoft-authentication-library-for-go@v1.3.3/LICENSE: - MIT License - - Copyright (c) Microsoft Corporation. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE -------------------------------------------------------------------------------- @@ -35507,27 +35507,27 @@ Licence type (autodetected): MIT Contents of probable licence file $GOMODCACHE/github.com/akavel/rsrc@v0.8.0/LICENSE.txt: -The MIT License (MIT) - -Copyright (c) 2013-2017 The rsrc Authors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +The MIT License (MIT) + +Copyright (c) 2013-2017 The rsrc Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -------------------------------------------------------------------------------- From c6d61140efe43150e7636866342bc63d74f5801c Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Wed, 5 Mar 2025 13:57:07 +0530 Subject: [PATCH 14/15] add test --- .../filebeat/tests/integration/otel_test.go | 59 ++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/tests/integration/otel_test.go b/x-pack/filebeat/tests/integration/otel_test.go index 7a147add0dbe..2ffc78a283b8 100644 --- a/x-pack/filebeat/tests/integration/otel_test.go +++ b/x-pack/filebeat/tests/integration/otel_test.go @@ -7,12 +7,14 @@ package integration import ( + "bufio" "context" "crypto/tls" "fmt" "net/http" "os" "path/filepath" + "strings" "testing" "time" @@ -41,7 +43,6 @@ output: - localhost:9200 protocol: http username: admin - password: testing index: %s queue.mem.flush.timeout: 0s processors: @@ -170,3 +171,59 @@ func assertMapsEqual(t *testing.T, m1, m2 mapstr.M, ignoredFields []string, msg } require.Equal(t, "", cmp.Diff(flatM1, flatM2), "expected maps to be equal") } + +// This test ensures the functionality of `logging.with_fields` on beatreceivers +func TestBeatReceiverLoggingWithFields(t *testing.T) { + var filebeatBasicConfig = ` +filebeat.inputs: + - type: filestream + id: filestream-input-id + enabled: true + paths: + - /tmp/flog.log +output: + elasticsearch: + hosts: + - localhost:9200 + protocol: http + username: admin + password: testing +logging.with_fields: + component: filebeat-otel-test +path.home: %s + +` + // start filebeat in otel mode + filebeatOTel := integration.NewBeat( + t, + "filebeat-otel", + "../../filebeat.test", + "otel", + ) + + tempDir := filebeatOTel.TempDir() + s := fmt.Sprintf(filebeatBasicConfig, tempDir) + filebeatOTel.WriteConfigFile(s) + filebeatOTel.Start() + + require.Eventually(t, func() bool { + // we must open it each time in case no logs have arrived yet + f, err := os.Open(filepath.Join(tempDir, "stderr")) + if err != nil { + return false + } + defer f.Close() + + r := bufio.NewScanner(f) + + for r.Scan() { + line := r.Text() + if strings.Contains(line, `"name": "filebeatreceiver"`) { + fmt.Println(line) + return assert.True(t, strings.Contains(line, `"component": "filebeat-otel-test"`)) + } + } + return false + }, 10*time.Second, 100*time.Millisecond, "additional logging context was not added") + +} From e65782328ff6bcd55e8c0fc8ab23cbd0cfe63ae8 Mon Sep 17 00:00:00 2001 From: Khushi Jain Date: Wed, 5 Mar 2025 17:58:48 +0530 Subject: [PATCH 15/15] fix test --- x-pack/filebeat/tests/integration/otel_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/filebeat/tests/integration/otel_test.go b/x-pack/filebeat/tests/integration/otel_test.go index 2ffc78a283b8..cf7bb4d5932b 100644 --- a/x-pack/filebeat/tests/integration/otel_test.go +++ b/x-pack/filebeat/tests/integration/otel_test.go @@ -42,6 +42,7 @@ output: hosts: - localhost:9200 protocol: http + password: testing username: admin index: %s queue.mem.flush.timeout: 0s