From 2c6e5affa25bd29bda0e56d506074967cda28f7d Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 6 Mar 2025 10:35:27 +0200 Subject: [PATCH 1/2] Don't package arm64 on amd64 workers (#43026) This commit is the counterpart of #43019 for the DRA packaging pipeline. It ensures that arm64 packages get built without qemu emulation on dedicated arm64 workers. --- .buildkite/packaging.pipeline.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 34b103b23736..6063cd8c7164 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -8,8 +8,8 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-standard-8" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - PLATFORMS_ARM: "linux/arm64" + PLATFORMS: "+all linux/amd64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS_ARM: "+all linux/arm64" steps: # we use concurrency gates (https://buildkite.com/blog/concurrency-gates) @@ -121,10 +121,9 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" + - label: "SNAPSHOT: {{matrix}} Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" - PACKAGES: "docker" SNAPSHOT: true # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270 DEV: false @@ -151,9 +150,11 @@ steps: - x-pack/heartbeat - x-pack/metricbeat - x-pack/packetbeat + - x-pack/osquerybeat + - x-pack/agentbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: "SNAPSHOT: x-pack/agentbeat" + - label: "SNAPSHOT: x-pack/agentbeat all artifacts apart from linux/arm64" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: true @@ -211,10 +212,9 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - - label: "STAGING: {{matrix}} docker Linux/arm64" + - label: "STAGING: {{matrix}} Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" - PACKAGES: "docker" SNAPSHOT: false DEV: false command: | @@ -242,9 +242,11 @@ steps: - x-pack/heartbeat - x-pack/metricbeat - x-pack/packetbeat + - x-pack/osquerybeat + - x-pack/agentbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: "STAGING: x-pack/agentbeat" + - label: "STAGING: x-pack/agentbeat all artifacts apart from linux/arm64" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false From 180bd96836d503887fdb9044555b46521bc228d9 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 6 Mar 2025 11:01:38 +0100 Subject: [PATCH 2/2] [metricbeat] Add a new 'match_by_parent_instance' option to 'perfmon' module (#43002) * Add a new 'match_by_parent_instance' option to 'perfmon' module * Add type assertion check --- CHANGELOG.next.asciidoc | 1 + .../windows/perfmon/_meta/docs.asciidoc | 7 ++ metricbeat/module/windows/perfmon/config.go | 5 ++ metricbeat/module/windows/perfmon/data.go | 20 ++--- .../module/windows/perfmon/data_test.go | 76 +++++++++++++++++++ 5 files changed, 100 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f04e8e0d6d41..79dda48534a1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -473,6 +473,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - Add new metricset wmi for the windows module. {pull}42017[42017] - Update beat module with apm-server tail sampling monitoring metrics fields {pull}42569[42569] - Log every 401 response from Kubernetes API Server {pull}42714[42714] +- Add a new `match_by_parent_instance` option to `perfmon` module. {pull}43002[43002] *Metricbeat* - Add benchmark module {pull}41801[41801] diff --git a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc index 9550e771ebc7..9df8c740fb49 100644 --- a/metricbeat/module/windows/perfmon/_meta/docs.asciidoc +++ b/metricbeat/module/windows/perfmon/_meta/docs.asciidoc @@ -49,6 +49,13 @@ The default behaviour is for all measurements to be sent as separate events. *`refresh_wildcard_counters`*:: A boolean option to refresh the counter list at each fetch. By default, the counter list will be retrieved at the starting time, to refresh the list at each fetch, users will have to enable this setting. +*`match_by_parent_instance`*:: A boolean option that causes all instances +of the same parent to have the same instance value. In the above example, +this will cause metrics for `svchost`, `svchost#1`, etc. to have an `instance` +value of `svchost`. If set to `false` they will keep the original values +(`svchost`, `svchost#1`, etc.). +It defaults to `true`. + [float] ==== Query Configuration diff --git a/metricbeat/module/windows/perfmon/config.go b/metricbeat/module/windows/perfmon/config.go index f16c9c3b324b..4033fc404aeb 100644 --- a/metricbeat/module/windows/perfmon/config.go +++ b/metricbeat/module/windows/perfmon/config.go @@ -35,6 +35,7 @@ type Config struct { RefreshWildcardCounters bool `config:"perfmon.refresh_wildcard_counters"` Queries []Query `config:"perfmon.queries"` GroupAllCountersTo string `config:"perfmon.group_all_counter"` + MatchByParentInstance *bool `config:"perfmon.match_by_parent_instance"` } // QueryConfig for perfmon queries. This will be used as the new configuration format @@ -77,6 +78,10 @@ func (conf *Config) Validate() error { return nil } +func (conf *Config) ShouldMatchByParentInstance() bool { + return conf.MatchByParentInstance == nil || *conf.MatchByParentInstance +} + func isValidFormat(format string) bool { for _, form := range allowedFormats { if form == format { diff --git a/metricbeat/module/windows/perfmon/data.go b/metricbeat/module/windows/perfmon/data.go index 0391266e65a5..5e35c9ae4e1b 100644 --- a/metricbeat/module/windows/perfmon/data.go +++ b/metricbeat/module/windows/perfmon/data.go @@ -75,12 +75,14 @@ func (re *Reader) groupToEvents(counters map[string][]pdh.CounterValue) []mb.Eve eventMap[eventKey].Error = fmt.Errorf("failed on query=%v: %w", counterPath, val.Err.Error) } if val.Instance != "" { - // will ignore instance index - if ok, match := matchesParentProcess(val.Instance); ok { - eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, match) - } else { - eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, val.Instance) + instanceName := val.Instance + if re.config.ShouldMatchByParentInstance() { + if ok, match := matchesParentProcess(val.Instance); ok { + // will ignore instance index + instanceName = match + } } + eventMap[eventKey].MetricSetFields.Put(counter.InstanceField, instanceName) } } @@ -126,13 +128,13 @@ func (re *Reader) groupToSingleEvent(counters map[string][]pdh.CounterValue) mb. continue } var counterVal float64 - switch val.Measurement.(type) { + switch m := val.Measurement.(type) { case int64: - counterVal = float64(val.Measurement.(int64)) + counterVal = float64(m) case int: - counterVal = float64(val.Measurement.(int)) + counterVal = float64(m) default: - counterVal = val.Measurement.(float64) + counterVal, _ = val.Measurement.(float64) } if _, ok := measurements[readerCounter.QueryField]; !ok { measurements[readerCounter.QueryField] = counterVal diff --git a/metricbeat/module/windows/perfmon/data_test.go b/metricbeat/module/windows/perfmon/data_test.go index 9c4691216b34..fdb6683e897e 100644 --- a/metricbeat/module/windows/perfmon/data_test.go +++ b/metricbeat/module/windows/perfmon/data_test.go @@ -292,6 +292,82 @@ func TestGroupToSingleEvent(t *testing.T) { assert.Equal(t, val, mapstr.M{"processor_count": float64(2)}) } +func TestMatchByParentInstance(t *testing.T) { + _true := true + _false := false + reader := Reader{ + query: pdh.Query{}, + log: nil, + config: Config{ + MatchByParentInstance: &_true, + }, + counters: []PerfCounter{ + { + QueryField: "%_processor_time", + QueryName: `\Processor Information(*)\% Processor Time`, + Format: "float", + ObjectName: "Processor Information", + ObjectField: "object", + InstanceName: "*", + InstanceField: "instance", + ChildQueries: []string{`\Processor Information(processor)\% Processor Time`, `\Processor Information(processor#1)\% Processor Time`}, + }, + }, + } + + counters := map[string][]pdh.CounterValue{ + `\Processor Information(processor)\% Processor Time`: { + { + Instance: "processor", + Measurement: 23, + }, + }, + `\Processor Information(processor#1)\% Processor Time`: { + { + Instance: "processor#1", + Measurement: 21, + }, + }, + } + + { + events := reader.groupToEvents(counters) + assert.NotNil(t, events) + assert.Equal(t, 2, len(events)) + ok, err := events[0].MetricSetFields.HasKey("instance") + assert.NoError(t, err) + assert.True(t, ok) + ok, err = events[1].MetricSetFields.HasKey("instance") + assert.NoError(t, err) + assert.True(t, ok) + val1, err := events[0].MetricSetFields.GetValue("instance") + assert.NoError(t, err) + assert.Equal(t, val1, "processor") + val2, err := events[1].MetricSetFields.GetValue("instance") + assert.NoError(t, err) + assert.Equal(t, val2, "processor") + } + + reader.config.MatchByParentInstance = &_false + { + events := reader.groupToEvents(counters) + assert.NotNil(t, events) + assert.Equal(t, 2, len(events)) + ok, err := events[0].MetricSetFields.HasKey("instance") + assert.NoError(t, err) + assert.True(t, ok) + ok, err = events[1].MetricSetFields.HasKey("instance") + assert.NoError(t, err) + assert.True(t, ok) + val1, err := events[0].MetricSetFields.GetValue("instance") + assert.NoError(t, err) + assert.Equal(t, val1, "processor") + val2, err := events[1].MetricSetFields.GetValue("instance") + assert.NoError(t, err) + assert.Equal(t, val2, "processor#1") + } +} + func TestMatchesParentProcess(t *testing.T) { ok, val := matchesParentProcess("svchost") assert.True(t, ok)