Merge branch 'elastic:main' into main

agithomas · Jan 22, 2024 · 1cf3e15 · 1cf3e15
2 parents 3adc4e5 + 7494535
commit 1cf3e15
Show file tree

Hide file tree

Showing 155 changed files with 18,152 additions and 5,941 deletions.
diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - label: "Example test"
+    command: echo "Hello!"
diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - label: "Example test"
+    command: echo "Hello!"
diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - label: "Example test"
+    command: echo "Hello!"
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
@@ -47,6 +47,54 @@
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": [ ],
             "always_require_ci_on_changed": [ "^metricbeat/.*", ".buildkite/metricbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
+        },
+        {
+            "enabled": true,
+            "pipelineSlug": "auditbeat",
+            "allow_org_users": true,
+            "allowed_repo_permissions": ["admin", "write"],
+            "allowed_list": [ ],
+            "set_commit_status": true,
+            "build_on_commit": true,
+            "build_on_comment": true,
+            "trigger_comment_regex": "^/test auditbeat$",
+            "always_trigger_comment_regex": "^/test auditbeat$",
+            "skip_ci_labels": [  ],
+            "skip_target_branches": [ ],
+            "skip_ci_on_only_changed": [ ],
+            "always_require_ci_on_changed": [ "^auditbeat/.*", ".buildkite/auditbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
+        },
+        {
+            "enabled": true,
+            "pipelineSlug": "heartbeat",
+            "allow_org_users": true,
+            "allowed_repo_permissions": ["admin", "write"],
+            "allowed_list": [ ],
+            "set_commit_status": true,
+            "build_on_commit": true,
+            "build_on_comment": true,
+            "trigger_comment_regex": "^/test heartbeat$",
+            "always_trigger_comment_regex": "^/test heartbeat$",
+            "skip_ci_labels": [  ],
+            "skip_target_branches": [ ],
+            "skip_ci_on_only_changed": [ ],
+            "always_require_ci_on_changed": [ "^heartbeat/.*", ".buildkite/heartbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
+        },
+        {
+            "enabled": true,
+            "pipelineSlug": "deploy-k8s",
+            "allow_org_users": true,
+            "allowed_repo_permissions": ["admin", "write"],
+            "allowed_list": [ ],
+            "set_commit_status": true,
+            "build_on_commit": true,
+            "build_on_comment": true,
+            "trigger_comment_regex": "^/test deploy/kubernetes$",
+            "always_trigger_comment_regex": "^/test deploy/kubernetes$",
+            "skip_ci_labels": [  ],
+            "skip_target_branches": [ ],
+            "skip_ci_on_only_changed": [ ],
+            "always_require_ci_on_changed": [ "^deploy/kubernetes/.*", ".buildkite/deploy/kubernetes/.*", "^libbeat/docs/version.asciidoc"]
         }
     ]
 }
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -5,7 +5,7 @@
 {
 	"name": "Beats Development Container",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/go:1-1.20-bullseye",
+	"image": "mcr.microsoft.com/devcontainers/go:1-1.21-bullseye",
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	"features": {
 		"ghcr.io/devcontainers/features/node:1": {},

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -24,33 +24,33 @@ CHANGELOG*
 /deploy/ @elastic/elastic-agent-data-plane
 /deploy/kubernetes @elastic/elastic-agent-data-plane @elastic/obs-cloudnative-monitoring
 /dev-tools/ @elastic/elastic-agent-data-plane
+/dev-tools/kubernetes @elastic/obs-ds-hosted-services
 /docs/ @elastic/elastic-agent-data-plane
 /filebeat @elastic/elastic-agent-data-plane
 /filebeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
 /filebeat/input/syslog/ @elastic/security-external-integrations
 /filebeat/input/winlog/ @elastic/security-external-integrations
-/filebeat/module/ @elastic/integrations
 /filebeat/module/apache @elastic/obs-infraobs-integrations
 /filebeat/module/auditd @elastic/security-external-integrations
 /filebeat/module/elasticsearch/ @elastic/infra-monitoring-ui
 /filebeat/module/haproxy @elastic/obs-infraobs-integrations
-/filebeat/module/icinga @elastic/integrations
+/filebeat/module/icinga # TODO: find right team
 /filebeat/module/iis @elastic/obs-infraobs-integrations
 /filebeat/module/kafka @elastic/obs-infraobs-integrations
 /filebeat/module/kibana/ @elastic/infra-monitoring-ui
 /filebeat/module/logstash/ @elastic/infra-monitoring-ui
 /filebeat/module/mongodb @elastic/obs-infraobs-integrations
 /filebeat/module/mysql @elastic/security-external-integrations
-/filebeat/module/nats @elastic/integrations
+/filebeat/module/nats @elastic/obs-infraobs-integrations
 /filebeat/module/nginx @elastic/obs-infraobs-integrations
 /filebeat/module/osquery @elastic/security-external-integrations
 /filebeat/module/pensando @elastic/security-external-integrations
 /filebeat/module/postgresql @elastic/obs-infraobs-integrations
 /filebeat/module/redis @elastic/obs-infraobs-integrations
 /filebeat/module/santa @elastic/security-external-integrations
 /filebeat/module/system @elastic/elastic-agent-data-plane
-/filebeat/module/traefik @elastic/integrations
-/heartbeat/ @elastic/hosted-services
+/filebeat/module/traefik # TODO: find right team
+/heartbeat/ @elastic/obs-ds-hosted-services
 /journalbeat @elastic/elastic-agent-data-plane
 /libbeat/ @elastic/elastic-agent-data-plane
 /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs
@@ -68,7 +68,6 @@ CHANGELOG*
 /metricbeat/ @elastic/elastic-agent-data-plane
 /metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
 /metricbeat/helper/kubernetes @elastic/obs-cloudnative-monitoring
-/metricbeat/module/ @elastic/integrations
 /metricbeat/module/apache @elastic/obs-infraobs-integrations
 /metricbeat/module/beat/ @elastic/infra-monitoring-ui
 /metricbeat/module/ceph @elastic/obs-infraobs-integrations
@@ -122,7 +121,6 @@ CHANGELOG*
 /x-pack/filebeat/input/lumberjack/ @elastic/security-external-integrations
 /x-pack/filebeat/input/netflow/ @elastic/security-external-integrations
 /x-pack/filebeat/input/o365audit/ @elastic/security-external-integrations
-/x-pack/filebeat/module/ @elastic/integrations
 /x-pack/filebeat/module/activemq @elastic/obs-infraobs-integrations
 /x-pack/filebeat/module/aws @elastic/obs-cloud-monitoring
 /x-pack/filebeat/module/awsfargate @elastic/obs-cloud-monitoring
@@ -174,17 +172,39 @@ CHANGELOG*
 /x-pack/filebeat/module/zscaler @elastic/security-external-integrations
 /x-pack/filebeat/modules.d/zoom.yml.disabled @elastic/security-external-integrations
 /x-pack/filebeat/processors/decode_cef/ @elastic/security-external-integrations
-/x-pack/heartbeat/ @elastic/hosted-services
+/x-pack/heartbeat/ @elastic/obs-ds-hosted-services
 /x-pack/metricbeat/ @elastic/elastic-agent-data-plane
 /x-pack/metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
-/x-pack/metricbeat/module/ @elastic/integrations
 /x-pack/metricbeat/module/activemq @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/airflow @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/aws @elastic/obs-ds-hosted-services
+/x-pack/metricbeat/module/awsfargate @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/azure @elastic/obs-ds-hosted-services
+/x-pack/metricbeat/module/azure/billing @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/cloudfoundry @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/cockroachdb @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/containerd/ @elastic/obs-cloudnative-monitoring
 /x-pack/metricbeat/module/coredns @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/enterprisesearch @elastic/ent-search-application-backend
+/x-pack/metricbeat/module/gcp @elastic/obs-ds-hosted-services @elastic/obs-infraobs-integrations @elastic/security-external-integrations
+/x-pack/metricbeat/module/gcp/billing @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/cloudrun_metrics @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/cloudsql_mysql @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/cloudsql_postgressql @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/cloudsql_sqlserver @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/carbon @elastic/obs-ds-hosted-services
+/x-pack/metricbeat/module/gcp/compute @elastic/obs-ds-hosted-services
+/x-pack/metricbeat/module/gcp/dataproc @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/dns @elastic/security-external-integrations
+/x-pack/metricbeat/module/gcp/firestore @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/firewall @elastic/security-external-integrations
+/x-pack/metricbeat/module/gcp/gke @elastic/obs-ds-hosted-services
+/x-pack/metricbeat/module/gcp/loadbalancing_logs @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/loadbalancing_metrics @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/pubsub @elastic/obs-ds-hosted-services
+/x-pack/metricbeat/module/gcp/redis @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/gcp/storage @elastic/obs-ds-hosted-services
+/x-pack/metricbeat/module/gcp/vpcflow @elastic/security-external-integrations
 /x-pack/metricbeat/module/ibmmq @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/iis @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/istio/ @elastic/obs-cloudnative-monitoring

diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml
@@ -39,7 +39,7 @@ jobs:
         uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.51.2
+          version: v1.55.2
 
           # Give the job more time to execute.
           # Regarding `--whole-files`, the linter is supposed to support linting of changed a patch only but,

diff --git a/.go-version b/.go-version
@@ -1 +1 @@
-1.20.11
+1.21.6
diff --git a/.golangci.yml b/.golangci.yml
@@ -2,7 +2,7 @@
 run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m
   timeout: 15m
-  build-tags: 
+  build-tags:
     - synthetics
     - integration
 
@@ -114,7 +114,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.11"
+    go: "1.21.6"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -132,19 +132,19 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.11"
+    go: "1.21.6"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.11"
+    go: "1.21.6"
     # Disabled:
     # ST1005: error strings should not be capitalized
     checks: ["all", "-ST1005"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.20.11"
+    go: "1.21.6"
 
   gosec:
     excludes:

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -3,6 +3,118 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.12.0]]
+=== Beats version 8.12.0
+https://github.com/elastic/beats/compare/v8.11.4\...v8.12.0[View commits]
+
+==== Breaking changes
+
+*Heartbeat*
+- Decrease the ES default timeout to 10 for the load monitor state requests.
+
+*Osquerybeat*
+
+- Upgrade to osquery 5.10.2. {pull}37115[37115]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Add validation to the `http_endpoint` config for empty URL. {pull}36816[36816] {issue}36772[36772]
+- Fix merging of array fields (processors, paths, parsers) in configurations generated from hints and default config. {issue}36838[36838] {pull}36857[36857]
+
+==== Added
+
+*Affecting all Beats*
+
+- Allow `queue` configuration settings to be set under the output. {issue}35615[35615] {pull}36788[36788]
+- Raise up logging level to warning when attempting to configure {beats} with unknown fields from autodiscovered events/environments.
+- Elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843]
+- Upgrade to Go 1.20.12. {pull}37350[37350]
+- The Elasticsearch output can now configure performance presets with the `preset` configuration field. {pull}37259[37259]
+- Upgrade `elastic-agent-system-metrics` to v0.9.1. See https://github.com/elastic/elastic-agent-system-metrics/releases/tag/v0.9.1. {pull}37353[37353]
+- Upgrade to elastic-agent-libs v0.7.3 and golang.org/x/crypto v0.17.0. {pull}37544[37544]
+
+*Auditbeat*
+
+- Add `ignore_errors` option to audit module. {issue}15768[15768] {pull}36851[36851]
+- Fix copy arguments for strict aligned architectures. {pull}36976[36976]
+
+*Filebeat*
+
+- Allow http_endpoint input to receive PUT and PATCH requests. {pull}36734[36734]
+- Avoid unwanted publication of Azure entity records. {pull}36753[36753]
+- Avoid unwanted publication of Okta entity records. {pull}36770[36770]
+- Add support for Digest Authentication to CEL input. {issue}35514[35514] {pull}36932[36932]
+- Use filestream input with `file_identity.fingerprint` as default for hints autodiscover. {issue}35984[35984] {pull}36950[36950]
+- Add network processor in addition to interface based direction resolution. {pull}37023[37023]
+- Make CEL input log current transaction ID when request tracing is turned on. {pull}37065[37065]
+- Make Azure Blob Storage input GA and update docs accordingly. {pull}37128[37128]
+- Add request trace logging to http_endpoint input. {issue}36951[36951] {pull}36957[36957]
+- Make GCS input GA and update docs accordingly. {pull}37127[37127]
+- Suppress and log max HTTP request retry errors in CEL input. {pull}37160[37160]
+- Prevent CEL input from re-entering the eval loop when an evaluation failed. {pull}37161[37161]
+- Update CEL extensions library to v1.7.0. {pull}37172[37172]
+
+*Auditbeat*
+
+- Upgrade go-libaudit to v2.4.0. {issue}36776[36776] {pull}36964[36964]
+- Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each dataset instance. {pull}36971[36971]
+
+*Heartbeat*
+- Capture and log the individual connection metrics for all the lightweight monitors.
+
+*Metricbeat*
+
+- Add metrics grouping by dimensions and time to Azure app insights. {pull}36634[36634]
+- Align on the algorithm used to transform Prometheus histograms into Elasticsearch histograms. {pull}36647[36647]
+- Enhance GCP billing with detailed tables identification, additional fields, and optimized data handling. {pull}36902[36902]
+- Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each metricset instance. {pull}36971[36971]
+- Add Linux IO metrics to system/process. {pull}37213[37213]
+- Add new memory/cgroup metrics to Kibana module. {pull}37232[37232]
+
+*Packetbeat*
+
+- Add metrics for TCP flags. {issue}36992[36992] {pull}36975[36975]
+
+*Winlogbeat*
+
+- Make ingest pipeline routing robust to letter case of channel names for forwarded events. {issue}36670[36670] {pull}36899[36899]
+- Document minimum permissions required for local user account. {issue}15773[15773] {pull}37176[37176]
+
+==== Deprecated
+
+*Filebeat*
+
+- Deprecate rsa2elk Filebeat modules. {issue}36125[36125] {pull}36887[36887]
+
+
+[[release-notes-8.11.4]]
+=== Beats version 8.11.4
+https://github.com/elastic/beats/compare/v8.11.3\...v8.11.4[View commits]
+
+==== Bugfixes
+
+*Heartbeat*
+
+- Added fix for formatting the logs from stateloader properly. {pull}37369[37369]
+- Remove duplicated syscall from ARM seccomp profile. {pull}37440[37440]
+
+*Metricbeat*
+
+- Nest the `region` and `availability_zone` ECS fields within the cloud field. {pull}37015[37015]
+- Fix CPU and memory metrics collection from privileged process on Windows. {issue}17314[17314]{pull}37027[37027]
+- Add memory hard limit from container metadata and remove usage percentage in AWS Fargate. {pull}37194[37194]
+- Ignore parser errors from unsupported metrics types on Prometheus client and continue parsing until EOF is reached. {pull}37383[37383]
+- Fix the reference time rounding on Azure Metrics. {issue}37204[37204] {pull}37365[37365]
+
+==== Added
+
+*Packetbeat*
+
+- Bump Windows Npcap version to v1.78. {issue}37300[37300] {pull}37370[37370]
+
+
 [[release-notes-8.11.3]]
 === Beats version 8.11.3
 https://github.com/elastic/beats/compare/v8.11.2\...v8.11.3[View commits]