Add the full set of ECS metadata sent to Fleet to Diagnostics (#7029)

* Add metadata to diagnostics

* Add changelog.

* Update agent info mock.

* Add missing method to manual mock.

(cherry picked from commit d668787)

changelog/fragments/1740521494-diagnostics-include-all-metadata-that-is-sent-to-Fleet-by-default.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Include all metadata that is sent to Fleet in the agent-info.yaml file in diagnostics by default.
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: "elastic-agent"
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/7029
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

internal/pkg/agent/application/coordinator/coordinator.go

@@ -791,20 +791,21 @@ func (c *Coordinator) DiagnosticHooks() diagnostics.Hooks {
 			Description: "current state of the agent information of the running Elastic Agent",
 			ContentType: "application/yaml",
 			Hook: func(_ context.Context) []byte {
+				meta, err := c.agentInfo.ECSMetadata(c.logger)
+				if err != nil {
+					c.logger.Errorw("Error getting ECS metadata", "error.message", err)
+				}
+
 				output := struct {
-					AgentID      string            `yaml:"agent_id"`
-					Headers      map[string]string `yaml:"headers"`
-					LogLevel     string            `yaml:"log_level"`
-					Snapshot     bool              `yaml:"snapshot"`
-					Version      string            `yaml:"version"`
-					Unprivileged bool              `yaml:"unprivileged"`
+					Headers     map[string]string `yaml:"headers"`
+					LogLevel    string            `yaml:"log_level"`
+					RawLogLevel string            `yaml:"log_level_raw"`
+					Metadata    *info.ECSMeta     `yaml:"metadata"`
 				}{
-					AgentID:      c.agentInfo.AgentID(),
-					Headers:      c.agentInfo.Headers(),
-					LogLevel:     c.agentInfo.LogLevel(),
-					Snapshot:     c.agentInfo.Snapshot(),
-					Version:      c.agentInfo.Version(),
-					Unprivileged: c.agentInfo.Unprivileged(),
+					Headers:     c.agentInfo.Headers(),
+					LogLevel:    c.agentInfo.LogLevel(),
+					RawLogLevel: c.agentInfo.RawLogLevel(),
+					Metadata:    meta,
 				}
 				o, err := yaml.Marshal(output)
 				if err != nil {

internal/pkg/agent/application/coordinator/diagnostics_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/elastic/elastic-agent-client/v7/pkg/client"
 	"github.com/elastic/elastic-agent-client/v7/pkg/proto"
 
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/info"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/configuration"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/transpiler"
@@ -27,6 +28,7 @@ import (
 	"github.com/elastic/elastic-agent/pkg/component"
 	"github.com/elastic/elastic-agent/pkg/component/runtime"
 	agentclient "github.com/elastic/elastic-agent/pkg/control/v2/client"
+	"github.com/elastic/elastic-agent/pkg/core/logger"
 	"github.com/elastic/elastic-agent/pkg/utils/broadcaster"
 )
 
@@ -144,21 +146,61 @@ func TestDiagnosticAgentInfo(t *testing.T) {
 			"header1": "value1",
 			"header2": "value2",
 		},
-		logLevel:     "trace",
-		snapshot:     true,
-		version:      "8.14.0",
-		unprivileged: true,
+		logLevel: "trace",
+		meta: &info.ECSMeta{
+			Elastic: &info.ElasticECSMeta{
+				Agent: &info.AgentECSMeta{
+					BuildOriginal: "8.14.0-SNAPSHOT",
+					ID:            "agent-id",
+					LogLevel:      "trace",
+					Snapshot:      true,
+					Version:       "8.14.0",
+					Unprivileged:  true,
+					Upgradeable:   true,
+				},
+			},
+			Host: &info.HostECSMeta{
+				Arch:     "arm64",
+				Hostname: "Test-Macbook-Pro.local",
+			},
+			OS: &info.SystemECSMeta{
+				Name:     "macos",
+				Platform: "darwin",
+			},
+		},
 	}}
 
 	expected := `
-agent_id: agent-id
 headers:
   header1: value1
   header2: value2
 log_level: trace
-snapshot: true
-version: 8.14.0
-unprivileged: true
+log_level_raw: trace
+metadata:
+  elastic:
+    agent:
+      buildoriginal: "8.14.0-SNAPSHOT"
+      complete: false
+      id: agent-id
+      loglevel: trace
+      snapshot: true
+      unprivileged: true
+      upgradeable: true
+      version: 8.14.0
+  host:
+    arch: arm64
+    hostname: Test-Macbook-Pro.local
+    name: ""
+    id: ""
+    ip: []
+    mac: []
+  os:
+    family: ""
+    kernel: ""
+    platform: darwin
+    version: ""
+    name: macos
+    fullname: ""
 `
 
 	hook, ok := diagnosticHooksMap(coord)["agent-info"]
@@ -606,6 +648,7 @@ type fakeAgentInfo struct {
 	version      string
 	unprivileged bool
 	isStandalone bool
+	meta         *info.ECSMeta
 }
 
 func (a fakeAgentInfo) AgentID() string {
@@ -640,5 +683,9 @@ func (a fakeAgentInfo) IsStandalone() bool {
 	return a.isStandalone
 }
 
+func (a fakeAgentInfo) ECSMetadata(l *logger.Logger) (*info.ECSMeta, error) {
+	return a.meta, nil
+}
+
 func (a fakeAgentInfo) ReloadID(ctx context.Context) error                  { panic("implement me") }
 func (a fakeAgentInfo) SetLogLevel(ctx context.Context, level string) error { panic("implement me") }

internal/pkg/agent/application/info/agent_info.go

@@ -43,6 +43,9 @@ type Agent interface {
 
 	// IsStandalone returns true is the agent is running in standalone mode, i.e, without fleet
 	IsStandalone() bool
+
+	// ECSMetadata returns the ECS metadata that is attached as part of every Fleet checkin.
+	ECSMetadata(*logger.Logger) (*ECSMeta, error)
 }
 
 // AgentInfo is a collection of information about agent.

pkg/component/runtime/runtime_comm_test.go

@@ -15,7 +15,9 @@ import (
 
 	"github.com/elastic/elastic-agent-client/v7/pkg/client"
 	"github.com/elastic/elastic-agent-client/v7/pkg/proto"
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/info"
 	"github.com/elastic/elastic-agent/internal/pkg/core/authority"
+	"github.com/elastic/elastic-agent/pkg/core/logger"
 )
 
 type agentInfoMock struct {
@@ -50,6 +52,7 @@ func (a agentInfoMock) LogLevel() string                                    { pa
 func (a agentInfoMock) RawLogLevel() string                                 { panic("implement me") }
 func (a agentInfoMock) ReloadID(ctx context.Context) error                  { panic("implement me") }
 func (a agentInfoMock) SetLogLevel(ctx context.Context, level string) error { panic("implement me") }
+func (a agentInfoMock) ECSMetadata(l *logger.Logger) (*info.ECSMeta, error) { panic("implement me") }
 
 func TestCheckinExpected(t *testing.T) {
 	ca, err := authority.NewCA()