[8.12][Fleet] Upgrade details telemetry (#173356) (#173502)

Backport #173356

I got a merge conflict on backporting to 8.12, because the presets
telemetry was not backportet, is that intentional?
#172838

x-pack/plugins/fleet/server/collectors/agent_collectors.ts

@@ -75,13 +75,20 @@ export interface AgentData {
     version: string;
     count: number;
   }>;
+  upgrade_details: Array<{
+    target_version: string;
+    state: string;
+    error_msg: string;
+    agent_count: number;
+  }>;
 }
 
 const DEFAULT_AGENT_DATA = {
   agent_checkin_status: { error: 0, degraded: 0 },
   agents_per_policy: [],
   agents_per_version: [],
   agents_per_os: [],
+  upgrade_details: [],
 };
 
 export const getAgentData = async (
@@ -135,6 +142,23 @@ export const getAgentData = async (
               ],
             },
           },
+          upgrade_details: {
+            multi_terms: {
+              size: 1000,
+              terms: [
+                {
+                  field: 'upgrade_details.target_version.keyword',
+                },
+                {
+                  field: 'upgrade_details.state',
+                },
+                {
+                  field: 'upgrade_details.metadata.error_msg.keyword',
+                  missing: '',
+                },
+              ],
+            },
+          },
         },
       },
       { signal: abortController.signal }
@@ -190,11 +214,21 @@ export const getAgentData = async (
       count: bucket.doc_count,
     }));
 
+    const upgradeDetails = ((response?.aggregations?.upgrade_details as any).buckets ?? []).map(
+      (bucket: any) => ({
+        target_version: bucket.key[0],
+        state: bucket.key[1],
+        error_msg: bucket.key[2],
+        agent_count: bucket.doc_count,
+      })
+    );
+
     return {
       agent_checkin_status: statuses,
       agents_per_policy: agentsPerPolicy,
       agents_per_version: agentsPerVersion,
       agents_per_os: agentsPerOS,
+      upgrade_details: upgradeDetails,
     };
   } catch (error) {
     if (error.statusCode === 404) {

x-pack/plugins/fleet/server/collectors/agents_per_output.ts

@@ -61,5 +61,6 @@ export async function getAgentsPerOutput(
       }
       outputTypes[monitoringOutputType].count_as_monitoring += item.agents ?? 0;
     });
+
   return Object.values(outputTypes);
 }

x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts

@@ -146,6 +146,13 @@ describe('fleet usage telemetry', () => {
               status: 'HEALTHY',
             },
           ],
+          upgrade_details: {
+            target_version: '8.12.0',
+            state: 'UPG_FAILED',
+            metadata: {
+              error_msg: 'Download failed',
+            },
+          },
         },
         {
           create: {
@@ -176,6 +183,13 @@ describe('fleet usage telemetry', () => {
               status: 'HEALTHY',
             },
           ],
+          upgrade_details: {
+            target_version: '8.12.0',
+            state: 'UPG_FAILED',
+            metadata: {
+              error_msg: 'Agent crash detected',
+            },
+          },
         },
         {
           create: {
@@ -220,6 +234,11 @@ describe('fleet usage telemetry', () => {
           last_checkin: new Date(Date.now() - 1000 * 60 * 6).toISOString(),
           active: true,
           policy_id: 'policy2',
+          upgrade_details: {
+            target_version: '8.11.0',
+            state: 'UPG_ROLLBACK',
+            metadata: {},
+          },
         },
       ],
       refresh: 'wait_for',
@@ -498,5 +517,24 @@ describe('fleet usage telemetry', () => {
         fleet_server_logs_top_errors: ['failed to unenroll offline agents'],
       })
     );
+    expect(usage?.upgrade_details.length).toBe(3);
+    expect(usage?.upgrade_details).toContainEqual({
+      target_version: '8.12.0',
+      state: 'UPG_FAILED',
+      error_msg: 'Download failed',
+      agent_count: 1,
+    });
+    expect(usage?.upgrade_details).toContainEqual({
+      target_version: '8.12.0',
+      state: 'UPG_FAILED',
+      error_msg: 'Agent crash detected',
+      agent_count: 1,
+    });
+    expect(usage?.upgrade_details).toContainEqual({
+      target_version: '8.11.0',
+      state: 'UPG_ROLLBACK',
+      error_msg: '',
+      agent_count: 1,
+    });
   });
 });

x-pack/plugins/fleet/server/services/telemetry/fleet_usage_sender.ts

@@ -24,7 +24,7 @@ const FLEET_AGENTS_EVENT_TYPE = 'fleet_agents';
 
 export class FleetUsageSender {
   private taskManager?: TaskManagerStartContract;
-  private taskVersion = '1.1.3';
+  private taskVersion = '1.1.4';
   private taskType = 'Fleet-Usage-Sender';
   private wasStarted: boolean = false;
   private interval = '1h';
@@ -83,6 +83,7 @@ export class FleetUsageSender {
       const {
         agents_per_version: agentsPerVersion,
         agents_per_output_type: agentsPerOutputType,
+        upgrade_details: upgradeDetails,
         ...fleetUsageData
       } = usageData;
       appContextService
@@ -106,6 +107,13 @@ export class FleetUsageSender {
           agents_per_output_type: byOutputType,
         });
       });
+
+      appContextService
+        .getLogger()
+        .debug('Agents upgrade details telemetry: ' + JSON.stringify(upgradeDetails));
+      upgradeDetails.forEach((upgradeDetailsObj) => {
+        core.analytics.reportEvent(FLEET_AGENTS_EVENT_TYPE, { upgrade_details: upgradeDetailsObj });
+      });
     } catch (error) {
       appContextService
         .getLogger()

x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts

@@ -90,6 +90,38 @@ export const fleetAgentsSchema: RootSchema<any> = {
       },
     },
   },
+  upgrade_details: {
+    _meta: {
+      description: 'Agent upgrade details telemetry',
+      optional: true,
+    },
+    properties: {
+      target_version: {
+        type: 'keyword',
+        _meta: {
+          description: 'Target version of the agent upgrade',
+        },
+      },
+      state: {
+        type: 'keyword',
+        _meta: {
+          description: 'State of the agent upgrade',
+        },
+      },
+      error_msg: {
+        type: 'keyword',
+        _meta: {
+          description: 'Error message of the agent upgrade if failed',
+        },
+      },
+      agent_count: {
+        type: 'long',
+        _meta: {
+          description: 'How many agents have this upgrade details',
+        },
+      },
+    },
+  },
 };
 
 export const fleetUsagesSchema: RootSchema<any> = {