diff --git a/x-pack/platform/plugins/private/telemetry_collection_xpack/schema/xpack_security.json b/x-pack/platform/plugins/private/telemetry_collection_xpack/schema/xpack_security.json index 1ab6ba490f0b5..cb9881403ebe8 100644 --- a/x-pack/platform/plugins/private/telemetry_collection_xpack/schema/xpack_security.json +++ b/x-pack/platform/plugins/private/telemetry_collection_xpack/schema/xpack_security.json @@ -1,292 +1,5 @@ { "properties": { - "cloud_defend": { - "properties": { - "indices": { - "properties": { - "alerts": { - "properties": { - "doc_count": { - "type": "long" - }, - "deleted": { - "type": "long" - }, - "size_in_bytes": { - "type": "long" - }, - "last_doc_timestamp": { - "type": "date" - } - } - }, - "file": { - "properties": { - "doc_count": { - "type": "long" - }, - "deleted": { - "type": "long" - }, - "size_in_bytes": { - "type": "long" - }, - "last_doc_timestamp": { - "type": "date" - } - } - }, - "process": { - "properties": { - "doc_count": { - "type": "long" - }, - "deleted": { - "type": "long" - }, - "size_in_bytes": { - "type": "long" - }, - "last_doc_timestamp": { - "type": "date" - } - } - }, - "latestPackageVersion": { - "type": "keyword" - }, - "packageStatus": { - "properties": { - "status": { - "type": "keyword" - }, - "installedPackagePolicies": { - "type": "long" - }, - "healthyAgents": { - "type": "long" - } - } - } - } - }, - "pods_stats": { - "type": "array", - "items": { - "properties": { - "account_id": { - "type": "keyword" - }, - "container_image_name": { - "type": "keyword" - }, - "container_image_tag": { - "type": "keyword" - }, - "pod_name": { - "type": "keyword" - }, - "total_doc_count": { - "type": "long" - }, - "process_doc_count": { - "type": "long" - }, - "file_doc_count": { - "type": "long" - }, - "alert_doc_count": { - "type": "long" - } - } - } - }, - "accounts_stats": { - "type": "array", - "items": { - "properties": { - "account_id": { - "type": "keyword" - }, - "cloud_provider": { - "type": "keyword" - }, - "kubernetes_version": { - "type": "keyword" - }, - "total_doc_count": { - "type": "long" - }, - "file_doc_count": { - "type": "long" - }, - "process_doc_count": { - "type": "long" - }, - "alert_doc_count": { - "type": "long" - }, - "agents_count": { - "type": "short" - }, - "nodes_count": { - "type": "short" - }, - "pods_count": { - "type": "short" - } - } - } - }, - "installation_stats": { - "type": "array", - "items": { - "properties": { - "package_policy_id": { - "type": "keyword" - }, - "package_version": { - "type": "keyword" - }, - "agent_policy_id": { - "type": "keyword" - }, - "created_at": { - "type": "date" - }, - "agent_count": { - "type": "long" - }, - "policy_yaml": { - "type": "keyword" - }, - "selectors": { - "type": "array", - "items": { - "properties": { - "type": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "operation": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "containerImageFullName": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "containerImageName": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "containerImageTag": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "kubernetesClusterId": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "kubernetesClusterName": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "kubernetesNamespace": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "kubernetesPodLabel": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "kubernetesPodName": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "targetFilePath": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "ignoreVolumeFiles": { - "type": "boolean" - }, - "ignoreVolumeMounts": { - "type": "boolean" - }, - "processExecutable": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "processName": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "sessionLeaderInteractive": { - "type": "boolean" - } - } - } - }, - "responses": { - "type": "array", - "items": { - "properties": { - "type": { - "type": "keyword" - }, - "match": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "exclude": { - "type": "array", - "items": { - "type": "keyword" - } - }, - "actions": { - "type": "array", - "items": { - "type": "keyword" - } - } - } - } - } - } - } - } - } - }, "cloud_security_posture": { "properties": { "indices": { diff --git a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/accounts_stats_collector.ts b/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/accounts_stats_collector.ts deleted file mode 100644 index af29fad6e5918..0000000000000 --- a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/accounts_stats_collector.ts +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { Logger } from '@kbn/core/server'; -import type { - AggregationsMultiBucketBase, - SearchRequest, -} from '@elastic/elasticsearch/lib/api/types'; -import type { CloudDefendAccountsStats } from './types'; -import { LOGS_CLOUD_DEFEND_PATTERN } from '../../../../common/constants'; - -interface Value { - value: number; -} - -interface KubernetesVersion { - metrics: { 'orchestrator.version': string }; -} - -interface CloudProvider { - metrics: { 'cloud.provider': string }; -} - -interface AccountsStats { - accounts: { - buckets: AccountEntity[]; - }; -} -interface AccountEntity { - key: string; // aggregation bucket key (currently: orchestrator.cluster.id) - doc_count: number; // total doc count (process + file + alerts) - process_doc_count: AggregationsMultiBucketBase; - file_doc_count: AggregationsMultiBucketBase; - alert_doc_count: AggregationsMultiBucketBase; - cloud_provider: { top: CloudProvider[] }; - kubernetes_version: { top: KubernetesVersion[] }; - agents_count: Value; - nodes_count: Value; - pods_count: Value; - resources: { - pods_count: Value; - }; -} - -const getAccountsStatsQuery = (): SearchRequest => ({ - index: LOGS_CLOUD_DEFEND_PATTERN, - query: { - match_all: {}, - }, - aggs: { - accounts: { - terms: { - field: 'orchestrator.cluster.id', - order: { - _count: 'desc', - }, - size: 100, - }, - aggs: { - nodes_count: { - cardinality: { - field: 'cloud.instance.name', - }, - }, - agents_count: { - cardinality: { - field: 'agent.id', - }, - }, - kubernetes_version: { - top_metrics: { - metrics: { - field: 'orchestrator.version', - }, - size: 1, - sort: { - '@timestamp': 'desc', - }, - }, - }, - cloud_provider: { - top_metrics: { - metrics: { - field: 'cloud.provider', - }, - size: 1, - sort: { - '@timestamp': 'desc', - }, - }, - }, - file_doc_count: { - filter: { - bool: { - filter: [ - { - bool: { - should: [ - { - term: { - 'event.category': 'file', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - }, - process_doc_count: { - filter: { - bool: { - filter: [ - { - bool: { - should: [ - { - term: { - 'event.category': 'process', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - }, - alert_doc_count: { - filter: { - bool: { - filter: [ - { - bool: { - should: [ - { - term: { - 'event.kind': 'alert', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - }, - pods_count: { - cardinality: { - field: 'orchestrator.resource.name', - }, - }, - }, - }, - }, - - size: 0, - _source: false, -}); - -const getCloudDefendAccountsStats = ( - aggregatedResourcesStats: AccountsStats, - logger: Logger -): CloudDefendAccountsStats[] => { - const accounts = aggregatedResourcesStats.accounts.buckets; - - const cloudDefendAccountsStats = accounts.map((account) => ({ - account_id: account.key, - total_doc_count: account.doc_count, - file_doc_count: account.file_doc_count.doc_count, - process_doc_count: account.process_doc_count.doc_count, - alert_doc_count: account.alert_doc_count.doc_count, - kubernetes_version: account.kubernetes_version?.top?.[0]?.metrics['orchestrator.version'], - cloud_provider: account.cloud_provider?.top?.[0]?.metrics['cloud.provider'], - agents_count: account.agents_count.value, - nodes_count: account.nodes_count.value, - pods_count: account.pods_count.value, - })); - logger.info('CloudDefend telemetry: accounts stats was sent'); - - return cloudDefendAccountsStats; -}; - -export const getAccountsStats = async ( - esClient: ElasticsearchClient, - logger: Logger -): Promise => { - try { - const isIndexExists = await esClient.indices.exists({ - index: LOGS_CLOUD_DEFEND_PATTERN, - }); - - if (isIndexExists) { - const accountsStatsResponse = await esClient.search( - getAccountsStatsQuery() - ); - - const cloudDefendAccountsStats = accountsStatsResponse.aggregations - ? getCloudDefendAccountsStats(accountsStatsResponse.aggregations, logger) - : []; - - return cloudDefendAccountsStats; - } - - return []; - } catch (e) { - logger.error(`Failed to get account stats ${e}`); - return []; - } -}; diff --git a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/indices_stats_collector.ts b/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/indices_stats_collector.ts deleted file mode 100644 index 2c1279952391d..0000000000000 --- a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/indices_stats_collector.ts +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { CoreStart, Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { getCloudDefendStatus } from '../../../routes/status/status'; -import type { CloudDefendPluginStart, CloudDefendPluginStartDeps } from '../../../types'; - -import type { CloudDefendIndicesStats, IndexStats } from './types'; -import { - ALERTS_INDEX_PATTERN, - FILE_INDEX_PATTERN, - PROCESS_INDEX_PATTERN, -} from '../../../../common/constants'; - -const getIndexDocCount = (esClient: ElasticsearchClient, index: string) => - esClient.indices.stats({ index }); - -const getLatestDocTimestamp = async ( - esClient: ElasticsearchClient, - index: string -): Promise => { - const latestTimestamp = await esClient.search({ - index, - query: { - match_all: {}, - }, - sort: '@timestamp:desc', - size: 1, - fields: ['@timestamp'], - _source: false, - }); - - const latestEventTimestamp = latestTimestamp.hits?.hits[0]?.fields; - - return latestEventTimestamp ? latestEventTimestamp['@timestamp'][0] : null; -}; - -const getIndexStats = async ( - esClient: ElasticsearchClient, - index: string, - logger: Logger -): Promise => { - try { - const lastDocTimestamp = await getLatestDocTimestamp(esClient, index); - - if (lastDocTimestamp) { - const indexStats = await getIndexDocCount(esClient, index); - return { - doc_count: indexStats._all.primaries?.docs ? indexStats._all.primaries?.docs?.count : 0, - deleted: indexStats._all.primaries?.docs?.deleted - ? indexStats._all.primaries?.docs?.deleted - : 0, - size_in_bytes: indexStats._all.primaries?.store - ? indexStats._all.primaries?.store.size_in_bytes - : 0, - last_doc_timestamp: lastDocTimestamp, - }; - } - - return {}; - } catch (e) { - logger.error(`Failed to get index stats for ${index}`); - return {}; - } -}; - -export const getIndicesStats = async ( - esClient: ElasticsearchClient, - soClient: SavedObjectsClientContract, - coreServices: Promise<[CoreStart, CloudDefendPluginStartDeps, CloudDefendPluginStart]>, - logger: Logger -): Promise => { - const [alerts, file, process] = await Promise.all([ - getIndexStats(esClient, ALERTS_INDEX_PATTERN, logger), - getIndexStats(esClient, FILE_INDEX_PATTERN, logger), - getIndexStats(esClient, PROCESS_INDEX_PATTERN, logger), - ]); - - const [, cloudDefendPluginStartDeps] = await coreServices; - - const { status, latestPackageVersion, installedPackagePolicies, healthyAgents } = - await getCloudDefendStatus({ - logger, - esClient, - soClient, - agentPolicyService: cloudDefendPluginStartDeps.fleet.agentPolicyService, - agentService: cloudDefendPluginStartDeps.fleet.agentService, - packagePolicyService: cloudDefendPluginStartDeps.fleet.packagePolicyService, - packageService: cloudDefendPluginStartDeps.fleet.packageService, - }); - - return { - alerts, - file, - process, - latestPackageVersion, - packageStatus: { - status, - installedPackagePolicies, - healthyAgents, - }, - }; -}; diff --git a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/installation_stats_collector.ts b/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/installation_stats_collector.ts deleted file mode 100644 index 98c4a3c49c07b..0000000000000 --- a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/installation_stats_collector.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { CoreStart, Logger, SavedObjectsClientContract } from '@kbn/core/server'; -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { - AgentPolicy, - PackagePolicy, - PACKAGE_POLICY_SAVED_OBJECT_TYPE, - SO_SEARCH_LIMIT, -} from '@kbn/fleet-plugin/common'; -import { agentPolicyService } from '@kbn/fleet-plugin/server/services'; -import type { CloudDefendInstallationStats } from './types'; -import type { CloudDefendPluginStart, CloudDefendPluginStartDeps } from '../../../types'; -import { INTEGRATION_PACKAGE_NAME, INPUT_CONTROL } from '../../../../common/constants'; -import { - getInputFromPolicy, - getSelectorsAndResponsesFromYaml, -} from '../../../../common/utils/helpers'; - -export const getInstallationStats = async ( - esClient: ElasticsearchClient, - soClient: SavedObjectsClientContract, - coreServices: Promise<[CoreStart, CloudDefendPluginStartDeps, CloudDefendPluginStart]>, - logger: Logger -): Promise => { - const [, cloudDefendServerPluginStartDeps] = await coreServices; - - const cloudDefendContext = { - logger, - esClient, - soClient, - agentPolicyService: cloudDefendServerPluginStartDeps.fleet.agentPolicyService, - packagePolicyService: cloudDefendServerPluginStartDeps.fleet.packagePolicyService, - }; - - const getInstalledPackagePolicies = async ( - packagePolicies: PackagePolicy[], - agentPolicies: AgentPolicy[] - ) => { - const installationStats = packagePolicies.map( - (packagePolicy: PackagePolicy): CloudDefendInstallationStats => { - const agentCounts = - agentPolicies?.find((agentPolicy) => agentPolicy?.id === packagePolicy.policy_id) - ?.agents ?? 0; - - const input = getInputFromPolicy(packagePolicy, INPUT_CONTROL); - const policyYaml = input?.vars?.configuration?.value; - const { selectors, responses } = getSelectorsAndResponsesFromYaml(policyYaml); - - return { - package_policy_id: packagePolicy.id, - package_version: packagePolicy.package?.version as string, - created_at: packagePolicy.created_at, - agent_policy_id: packagePolicy.policy_ids[0], - agent_count: agentCounts, - policy_yaml: policyYaml, - selectors, - responses, - }; - } - ); - return installationStats; - }; - - const packagePolicies = await cloudDefendContext.packagePolicyService.list(soClient, { - perPage: SO_SEARCH_LIMIT, - kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:"${INTEGRATION_PACKAGE_NAME}"`, - }); - - const agentPolicies = await agentPolicyService.list(soClient, { - perPage: SO_SEARCH_LIMIT, - kuery: '', - esClient, - withAgentCount: true, - }); - - if (!packagePolicies) return []; - - const installationStats: CloudDefendInstallationStats[] = await getInstalledPackagePolicies( - packagePolicies.items, - agentPolicies?.items || [] - ); - - return installationStats; -}; diff --git a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/pods_stats_collector.ts b/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/pods_stats_collector.ts deleted file mode 100644 index 311411ca92cad..0000000000000 --- a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/pods_stats_collector.ts +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { Logger } from '@kbn/core/server'; -import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; -import type { CloudDefendPodsStats } from './types'; -import { LOGS_CLOUD_DEFEND_PATTERN } from '../../../../common/constants'; - -interface PodsStats { - accounts: { - buckets: AccountEntity[]; - }; -} - -interface Bucket { - key: string; -} - -export interface AccountEntity { - key: string; // orchestrator.cluster.id - doc_count: number; - pods: { - buckets: Pod[]; - }; -} - -interface Pod { - key: string; // orchestrator.resource.name - container_image_name: { - buckets: Bucket[]; - }; - container_image_tag: { - buckets: Bucket[]; - }; - doc_count: number; - file_doc_count: { - doc_count: number; - }; - process_doc_count: { - doc_count: number; - }; - alert_doc_count: { - doc_count: number; - }; -} - -const getPodsStatsQuery = (index: string): SearchRequest => ({ - index, - query: { - match_all: {}, - }, - aggs: { - accounts: { - terms: { - field: 'orchestrator.cluster.id', - order: { - _count: 'desc', - }, - size: 100, - }, - aggs: { - // all cloud-defend logs are from the viewpoint of an orchestrator.resource.type = "pod" - // so no need to filter by orchestrator.resource.type. - pods: { - terms: { - field: 'orchestrator.resource.name', - order: { - _count: 'desc', - }, - size: 100, - }, - aggs: { - container_image_name: { - terms: { - field: 'container.image.name', - size: 1, - }, - }, - container_image_tag: { - terms: { - field: 'container.image.tag', - size: 1, - }, - }, - file_doc_count: { - filter: { - bool: { - filter: [ - { - bool: { - should: [ - { - term: { - 'event.category': 'file', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - }, - process_doc_count: { - filter: { - bool: { - filter: [ - { - bool: { - should: [ - { - term: { - 'event.category': 'process', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - }, - alert_doc_count: { - filter: { - bool: { - filter: [ - { - bool: { - should: [ - { - term: { - 'event.kind': 'alert', - }, - }, - ], - minimum_should_match: 1, - }, - }, - ], - }, - }, - }, - }, - }, - }, - }, - }, - - size: 0, - _source: false, -}); - -const getCloudDefendPodsStats = ( - aggregatedPodsStats: PodsStats, - logger: Logger -): CloudDefendPodsStats[] => { - const accounts = aggregatedPodsStats.accounts.buckets; - const podsStats = accounts.map((account) => { - const accountId = account.key; - return account.pods.buckets.map((pod) => { - return { - account_id: accountId, - pod_name: pod.key, - container_image_name: pod.container_image_name?.buckets?.[0]?.key, - container_image_tag: pod.container_image_tag?.buckets?.[0]?.key, - total_doc_count: pod.doc_count, - file_doc_count: pod.file_doc_count.doc_count, - process_doc_count: pod.process_doc_count.doc_count, - alert_doc_count: pod.alert_doc_count.doc_count, - }; - }); - }); - logger.info('Cloud defend telemetry: pods stats was sent'); - - return podsStats.flat(2); -}; - -export const getPodsStats = async ( - esClient: ElasticsearchClient, - logger: Logger -): Promise => { - try { - const isIndexExists = await esClient.indices.exists({ - index: LOGS_CLOUD_DEFEND_PATTERN, - }); - - if (isIndexExists) { - const podsStatsResponse = await esClient.search( - getPodsStatsQuery(LOGS_CLOUD_DEFEND_PATTERN) - ); - - const cloudDefendPodsStats = podsStatsResponse.aggregations - ? getCloudDefendPodsStats(podsStatsResponse.aggregations, logger) - : []; - - return cloudDefendPodsStats; - } - - return []; - } catch (e) { - logger.error(`Failed to get pods stats ${e}`); - return []; - } -}; diff --git a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/register.ts b/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/register.ts deleted file mode 100644 index 6b7d53e57f89e..0000000000000 --- a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/register.ts +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { CollectorFetchContext, UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; -import type { CoreStart, Logger } from '@kbn/core/server'; -import { CloudDefendPluginStart, CloudDefendPluginStartDeps } from '../../../types'; -import { getIndicesStats } from './indices_stats_collector'; -import { getPodsStats } from './pods_stats_collector'; -import { cloudDefendUsageSchema } from './schema'; -import { CloudDefendUsage } from './types'; -import { getAccountsStats } from './accounts_stats_collector'; -import { getInstallationStats } from './installation_stats_collector'; - -export function registerCloudDefendUsageCollector( - logger: Logger, - coreServices: Promise<[CoreStart, CloudDefendPluginStartDeps, CloudDefendPluginStart]>, - usageCollection?: UsageCollectionSetup -): void { - // usageCollection is an optional dependency, so make sure to return if it is not registered - if (!usageCollection) { - logger.debug('Usage collection disabled'); - return; - } - - // Create usage collector - const cloudDefendUsageCollector = usageCollection.makeUsageCollector({ - type: 'cloud_defend', - isReady: async () => { - await coreServices; - return true; - }, - fetch: async (collectorFetchContext: CollectorFetchContext) => { - logger.debug('Starting cloud_defend usage collection'); - - const [indicesStats, accountsStats, podsStats, installationStats] = await Promise.all([ - getIndicesStats( - collectorFetchContext.esClient, - collectorFetchContext.soClient, - coreServices, - logger - ), - getAccountsStats(collectorFetchContext.esClient, logger), - getPodsStats(collectorFetchContext.esClient, logger), - getInstallationStats( - collectorFetchContext.esClient, - collectorFetchContext.soClient, - coreServices, - logger - ), - ]).catch((err) => { - logger.error(err); - - return err; - }); - - return { - indices: indicesStats, - accounts_stats: accountsStats, - pods_stats: podsStats, - installation_stats: installationStats, - }; - }, - schema: cloudDefendUsageSchema, - }); - - // Register usage collector - usageCollection.registerCollector(cloudDefendUsageCollector); -} diff --git a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/schema.ts b/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/schema.ts deleted file mode 100644 index 55c6677128c09..0000000000000 --- a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/schema.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { MakeSchemaFrom } from '@kbn/usage-collection-plugin/server'; -import type { CloudDefendUsage } from './types'; - -export const cloudDefendUsageSchema: MakeSchemaFrom = { - indices: { - alerts: { - doc_count: { - type: 'long', - }, - deleted: { - type: 'long', - }, - size_in_bytes: { - type: 'long', - }, - last_doc_timestamp: { - type: 'date', - }, - }, - file: { - doc_count: { - type: 'long', - }, - deleted: { - type: 'long', - }, - size_in_bytes: { - type: 'long', - }, - last_doc_timestamp: { - type: 'date', - }, - }, - process: { - doc_count: { - type: 'long', - }, - deleted: { - type: 'long', - }, - size_in_bytes: { - type: 'long', - }, - last_doc_timestamp: { - type: 'date', - }, - }, - latestPackageVersion: { type: 'keyword' }, - packageStatus: { - status: { type: 'keyword' }, - installedPackagePolicies: { type: 'long' }, - healthyAgents: { type: 'long' }, - }, - }, - pods_stats: { - type: 'array', - items: { - account_id: { type: 'keyword' }, - container_image_name: { type: 'keyword' }, - container_image_tag: { type: 'keyword' }, - pod_name: { type: 'keyword' }, - total_doc_count: { type: 'long' }, - process_doc_count: { type: 'long' }, - file_doc_count: { type: 'long' }, - alert_doc_count: { type: 'long' }, - }, - }, - accounts_stats: { - type: 'array', - items: { - account_id: { type: 'keyword' }, - cloud_provider: { type: 'keyword' }, - kubernetes_version: { type: 'keyword' }, - total_doc_count: { type: 'long' }, - file_doc_count: { type: 'long' }, - process_doc_count: { type: 'long' }, - alert_doc_count: { type: 'long' }, - agents_count: { type: 'short' }, - nodes_count: { type: 'short' }, - pods_count: { type: 'short' }, - }, - }, - installation_stats: { - type: 'array', - items: { - package_policy_id: { type: 'keyword' }, - package_version: { type: 'keyword' }, - agent_policy_id: { type: 'keyword' }, - created_at: { type: 'date' }, - agent_count: { type: 'long' }, - policy_yaml: { type: 'keyword' }, - selectors: { - type: 'array', - items: { - type: { type: 'keyword' }, - name: { type: 'keyword' }, - operation: { type: 'array', items: { type: 'keyword' } }, - containerImageFullName: { type: 'array', items: { type: 'keyword' } }, - containerImageName: { type: 'array', items: { type: 'keyword' } }, - containerImageTag: { type: 'array', items: { type: 'keyword' } }, - kubernetesClusterId: { type: 'array', items: { type: 'keyword' } }, - kubernetesClusterName: { type: 'array', items: { type: 'keyword' } }, - kubernetesNamespace: { type: 'array', items: { type: 'keyword' } }, - kubernetesPodLabel: { type: 'array', items: { type: 'keyword' } }, - kubernetesPodName: { type: 'array', items: { type: 'keyword' } }, - targetFilePath: { type: 'array', items: { type: 'keyword' } }, - ignoreVolumeFiles: { type: 'boolean' }, - ignoreVolumeMounts: { type: 'boolean' }, - processExecutable: { type: 'array', items: { type: 'keyword' } }, - processName: { type: 'array', items: { type: 'keyword' } }, - sessionLeaderInteractive: { type: 'boolean' }, - }, - }, - responses: { - type: 'array', - items: { - type: { type: 'keyword' }, - match: { type: 'array', items: { type: 'keyword' } }, - exclude: { type: 'array', items: { type: 'keyword' } }, - actions: { type: 'array', items: { type: 'keyword' } }, - }, - }, - }, - }, -}; diff --git a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/types.ts b/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/types.ts deleted file mode 100644 index 6aca6936af649..0000000000000 --- a/x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/types.ts +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -// for some reason we can't reference common/index.ts because -// the `node scripts/check_telemetry.js --fix` command fails with the error -// ERROR Error: Error extracting collector in x-pack/solutions/security/plugins/cloud_defend/server/lib/telemetry/collectors/register.ts -// Error: Unable to find identifier in source Selector -// at createFailError (dev_cli_errors.ts:27:24) -// at parseUsageCollection (ts_parser.ts:226:32) -// at parseUsageCollection.next () -// at extractCollectors (extract_collectors.ts:58:32) -// at extractCollectors.next () -// at Task.task (extract_collectors_task.ts:43:53) -// at runMicrotasks () -// at processTicksAndRejections (node:internal/process/task_queues:96:5) -// -// I guess the intermediate import/export is causing problems -// for now we will just point to the current version (v1) -import type { - Selector, - Response, - SelectorType, - SelectorCondition, - ResponseAction, -} from '../../../../common/v1'; - -export interface CloudDefendUsage { - indices: CloudDefendIndicesStats; - pods_stats: CloudDefendPodsStats[]; - accounts_stats: CloudDefendAccountsStats[]; - installation_stats: CloudDefendInstallationStats[]; -} - -export interface PackageSetupStatus { - status: string; - installedPackagePolicies: number; - healthyAgents: number; -} - -export interface CloudDefendIndicesStats { - alerts: IndexStats | {}; - file: IndexStats | {}; - process: IndexStats | {}; - latestPackageVersion: string; - packageStatus: PackageSetupStatus; -} - -export interface IndexStats { - doc_count: number; - deleted: number; - size_in_bytes: number; - last_doc_timestamp: string | null; -} - -export interface CloudDefendPodsStats { - account_id: string; - pod_name: string; - container_image_name: string; - container_image_tag: string; - total_doc_count: number; - file_doc_count: number; - process_doc_count: number; - alert_doc_count: number; -} - -export interface CloudDefendAccountsStats { - account_id: string; - total_doc_count: number; - cloud_provider: string; - kubernetes_version: string | null; - file_doc_count: number; - process_doc_count: number; - alert_doc_count: number; - agents_count: number; - nodes_count: number; - pods_count: number; -} - -export type CloudDefendSelectorTypeCounts = { - [key in SelectorType]: number; -}; - -export type CloudDefendResponseTypeCounts = { - [key in SelectorType]: number; -}; - -export type CloudDefendConditionsCounts = { - [key in SelectorCondition]?: number; -}; - -export type CloudDefendActionCounts = { - [key in ResponseAction]?: number; -}; - -export interface CloudDefendPolicyYamlStats { - policy_yaml: string; - policy_json: string; // to be used for further digging in BigQuery - selector_counts: CloudDefendSelectorTypeCounts; - response_counts: CloudDefendResponseTypeCounts; - selector_conditions_counts: CloudDefendConditionsCounts; - response_actions_counts: CloudDefendActionCounts; - response_match_names: string[]; - response_exclude_names: string[]; -} - -type CloudDefendSelector = Omit; -type CloudDefendResponse = Omit; - -export interface CloudDefendInstallationStats { - package_policy_id: string; - package_version: string; - agent_policy_id: string; - created_at: string; - agent_count: number; - policy_yaml: string; - selectors: CloudDefendSelector[]; - responses: CloudDefendResponse[]; -} diff --git a/x-pack/solutions/security/plugins/cloud_defend/server/plugin.ts b/x-pack/solutions/security/plugins/cloud_defend/server/plugin.ts index 5e889bc12e69b..05faa8556fd1c 100644 --- a/x-pack/solutions/security/plugins/cloud_defend/server/plugin.ts +++ b/x-pack/solutions/security/plugins/cloud_defend/server/plugin.ts @@ -23,7 +23,6 @@ import { setupRoutes } from './routes/setup_routes'; import { isCloudDefendPackage } from '../common/utils/helpers'; import { isSubscriptionAllowed } from '../common/utils/subscription'; import { onPackagePolicyPostCreateCallback } from './lib/fleet_util'; -import { registerCloudDefendUsageCollector } from './lib/telemetry/collectors/register'; export class CloudDefendPlugin implements Plugin { private readonly logger: Logger; @@ -44,9 +43,6 @@ export class CloudDefendPlugin implements Plugin