From 9c38ce00580d078ccc416894223fb4e575bd4c90 Mon Sep 17 00:00:00 2001 From: Harnish Chavda <118714680+harnish-elastic@users.noreply.github.com> Date: Fri, 17 Jan 2025 11:07:52 +0530 Subject: [PATCH] [O11y][MSSQL] Add optional mssql.query field to `performance` data stream for debugging purposes (#12212) * Remove unnecessary processor in performance data stream * add changelog entry * added debug flag that can preserve mssql.query field when enabled * Update packages/microsoft_sqlserver/data_stream/performance/manifest.yml Co-authored-by: Lalit Satapathy <69236064+lalit-satapathy@users.noreply.github.com> * update tag name * minor change in pipeline and system tests * minor change in tag description * address review comments --------- Co-authored-by: Lalit Satapathy <69236064+lalit-satapathy@users.noreply.github.com> --- packages/microsoft_sqlserver/changelog.yml | 5 + .../_dev/test/pipeline/test-common-config.yml | 3 + .../_dev/test/pipeline/test-performance.json | 71 +++++++++++ .../test-performance.json-expected.json | 73 +++++++++++ .../_dev/test/system/test-perf-config.yml | 3 + .../performance/agent/stream/stream.yml.hbs | 4 + .../elasticsearch/ingest_pipeline/default.yml | 9 +- .../data_stream/performance/fields/fields.yml | 3 + .../data_stream/performance/manifest.yml | 8 ++ .../data_stream/performance/sample_event.json | 116 ++++++++++-------- packages/microsoft_sqlserver/docs/README.md | 115 +++++++++-------- packages/microsoft_sqlserver/manifest.yml | 2 +- 12 files changed, 300 insertions(+), 112 deletions(-) create mode 100644 packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-performance.json create mode 100644 packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-performance.json-expected.json diff --git a/packages/microsoft_sqlserver/changelog.yml b/packages/microsoft_sqlserver/changelog.yml index b1eebeaf278..470a71e418f 100644 --- a/packages/microsoft_sqlserver/changelog.yml +++ b/packages/microsoft_sqlserver/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.10.0" + changes: + - description: Add optional mssql.query field to `performance` data stream for debugging purposes. + type: enhancement + link: https://github.com/elastic/integrations/pull/12212 - version: "2.9.6" changes: - description: Update links to getting started docs diff --git a/packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-common-config.yml b/packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..ff353bb2361 --- /dev/null +++ b/packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,3 @@ +fields: + tags: + - preserve_sql_queries \ No newline at end of file diff --git a/packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-performance.json b/packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-performance.json new file mode 100644 index 00000000000..55139aa4790 --- /dev/null +++ b/packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-performance.json @@ -0,0 +1,71 @@ +{ + "events": [ + { + "@timestamp": "2024-12-30T10:34:25.697Z", + "metricset": { + "name": "query", + "period": 10000 + }, + "ecs": { + "version": "8.11.0" + }, + "host": { + "name": "localhost" + }, + "agent": { + "version": "8.13.0", + "ephemeral_id": "70a3cef1-ba97-481f-95df-7c36355408c6", + "id": "90dfcb72-653c-4ce1-b473-98969e1f308c", + "name": "localhost", + "type": "metricbeat" + }, + "service": { + "address": "localhost:32782", + "type": "sql" + }, + "sql": { + "metrics": { + "page_splits_per_sec": 1192, + "transactions": 0, + "compilations_per_sec": 115026, + "buffer_checkpoint_pages_per_sec": 516, + "buffer_target_pages": 1196032, + "buffer_database_pages": 2940, + "logouts_per_sec": 71268, + "active_temp_tables": 0, + "lock_waits_per_sec": 2, + "batch_requests_per_sec": 455338, + "user_connections": 2, + "server_name": "f8ca37b1f01d", + "logins_per_sec": 71270, + "instance_name": "MSSQLSERVER", + "buffer_cache_hit_ratio": 74, + "connection_reset_per_sec": 274901, + "buffer_page_life_expectancy": 529720, + "re_compilations_per_sec": 400 + }, + "driver": "mssql", + "query": [ + "SELECT @@servername AS server_name, @@servicename AS instance_name;", + "SELECT cntr_value As 'active_temp_tables' FROM sys.dm_os_performance_counters WHERE counter_name = 'Active Temp Tables' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'batch_requests_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Batch Requests/sec'", + "SELECT cntr_value As 'buffer_cache_hit_ratio' FROM sys.dm_os_performance_counters WHERE counter_name = 'Buffer cache hit ratio' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_checkpoint_pages_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Checkpoint pages/sec' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_database_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Database pages' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_page_life_expectancy' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page life expectancy' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_target_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Target pages' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Compilations/sec'", + "SELECT cntr_value As 'connection_reset_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Connection Reset/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'lock_waits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Lock Waits/sec' AND instance_name = '_Total'", + "SELECT cntr_value As 'logins_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logins/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'logouts_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logouts/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'page_splits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page splits/sec'", + "SELECT cntr_value As 're_compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Re-Compilations/sec'", + "SELECT cntr_value As 'transactions' FROM sys.dm_os_performance_counters WHERE counter_name = 'Transactions' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'user_connections' FROM sys.dm_os_performance_counters WHERE counter_name= 'User Connections'", + "SELECT counter_name, cntr_value FROM sys.dm_os_performance_counters WHERE counter_name like 'Memory Grants Pend%'" + ] + } + } + ] +} \ No newline at end of file diff --git a/packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-performance.json-expected.json b/packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-performance.json-expected.json new file mode 100644 index 00000000000..1f9501271ec --- /dev/null +++ b/packages/microsoft_sqlserver/data_stream/performance/_dev/test/pipeline/test-performance.json-expected.json @@ -0,0 +1,73 @@ +{ + "expected": [ + { + "@timestamp": "2024-12-30T10:34:25.697Z", + "agent": { + "ephemeral_id": "70a3cef1-ba97-481f-95df-7c36355408c6", + "id": "90dfcb72-653c-4ce1-b473-98969e1f308c", + "name": "localhost", + "type": "metricbeat", + "version": "8.13.0" + }, + "ecs": { + "version": "8.11.0" + }, + "host": { + "name": "localhost" + }, + "metricset": { + "name": "query", + "period": 10000 + }, + "mssql": { + "metrics": { + "active_temp_tables": 0, + "batch_requests_per_sec": 455338, + "buffer_cache_hit_ratio": 74, + "buffer_checkpoint_pages_per_sec": 516, + "buffer_database_pages": 2940, + "buffer_page_life_expectancy": 529720, + "buffer_target_pages": 1196032, + "compilations_per_sec": 115026, + "connection_reset_per_sec": 274901, + "instance_name": "MSSQLSERVER", + "lock_waits_per_sec": 2, + "logins_per_sec": 71270, + "logouts_per_sec": 71268, + "page_splits_per_sec": 1192, + "re_compilations_per_sec": 400, + "server_name": "f8ca37b1f01d", + "transactions": 0, + "user_connections": 2 + }, + "query": [ + "SELECT @@servername AS server_name, @@servicename AS instance_name;", + "SELECT cntr_value As 'active_temp_tables' FROM sys.dm_os_performance_counters WHERE counter_name = 'Active Temp Tables' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'batch_requests_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Batch Requests/sec'", + "SELECT cntr_value As 'buffer_cache_hit_ratio' FROM sys.dm_os_performance_counters WHERE counter_name = 'Buffer cache hit ratio' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_checkpoint_pages_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Checkpoint pages/sec' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_database_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Database pages' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_page_life_expectancy' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page life expectancy' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_target_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Target pages' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Compilations/sec'", + "SELECT cntr_value As 'connection_reset_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Connection Reset/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'lock_waits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Lock Waits/sec' AND instance_name = '_Total'", + "SELECT cntr_value As 'logins_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logins/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'logouts_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logouts/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'page_splits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page splits/sec'", + "SELECT cntr_value As 're_compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Re-Compilations/sec'", + "SELECT cntr_value As 'transactions' FROM sys.dm_os_performance_counters WHERE counter_name = 'Transactions' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'user_connections' FROM sys.dm_os_performance_counters WHERE counter_name= 'User Connections'", + "SELECT counter_name, cntr_value FROM sys.dm_os_performance_counters WHERE counter_name like 'Memory Grants Pend%'" + ] + }, + "service": { + "address": "localhost:32782", + "type": "sql" + }, + "tags": [ + "preserve_sql_queries" + ] + } + ] +} diff --git a/packages/microsoft_sqlserver/data_stream/performance/_dev/test/system/test-perf-config.yml b/packages/microsoft_sqlserver/data_stream/performance/_dev/test/system/test-perf-config.yml index f5c4089b9cc..86055cc8ffa 100644 --- a/packages/microsoft_sqlserver/data_stream/performance/_dev/test/system/test-perf-config.yml +++ b/packages/microsoft_sqlserver/data_stream/performance/_dev/test/system/test-perf-config.yml @@ -3,3 +3,6 @@ vars: - "{{Hostname}}" username: sa password: 1234_asdf +data_stream: + vars: + preserve_sql_queries: true \ No newline at end of file diff --git a/packages/microsoft_sqlserver/data_stream/performance/agent/stream/stream.yml.hbs b/packages/microsoft_sqlserver/data_stream/performance/agent/stream/stream.yml.hbs index 66e657f9826..b027947ef75 100644 --- a/packages/microsoft_sqlserver/data_stream/performance/agent/stream/stream.yml.hbs +++ b/packages/microsoft_sqlserver/data_stream/performance/agent/stream/stream.yml.hbs @@ -49,3 +49,7 @@ sql_queries: processors: {{processors}} {{/if}} +tags: +{{#if preserve_sql_queries}} +- preserve_sql_queries +{{/if}} diff --git a/packages/microsoft_sqlserver/data_stream/performance/elasticsearch/ingest_pipeline/default.yml b/packages/microsoft_sqlserver/data_stream/performance/elasticsearch/ingest_pipeline/default.yml index 956ed538a77..547b8597733 100644 --- a/packages/microsoft_sqlserver/data_stream/performance/elasticsearch/ingest_pipeline/default.yml +++ b/packages/microsoft_sqlserver/data_stream/performance/elasticsearch/ingest_pipeline/default.yml @@ -5,15 +5,16 @@ processors: field: sql.driver ignore_missing: true ignore_failure: true -- remove: - field: sql.query - ignore_missing: true - ignore_failure: true - rename: field: sql target_field: mssql ignore_missing: true ignore_failure: true +- remove: + field: mssql.query + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_sql_queries'))" + ignore_missing: true + ignore_failure: true - foreach: field: mssql.metrics ignore_missing: true diff --git a/packages/microsoft_sqlserver/data_stream/performance/fields/fields.yml b/packages/microsoft_sqlserver/data_stream/performance/fields/fields.yml index 6dc59f33872..7877d62d63a 100644 --- a/packages/microsoft_sqlserver/data_stream/performance/fields/fields.yml +++ b/packages/microsoft_sqlserver/data_stream/performance/fields/fields.yml @@ -81,3 +81,6 @@ - name: memory_grants_pending type: long description: This is generated from the default pattern given for Dynamic Counter Name variable. This counter tells us how many processes are waiting for the memory to be assigned to them so they can get started. + - name: query + type: keyword + description: The SQL queries executed. \ No newline at end of file diff --git a/packages/microsoft_sqlserver/data_stream/performance/manifest.yml b/packages/microsoft_sqlserver/data_stream/performance/manifest.yml index 8d5a28d2c5f..472ebe46be5 100644 --- a/packages/microsoft_sqlserver/data_stream/performance/manifest.yml +++ b/packages/microsoft_sqlserver/data_stream/performance/manifest.yml @@ -18,6 +18,14 @@ streams: show_user: true default: 'Memory Grants Pend%' description: Collect the values for dynamic counters based on the provided pattern from the performance table. + - name: preserve_sql_queries + required: true + show_user: false + title: Preserve SQL Queries + description: Preserves SQL queries for debugging purposes. This feature is available in Elastic stack version 8.18 and later. + type: bool + multi: false + default: false - name: processors type: yaml title: Processors diff --git a/packages/microsoft_sqlserver/data_stream/performance/sample_event.json b/packages/microsoft_sqlserver/data_stream/performance/sample_event.json index f2d8395ea1c..6433bd14203 100644 --- a/packages/microsoft_sqlserver/data_stream/performance/sample_event.json +++ b/packages/microsoft_sqlserver/data_stream/performance/sample_event.json @@ -1,67 +1,52 @@ { - "@timestamp": "2022-11-23T05:03:28.987Z", + "@timestamp": "2025-01-16T06:17:48.647Z", "agent": { - "ephemeral_id": "70f5c0c1-37b1-486b-9806-8105b2cdcd20", - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", - "name": "docker-fleet-agent", + "ephemeral_id": "de13136c-dc52-4b51-87c3-7fe7592f46dc", + "id": "c0b6b234-d11b-4387-ab3b-fa3084042073", + "name": "elastic-agent-74192", "type": "metricbeat", - "version": "8.5.0" - }, - "cloud": { - "account": {}, - "instance": { - "id": "b30e45e6-7900-4900-8d67-e37cb13374bc", - "name": "obs-int-windows-dev" - }, - "machine": { - "type": "Standard_D16ds_v5" - }, - "provider": "azure", - "region": "CentralIndia", - "service": { - "name": "Virtual Machines" - } + "version": "8.18.0" }, "data_stream": { "dataset": "microsoft_sqlserver.performance", - "namespace": "ep", + "namespace": "93265", "type": "metrics" }, "ecs": { - "version": "8.11.0" + "version": "8.0.0" }, "elastic_agent": { - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", - "snapshot": false, - "version": "8.5.0" + "id": "c0b6b234-d11b-4387-ab3b-fa3084042073", + "snapshot": true, + "version": "8.18.0" }, "event": { "agent_id_status": "verified", "dataset": "microsoft_sqlserver.performance", - "duration": 41134100, - "ingested": "2022-11-23T05:03:30Z", + "duration": 71801450, + "ingested": "2025-01-16T06:17:51Z", "module": "sql" }, "host": { "architecture": "x86_64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", + "containerized": true, + "hostname": "elastic-agent-74192", "ip": [ - "172.18.0.5" + "192.168.242.2", + "192.168.255.6" ], "mac": [ - "02-42-AC-12-00-05" + "02-42-C0-A8-F2-02", + "02-42-C0-A8-FF-06" ], - "name": "docker-fleet-agent", + "name": "elastic-agent-74192", "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.104-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", + "family": "", + "kernel": "3.10.0-1160.92.1.el7.x86_64", + "name": "Wolfi", + "platform": "wolfi", "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" + "version": "20230201" } }, "metricset": { @@ -71,28 +56,51 @@ "mssql": { "metrics": { "active_temp_tables": 0, - "batch_requests_per_sec": 54, - "buffer_cache_hit_ratio": 24, - "buffer_checkpoint_pages_per_sec": 105, - "buffer_database_pages": 2215, - "buffer_page_life_expectancy": 16, - "buffer_target_pages": 2408448, - "compilations_per_sec": 80, + "batch_requests_per_sec": 15, + "buffer_cache_hit_ratio": 995, + "buffer_checkpoint_pages_per_sec": 70, + "buffer_database_pages": 2208, + "buffer_page_life_expectancy": 19, + "buffer_target_pages": 1196032, + "compilations_per_sec": 67, "connection_reset_per_sec": 13, "instance_name": "MSSQLSERVER", - "lock_waits_per_sec": 4, - "logins_per_sec": 16, - "logouts_per_sec": 15, + "lock_waits_per_sec": 3, + "logins_per_sec": 3, + "logouts_per_sec": 2, "memory_grants_pending": 0, "page_splits_per_sec": 9, "re_compilations_per_sec": 0, - "server_name": "d10aad520431", + "server_name": "ec6574eadb15", "transactions": 0, "user_connections": 1 - } + }, + "query": [ + "SELECT @@servername AS server_name, @@servicename AS instance_name;", + "SELECT cntr_value As 'active_temp_tables' FROM sys.dm_os_performance_counters WHERE counter_name = 'Active Temp Tables' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'batch_requests_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Batch Requests/sec'", + "SELECT cntr_value As 'buffer_cache_hit_ratio' FROM sys.dm_os_performance_counters WHERE counter_name = 'Buffer cache hit ratio' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_checkpoint_pages_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Checkpoint pages/sec' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_database_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Database pages' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_page_life_expectancy' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page life expectancy' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_target_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Target pages' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Compilations/sec'", + "SELECT cntr_value As 'connection_reset_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Connection Reset/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'lock_waits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Lock Waits/sec' AND instance_name = '_Total'", + "SELECT cntr_value As 'logins_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logins/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'logouts_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logouts/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'page_splits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page splits/sec'", + "SELECT cntr_value As 're_compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Re-Compilations/sec'", + "SELECT cntr_value As 'transactions' FROM sys.dm_os_performance_counters WHERE counter_name = 'Transactions' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'user_connections' FROM sys.dm_os_performance_counters WHERE counter_name= 'User Connections'", + "SELECT counter_name, cntr_value FROM sys.dm_os_performance_counters WHERE counter_name like 'Memory Grants Pend%'" + ] }, "service": { - "address": "elastic-package-service_microsoft_sqlserver_1", + "address": "svc-microsoft_sqlserver", "type": "sql" - } -} \ No newline at end of file + }, + "tags": [ + "preserve_sql_queries" + ] +} diff --git a/packages/microsoft_sqlserver/docs/README.md b/packages/microsoft_sqlserver/docs/README.md index 60072cd7fba..ca828cc5f0c 100644 --- a/packages/microsoft_sqlserver/docs/README.md +++ b/packages/microsoft_sqlserver/docs/README.md @@ -344,69 +344,54 @@ An example event for `performance` looks as following: ```json { - "@timestamp": "2022-11-23T05:03:28.987Z", + "@timestamp": "2025-01-16T06:17:48.647Z", "agent": { - "ephemeral_id": "70f5c0c1-37b1-486b-9806-8105b2cdcd20", - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", - "name": "docker-fleet-agent", + "ephemeral_id": "de13136c-dc52-4b51-87c3-7fe7592f46dc", + "id": "c0b6b234-d11b-4387-ab3b-fa3084042073", + "name": "elastic-agent-74192", "type": "metricbeat", - "version": "8.5.0" - }, - "cloud": { - "account": {}, - "instance": { - "id": "b30e45e6-7900-4900-8d67-e37cb13374bc", - "name": "obs-int-windows-dev" - }, - "machine": { - "type": "Standard_D16ds_v5" - }, - "provider": "azure", - "region": "CentralIndia", - "service": { - "name": "Virtual Machines" - } + "version": "8.18.0" }, "data_stream": { "dataset": "microsoft_sqlserver.performance", - "namespace": "ep", + "namespace": "93265", "type": "metrics" }, "ecs": { - "version": "8.11.0" + "version": "8.0.0" }, "elastic_agent": { - "id": "6d444a4a-2158-445e-8953-dc6eef720a34", - "snapshot": false, - "version": "8.5.0" + "id": "c0b6b234-d11b-4387-ab3b-fa3084042073", + "snapshot": true, + "version": "8.18.0" }, "event": { "agent_id_status": "verified", "dataset": "microsoft_sqlserver.performance", - "duration": 41134100, - "ingested": "2022-11-23T05:03:30Z", + "duration": 71801450, + "ingested": "2025-01-16T06:17:51Z", "module": "sql" }, "host": { "architecture": "x86_64", - "containerized": false, - "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", + "containerized": true, + "hostname": "elastic-agent-74192", "ip": [ - "172.18.0.5" + "192.168.242.2", + "192.168.255.6" ], "mac": [ - "02-42-AC-12-00-05" + "02-42-C0-A8-F2-02", + "02-42-C0-A8-FF-06" ], - "name": "docker-fleet-agent", + "name": "elastic-agent-74192", "os": { - "codename": "focal", - "family": "debian", - "kernel": "5.10.104-linuxkit", - "name": "Ubuntu", - "platform": "ubuntu", + "family": "", + "kernel": "3.10.0-1160.92.1.el7.x86_64", + "name": "Wolfi", + "platform": "wolfi", "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" + "version": "20230201" } }, "metricset": { @@ -416,30 +401,53 @@ An example event for `performance` looks as following: "mssql": { "metrics": { "active_temp_tables": 0, - "batch_requests_per_sec": 54, - "buffer_cache_hit_ratio": 24, - "buffer_checkpoint_pages_per_sec": 105, - "buffer_database_pages": 2215, - "buffer_page_life_expectancy": 16, - "buffer_target_pages": 2408448, - "compilations_per_sec": 80, + "batch_requests_per_sec": 15, + "buffer_cache_hit_ratio": 995, + "buffer_checkpoint_pages_per_sec": 70, + "buffer_database_pages": 2208, + "buffer_page_life_expectancy": 19, + "buffer_target_pages": 1196032, + "compilations_per_sec": 67, "connection_reset_per_sec": 13, "instance_name": "MSSQLSERVER", - "lock_waits_per_sec": 4, - "logins_per_sec": 16, - "logouts_per_sec": 15, + "lock_waits_per_sec": 3, + "logins_per_sec": 3, + "logouts_per_sec": 2, "memory_grants_pending": 0, "page_splits_per_sec": 9, "re_compilations_per_sec": 0, - "server_name": "d10aad520431", + "server_name": "ec6574eadb15", "transactions": 0, "user_connections": 1 - } + }, + "query": [ + "SELECT @@servername AS server_name, @@servicename AS instance_name;", + "SELECT cntr_value As 'active_temp_tables' FROM sys.dm_os_performance_counters WHERE counter_name = 'Active Temp Tables' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'batch_requests_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Batch Requests/sec'", + "SELECT cntr_value As 'buffer_cache_hit_ratio' FROM sys.dm_os_performance_counters WHERE counter_name = 'Buffer cache hit ratio' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_checkpoint_pages_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Checkpoint pages/sec' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_database_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Database pages' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_page_life_expectancy' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page life expectancy' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'buffer_target_pages' FROM sys.dm_os_performance_counters WHERE counter_name = 'Target pages' AND object_name like '%Buffer Manager%'", + "SELECT cntr_value As 'compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Compilations/sec'", + "SELECT cntr_value As 'connection_reset_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Connection Reset/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'lock_waits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Lock Waits/sec' AND instance_name = '_Total'", + "SELECT cntr_value As 'logins_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logins/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'logouts_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Logouts/sec' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'page_splits_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'Page splits/sec'", + "SELECT cntr_value As 're_compilations_per_sec' FROM sys.dm_os_performance_counters WHERE counter_name = 'SQL Re-Compilations/sec'", + "SELECT cntr_value As 'transactions' FROM sys.dm_os_performance_counters WHERE counter_name = 'Transactions' AND object_name like '%General Statistics%'", + "SELECT cntr_value As 'user_connections' FROM sys.dm_os_performance_counters WHERE counter_name= 'User Connections'", + "SELECT counter_name, cntr_value FROM sys.dm_os_performance_counters WHERE counter_name like 'Memory Grants Pend%'" + ] }, "service": { - "address": "elastic-package-service_microsoft_sqlserver_1", + "address": "svc-microsoft_sqlserver", "type": "sql" - } + }, + "tags": [ + "preserve_sql_queries" + ] } ``` @@ -486,6 +494,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | mssql.metrics.server_name | Name of the mssql server. | keyword | | | mssql.metrics.transactions | Total number of transactions | long | gauge | | mssql.metrics.user_connections | Total number of user connections. | long | gauge | +| mssql.query | The SQL queries executed. | keyword | | | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | diff --git a/packages/microsoft_sqlserver/manifest.yml b/packages/microsoft_sqlserver/manifest.yml index 0de0705cbdc..4fd3a2048ae 100644 --- a/packages/microsoft_sqlserver/manifest.yml +++ b/packages/microsoft_sqlserver/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: microsoft_sqlserver title: "Microsoft SQL Server" -version: "2.9.6" +version: "2.10.0" description: Collect events from Microsoft SQL Server with Elastic Agent type: integration categories: