From a8ccf0c16515c0aea525dfa737f2db56890710bc Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 12 Jan 2024 10:01:58 +0100 Subject: [PATCH] Add index_sorting, synthetic_source_mode and force_merge_max_num_segments track parameters to elastic/logs track. (#522) This allows to enable indexing sorting, that either sorts by hostname and timestamp or timestamp and hostname. Additionally this change adds force_merge_max_num_segments and synthetic_source parameters. The former controls the number of segments after indexing and the latter whether synthetic source is enabled. All newly added track params are unset by default, meaning index sorting, synthetic source and force merging to N segments are not enabled. --- elastic/logs/README.md | 3 ++ elastic/logs/challenges/logging-querying.json | 46 +++++++++++++++++++ .../component/auditbeat-mappings.json | 6 +++ .../component/track-shared-logsdb-mode.json | 22 +++++++++ .../composable/auditbeat-frozen.json | 2 +- .../composable/auditbeat-quantitative.json | 2 +- .../logs/templates/composable/auditbeat.json | 2 +- .../composable/logs-apache.access.json | 3 +- .../composable/logs-apache.error.json | 3 +- .../composable/logs-k8-application.log.json | 3 +- .../templates/composable/logs-kafka.log.json | 12 ++++- .../composable/logs-mysql.error.json | 3 +- .../composable/logs-mysql.slowlog.json | 3 +- .../composable/logs-nginx.access.json | 3 +- .../composable/logs-nginx.error.json | 3 +- .../composable/logs-postgresql.log.json | 3 +- .../templates/composable/logs-redis.log.json | 3 +- .../composable/logs-redis.slowlog.json | 3 +- .../composable/logs-system.auth.json | 3 +- .../composable/logs-system.syslog.json | 3 +- elastic/logs/track.json | 4 ++ 21 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 elastic/logs/templates/component/track-shared-logsdb-mode.json diff --git a/elastic/logs/README.md b/elastic/logs/README.md index 6ce000e4..3a2b0cec 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -246,6 +246,9 @@ The following parameters are available: * `disable_pipelines` (default: `false`) - Prevent installing ingest node pipelines. This parameter is experimental and is to be used with indexing-only challenges. * `initial_indices_count` (default: 0) - Number of initial indices to create, each containing `100` auditbeat style documents. Parameter is applicable in [many-shards-quantitative challenge](#many-shards-quantitative-many-shards-quantitative) and in [many-shards-snapshots challenge](#many-shards-snapshots-many-shards-snapshots). * `ingest_percentage` (default: 100) - The percentage of data to be ingested. +* `index_sorting` (default: unset): Whether index sorting should be used. Accepted values: `hostname` and `timestamp`. +* `synthetic_source_mode` (default: `false`): Whether to enable synthetic source. +* `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. Only supported in `logging-querying` track. ### Querying parameters diff --git a/elastic/logs/challenges/logging-querying.json b/elastic/logs/challenges/logging-querying.json index c8cc10ac..3b523855 100644 --- a/elastic/logs/challenges/logging-querying.json +++ b/elastic/logs/challenges/logging-querying.json @@ -22,6 +22,52 @@ "clients": {{ p_bulk_indexing_clients }}, "ignore-response-error-level": "{{error_level | default('non-fatal')}}" } + {%- if force_merge_max_num_segments is defined %}, + { + "name": "refresh-after-index", + "index": "logs-*", + "operation": "refresh" + }, + { + "name": "wait-until-index-merges-fininshes", + "operation": { + "operation-type": "index-stats", + "index": "logs-*", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + } + }, + { + "operation": { + "operation-type": "force-merge", + "index": "logs-*", + "request-timeout": 36000, + "max-num-segments": {{ force_merge_max_num_segments | tojson }} + } + }, + { + "name": "wait-until-merges-finish", + "operation": { + "operation-type": "index-stats", + "index": "logs-*", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + } + }, + { + "name": "refresh-after-force-merge", + "index": "logs-*", + "operation": "refresh" + } + {%- endif %} {# non-serverless-index-statistics-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}, { "name": "compression-stats", diff --git a/elastic/logs/templates/component/auditbeat-mappings.json b/elastic/logs/templates/component/auditbeat-mappings.json index 84cfd08f..c63978d1 100644 --- a/elastic/logs/templates/component/auditbeat-mappings.json +++ b/elastic/logs/templates/component/auditbeat-mappings.json @@ -1785,6 +1785,9 @@ "message": { "norms": false, "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} }, "stack_trace": { "fields": { @@ -2941,6 +2944,9 @@ "message": { "norms": false, "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} }, "network": { "properties": { diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json new file mode 100644 index 00000000..e8ee4027 --- /dev/null +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -0,0 +1,22 @@ +{ + "template": { + "settings": { + "index": { + {% if index_sorting == "hostname" %} + "sort.field": [ "host.name", "@timestamp" ], + "sort.order": [ "asc", "desc" ] + {% elif index_sorting == "timestamp" %} + "sort.field": [ "@timestamp", "host.name" ], + "sort.order": [ "desc", "asc" ] + {% endif %} + } + }, + "mappings": { + {% if synthetic_source_mode | default(false) is true %} + "_source": { + "mode": "synthetic" + } + {% endif %} + } + } + } \ No newline at end of file diff --git a/elastic/logs/templates/composable/auditbeat-frozen.json b/elastic/logs/templates/composable/auditbeat-frozen.json index 80bdf0d3..4bb326a4 100644 --- a/elastic/logs/templates/composable/auditbeat-frozen.json +++ b/elastic/logs/templates/composable/auditbeat-frozen.json @@ -15,7 +15,7 @@ "lifecycle": {} {%- endif -%} }, - "composed_of" : ["auditbeat-mappings"], + "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"], "priority": 1, "data_stream" : { } } diff --git a/elastic/logs/templates/composable/auditbeat-quantitative.json b/elastic/logs/templates/composable/auditbeat-quantitative.json index e8927a8e..ab6816a2 100644 --- a/elastic/logs/templates/composable/auditbeat-quantitative.json +++ b/elastic/logs/templates/composable/auditbeat-quantitative.json @@ -15,7 +15,7 @@ "lifecycle": {} {%- endif -%} }, - "composed_of" : ["auditbeat-mappings"], + "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"], "priority": 1, "data_stream" : { } } diff --git a/elastic/logs/templates/composable/auditbeat.json b/elastic/logs/templates/composable/auditbeat.json index d63433c7..1da39f96 100644 --- a/elastic/logs/templates/composable/auditbeat.json +++ b/elastic/logs/templates/composable/auditbeat.json @@ -15,7 +15,7 @@ "lifecycle": {} {%- endif -%} }, - "composed_of" : ["auditbeat-mappings"], + "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"], "priority": 1, "data_stream" : { } } diff --git a/elastic/logs/templates/composable/logs-apache.access.json b/elastic/logs/templates/composable/logs-apache.access.json index ecade6ed..145355d8 100644 --- a/elastic/logs/templates/composable/logs-apache.access.json +++ b/elastic/logs/templates/composable/logs-apache.access.json @@ -521,7 +521,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-apache.error.json b/elastic/logs/templates/composable/logs-apache.error.json index 010bdfa2..3f589e1b 100644 --- a/elastic/logs/templates/composable/logs-apache.error.json +++ b/elastic/logs/templates/composable/logs-apache.error.json @@ -477,7 +477,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-k8-application.log.json b/elastic/logs/templates/composable/logs-k8-application.log.json index 37e886e9..2c8d8bfe 100644 --- a/elastic/logs/templates/composable/logs-k8-application.log.json +++ b/elastic/logs/templates/composable/logs-k8-application.log.json @@ -2000,7 +2000,8 @@ "composed_of": [ "logs-mappings", "track-custom-mappings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-kafka.log.json b/elastic/logs/templates/composable/logs-kafka.log.json index c670bfaa..883aac4e 100644 --- a/elastic/logs/templates/composable/logs-kafka.log.json +++ b/elastic/logs/templates/composable/logs-kafka.log.json @@ -158,6 +158,9 @@ "properties": { "message": { "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} }, "class": { "ignore_above": 1024, @@ -266,11 +269,17 @@ }, "message": { "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} }, "error": { "properties": { "message": { "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} } } }, @@ -287,7 +296,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-mysql.error.json b/elastic/logs/templates/composable/logs-mysql.error.json index 0f130ef6..510b9230 100644 --- a/elastic/logs/templates/composable/logs-mysql.error.json +++ b/elastic/logs/templates/composable/logs-mysql.error.json @@ -305,7 +305,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-mysql.slowlog.json b/elastic/logs/templates/composable/logs-mysql.slowlog.json index 2a32bef4..ac8cea55 100644 --- a/elastic/logs/templates/composable/logs-mysql.slowlog.json +++ b/elastic/logs/templates/composable/logs-mysql.slowlog.json @@ -450,7 +450,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-nginx.access.json b/elastic/logs/templates/composable/logs-nginx.access.json index 23453a08..6003836d 100644 --- a/elastic/logs/templates/composable/logs-nginx.access.json +++ b/elastic/logs/templates/composable/logs-nginx.access.json @@ -461,7 +461,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-nginx.error.json b/elastic/logs/templates/composable/logs-nginx.error.json index a2f3dea1..e411b4ed 100644 --- a/elastic/logs/templates/composable/logs-nginx.error.json +++ b/elastic/logs/templates/composable/logs-nginx.error.json @@ -296,7 +296,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-postgresql.log.json b/elastic/logs/templates/composable/logs-postgresql.log.json index 6dd5862d..68c54142 100644 --- a/elastic/logs/templates/composable/logs-postgresql.log.json +++ b/elastic/logs/templates/composable/logs-postgresql.log.json @@ -407,7 +407,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-redis.log.json b/elastic/logs/templates/composable/logs-redis.log.json index 62eed0ca..8e9d3417 100644 --- a/elastic/logs/templates/composable/logs-redis.log.json +++ b/elastic/logs/templates/composable/logs-redis.log.json @@ -278,7 +278,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-redis.slowlog.json b/elastic/logs/templates/composable/logs-redis.slowlog.json index 4ec11f1b..2bbd8760 100644 --- a/elastic/logs/templates/composable/logs-redis.slowlog.json +++ b/elastic/logs/templates/composable/logs-redis.slowlog.json @@ -259,7 +259,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-system.auth.json b/elastic/logs/templates/composable/logs-system.auth.json index 51cdbe39..1f795226 100644 --- a/elastic/logs/templates/composable/logs-system.auth.json +++ b/elastic/logs/templates/composable/logs-system.auth.json @@ -473,7 +473,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-system.syslog.json b/elastic/logs/templates/composable/logs-system.syslog.json index 87828803..6585fd64 100644 --- a/elastic/logs/templates/composable/logs-system.syslog.json +++ b/elastic/logs/templates/composable/logs-system.syslog.json @@ -297,7 +297,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/track.json b/elastic/logs/track.json index b751659b..378b62f8 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -303,6 +303,10 @@ { "name": "track-data-stream-lifecycle", "template": "./templates/component/track-data-stream-lifecycle.json" + }, + { + "name": "track-shared-logsdb-mode", + "template": "./templates/component/track-shared-logsdb-mode.json" } ], "composable-templates": [