Add index_sorting, synthetic_source_mode and force_merge_max_num_segm…

…ents track parameters to elastic/logs track. (#522) This allows to enable indexing sorting, that either sorts by hostname and timestamp or timestamp and hostname. Additionally this change adds force_merge_max_num_segments and synthetic_source parameters. The former controls the number of segments after indexing and the latter whether synthetic source is enabled. All newly added track params are unset by default, meaning index sorting, synthetic source and force merging to N segments are not enabled.
elastic · Jan 12, 2024 · a8ccf0c · a8ccf0c
1 parent 0ebc6d4
commit a8ccf0c
Show file tree

Hide file tree

Showing 21 changed files with 119 additions and 16 deletions.
diff --git a/elastic/logs/README.md b/elastic/logs/README.md
@@ -246,6 +246,9 @@ The following parameters are available:
 * `disable_pipelines` (default: `false`) - Prevent installing ingest node pipelines. This parameter is experimental and is to be used with indexing-only challenges.
 * `initial_indices_count` (default: 0) - Number of initial indices to create, each containing `100` auditbeat style documents. Parameter is applicable in [many-shards-quantitative challenge](#many-shards-quantitative-many-shards-quantitative) and in [many-shards-snapshots challenge](#many-shards-snapshots-many-shards-snapshots).
 * `ingest_percentage` (default: 100) - The percentage of data to be ingested.
+* `index_sorting` (default: unset): Whether index sorting should be used. Accepted values: `hostname` and `timestamp`. 
+* `synthetic_source_mode` (default: `false`): Whether to enable synthetic source.
+* `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. Only supported in `logging-querying` track.
 
 ### Querying parameters
 

diff --git a/elastic/logs/challenges/logging-querying.json b/elastic/logs/challenges/logging-querying.json
@@ -22,6 +22,52 @@
         "clients": {{ p_bulk_indexing_clients }},
         "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
       }
+      {%- if force_merge_max_num_segments is defined %},
+      {
+        "name": "refresh-after-index",
+        "index": "logs-*",
+        "operation": "refresh"
+      },
+      {
+        "name": "wait-until-index-merges-fininshes",
+        "operation": {
+          "operation-type": "index-stats",
+          "index": "logs-*",
+          "condition": {
+            "path": "_all.total.merges.current",
+            "expected-value": 0
+          },
+          "retry-until-success": true,
+          "include-in-reporting": false
+        }
+      },
+      {
+        "operation": {
+          "operation-type": "force-merge",
+          "index": "logs-*",
+          "request-timeout": 36000,
+          "max-num-segments": {{ force_merge_max_num_segments | tojson }}
+        }
+      },
+      {
+        "name": "wait-until-merges-finish",
+        "operation": {
+          "operation-type": "index-stats",
+          "index": "logs-*",
+          "condition": {
+            "path": "_all.total.merges.current",
+            "expected-value": 0
+          },
+          "retry-until-success": true,
+          "include-in-reporting": false
+        }
+      },
+      {
+        "name": "refresh-after-force-merge",
+        "index": "logs-*",
+        "operation": "refresh"
+      }
+      {%- endif %}
       {# non-serverless-index-statistics-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%},
       {
         "name": "compression-stats",

diff --git a/elastic/logs/templates/component/auditbeat-mappings.json b/elastic/logs/templates/component/auditbeat-mappings.json
@@ -1785,6 +1785,9 @@
             "message": {
               "norms": false,
               "type": "text"
+              {% if synthetic_source_mode | default(false) is true %},
+              "store": true
+              {% endif %}
             },
             "stack_trace": {
               "fields": {
@@ -2941,6 +2944,9 @@
         "message": {
           "norms": false,
           "type": "text"
+          {% if synthetic_source_mode | default(false) is true %},
+          "store": true
+          {% endif %}
         },
         "network": {
           "properties": {

diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json
@@ -0,0 +1,22 @@
+{
+    "template": {
+      "settings": {
+        "index": {
+          {% if index_sorting == "hostname" %}
+            "sort.field": [ "host.name", "@timestamp" ],
+            "sort.order": [ "asc", "desc" ]
+          {% elif index_sorting == "timestamp" %}
+            "sort.field": [ "@timestamp", "host.name" ],
+            "sort.order": [ "desc", "asc" ]
+          {% endif %}
+        }
+      },
+      "mappings": {
+        {% if synthetic_source_mode | default(false) is true %}
+            "_source": {
+                "mode": "synthetic"
+            }
+        {% endif %}
+      }
+    }
+  }
diff --git a/elastic/logs/templates/composable/auditbeat-frozen.json b/elastic/logs/templates/composable/auditbeat-frozen.json
@@ -15,7 +15,7 @@
     "lifecycle": {}
   {%- endif -%}
   },
-  "composed_of" : ["auditbeat-mappings"],
+  "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"],
   "priority": 1,
   "data_stream" : { }
 }
diff --git a/elastic/logs/templates/composable/auditbeat-quantitative.json b/elastic/logs/templates/composable/auditbeat-quantitative.json
@@ -15,7 +15,7 @@
     "lifecycle": {}
   {%- endif -%}
   },
-  "composed_of" : ["auditbeat-mappings"],
+  "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"],
   "priority": 1,
   "data_stream" : { }
 }
diff --git a/elastic/logs/templates/composable/auditbeat.json b/elastic/logs/templates/composable/auditbeat.json
@@ -15,7 +15,7 @@
     "lifecycle": {}
   {%- endif -%}
   },
-  "composed_of" : ["auditbeat-mappings"],
+  "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"],
   "priority": 1,
   "data_stream" : { }
 }
diff --git a/elastic/logs/templates/composable/logs-apache.access.json b/elastic/logs/templates/composable/logs-apache.access.json
@@ -521,7 +521,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-apache.error.json b/elastic/logs/templates/composable/logs-apache.error.json
@@ -477,7 +477,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-k8-application.log.json b/elastic/logs/templates/composable/logs-k8-application.log.json
@@ -2000,7 +2000,8 @@
   "composed_of": [
     "logs-mappings",
     "track-custom-mappings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-kafka.log.json b/elastic/logs/templates/composable/logs-kafka.log.json
@@ -158,6 +158,9 @@
                   "properties": {
                     "message": {
                       "type": "text"
+                      {% if synthetic_source_mode | default(false) is true %},
+                      "store": true
+                      {% endif %}
                     },
                     "class": {
                       "ignore_above": 1024,
@@ -266,11 +269,17 @@
         },
         "message": {
           "type": "text"
+          {% if synthetic_source_mode | default(false) is true %},
+          "store": true
+          {% endif %}
         },
         "error": {
           "properties": {
             "message": {
               "type": "text"
+              {% if synthetic_source_mode | default(false) is true %},
+              "store": true
+              {% endif %}
             }
           }
         },
@@ -287,7 +296,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-mysql.error.json b/elastic/logs/templates/composable/logs-mysql.error.json
@@ -305,7 +305,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-mysql.slowlog.json b/elastic/logs/templates/composable/logs-mysql.slowlog.json
@@ -450,7 +450,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-nginx.access.json b/elastic/logs/templates/composable/logs-nginx.access.json
@@ -461,7 +461,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-nginx.error.json b/elastic/logs/templates/composable/logs-nginx.error.json
@@ -296,7 +296,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-postgresql.log.json b/elastic/logs/templates/composable/logs-postgresql.log.json
@@ -407,7 +407,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-redis.log.json b/elastic/logs/templates/composable/logs-redis.log.json
@@ -278,7 +278,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-redis.slowlog.json b/elastic/logs/templates/composable/logs-redis.slowlog.json
@@ -259,7 +259,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-system.auth.json b/elastic/logs/templates/composable/logs-system.auth.json
@@ -473,7 +473,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/templates/composable/logs-system.syslog.json b/elastic/logs/templates/composable/logs-system.syslog.json
@@ -297,7 +297,8 @@
     ".fleet_component_template-1",
     "track-custom-mappings",
     "track-custom-shared-settings",
-    "track-data-stream-lifecycle"
+    "track-data-stream-lifecycle",
+    "track-shared-logsdb-mode"
   ],
   "priority": 200,
   "_meta": {

diff --git a/elastic/logs/track.json b/elastic/logs/track.json
@@ -303,6 +303,10 @@
     {
         "name": "track-data-stream-lifecycle",
         "template": "./templates/component/track-data-stream-lifecycle.json"
+    },
+    {
+      "name": "track-shared-logsdb-mode",
+      "template": "./templates/component/track-shared-logsdb-mode.json"
     }
   ],
   "composable-templates": [