From 546dd73acc8ae124a1ee283305030b9648b7c9d8 Mon Sep 17 00:00:00 2001
From: Xiaokui Shu <subbyte@gmail.com>
Date: Mon, 29 Jul 2024 20:41:30 -0400
Subject: [PATCH] fix kestrel-tool suffix field handling

---
 .../GoldenSAML_Microsoft365DefenderEvents.yaml       |  8 ++++----
 .../kestrel_core/src/kestrel/mapping/fields/ecs.yaml |  4 ++++
 .../src/kestrel/mapping/fields/stix.yaml             |  4 ++++
 packages/kestrel_tool/src/kestrel_tool/mkdb.py       | 12 ++++++++++++
 4 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/mapping-examples/security-datasets/GoldenSAML/GoldenSAML_Microsoft365DefenderEvents.yaml b/mapping-examples/security-datasets/GoldenSAML/GoldenSAML_Microsoft365DefenderEvents.yaml
index 1a24e239..1a74da1b 100644
--- a/mapping-examples/security-datasets/GoldenSAML/GoldenSAML_Microsoft365DefenderEvents.yaml
+++ b/mapping-examples/security-datasets/GoldenSAML/GoldenSAML_Microsoft365DefenderEvents.yaml
@@ -85,14 +85,14 @@ http_request:
 
 # https://schema.ocsf.io/1.1.0/objects/query_info
 query_info:
-    uid: ReportId_long
-    attr_list: AdditionalFields_string.AttributeList
-    search_filter: AdditionalFields_string.SearchFilter
+    uid: ReportId
+    attr_list: AdditionalFields.AttributeList
+    search_filter: AdditionalFields.SearchFilter
 
 
 # https://schema.ocsf.io/1.1.0/objects/managed_entity
 entity:
-    uid: ReportId_long
+    uid: ReportId
     data: ActivityObjects
 
 
diff --git a/packages/kestrel_core/src/kestrel/mapping/fields/ecs.yaml b/packages/kestrel_core/src/kestrel/mapping/fields/ecs.yaml
index 9eafabf2..df3dac35 100644
--- a/packages/kestrel_core/src/kestrel/mapping/fields/ecs.yaml
+++ b/packages/kestrel_core/src/kestrel/mapping/fields/ecs.yaml
@@ -1,6 +1,10 @@
 # "*" is for entity/event projection mapping besides a single field
 # if the submap is referred, there will be multiple reversed mappings, the first will be used
 
+
+time: "@timestamp"
+
+
 # endpoint: see https://schema.ocsf.io/1.1.0/objects/endpoint
 device: &endpoint
     "*": host.*
diff --git a/packages/kestrel_core/src/kestrel/mapping/fields/stix.yaml b/packages/kestrel_core/src/kestrel/mapping/fields/stix.yaml
index 1a01c1ca..a9bf06c6 100644
--- a/packages/kestrel_core/src/kestrel/mapping/fields/stix.yaml
+++ b/packages/kestrel_core/src/kestrel/mapping/fields/stix.yaml
@@ -1,5 +1,9 @@
 # "*" is for entity/event projection mapping besides a single field
 
+
+time: timestamp
+
+
 # https://schema.ocsf.io/1.1.0/objects/file
 file:
     "*": file.*
diff --git a/packages/kestrel_tool/src/kestrel_tool/mkdb.py b/packages/kestrel_tool/src/kestrel_tool/mkdb.py
index 90a03940..a85f4178 100644
--- a/packages/kestrel_tool/src/kestrel_tool/mkdb.py
+++ b/packages/kestrel_tool/src/kestrel_tool/mkdb.py
@@ -49,6 +49,18 @@ def _normalize_event(event: dict) -> dict:
             except json.JSONDecodeError:
                 pass  # maybe it's NOT JSON
 
+    for k in list(event):
+        if k.endswith("_string"):
+            base_key = k[:-7]
+            if base_key not in event or not event[base_key]:
+                event[base_key] = event[k]
+                del event[k]
+        if k.endswith("_long"):
+            base_key = k[:-5]
+            if base_key not in event or not event[base_key]:
+                event[base_key] = int(event[k])
+                del event[k]
+
     return event