From 07edd3048d31b2dfeb68ee499589fab0c602f110 Mon Sep 17 00:00:00 2001 From: DumbBoi Date: Wed, 15 Jan 2025 02:52:55 +0500 Subject: [PATCH 1/4] add initial interval --- x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml | 2 +- x-pack/filebeat/module/microsoft/defender_atp/manifest.yml | 2 ++ .../filebeat/module/microsoft/m365_defender/config/defender.yml | 2 +- x-pack/filebeat/module/microsoft/m365_defender/manifest.yml | 2 ++ 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml index 9107c2db3a76..c6eaf3816278 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml @@ -25,7 +25,7 @@ request.transforms: - set: target: "url.params.$filter" value: 'lastUpdateTime gt [[formatDate .cursor.lastUpdateTime "2006-01-02T15:04:05.9999999Z"]]' - default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05.9999999Z"]]' + default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-{{.initial_interval}}")) "2006-01-02T15:04:05.9999999Z"]]' response.split: target: body.value diff --git a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml index 2bf5bf65034b..105125033d68 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml @@ -9,6 +9,8 @@ var: default: [defender-atp, forwarded] - name: oauth2 - name: proxy_url + - name: initial_interval + default: 24h ingest_pipeline: ingest/pipeline.yml input: config/atp.yml diff --git a/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml index 3d8747586153..70ce998baa9f 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml @@ -20,7 +20,7 @@ request.transforms: - set: target: "url.params.$filter" value: 'lastUpdateTime gt [[.cursor.lastUpdateTime]]' - default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-55m")) "2006-01-02T15:04:05.9999999Z"]]' + default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-{{.initial_interval}}")) "2006-01-02T15:04:05.9999999Z"]]' response.split: target: body.value ignore_empty_value: true diff --git a/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml b/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml index e3524259d08c..2eb9181fe1bd 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml @@ -9,6 +9,8 @@ var: default: [m365-defender, forwarded] - name: oauth2 - name: proxy_url + - name: initial_interval + default: 24h ingest_pipeline: ingest/pipeline.yml input: config/defender.yml From 29bc91820c7e6b2649b14c11aad9d27471c6afa3 Mon Sep 17 00:00:00 2001 From: DumbBoi Date: Wed, 15 Jan 2025 03:05:12 +0500 Subject: [PATCH 2/4] changing default values to previously hardcoded --- x-pack/filebeat/module/microsoft/defender_atp/manifest.yml | 2 +- x-pack/filebeat/module/microsoft/m365_defender/manifest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml index 105125033d68..ab72a6a68233 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml @@ -10,7 +10,7 @@ var: - name: oauth2 - name: proxy_url - name: initial_interval - default: 24h + default: 5m ingest_pipeline: ingest/pipeline.yml input: config/atp.yml diff --git a/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml b/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml index 2eb9181fe1bd..b312a2330782 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml @@ -10,7 +10,7 @@ var: - name: oauth2 - name: proxy_url - name: initial_interval - default: 24h + default: 55m ingest_pipeline: ingest/pipeline.yml input: config/defender.yml From bd44f14dda36afe5adbae8bbd17741559a8c384b Mon Sep 17 00:00:00 2001 From: DumbBoi Date: Tue, 4 Mar 2025 12:02:58 +0500 Subject: [PATCH 3/4] adding doc and changelog --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/module/microsoft/_meta/docs.asciidoc | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8234289871a3..bb99c0ad1dee 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -431,6 +431,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403] - Add metrics for number of events and pages published by HTTPJSON input. {issue}42340[42340] {pull}42442[42442] - Add `etw` input fallback to attach an already existing session. {pull}42847[42847] - Update CEL mito extensions to v1.17.0. {pull}42851[42851] +- Add Initial Interval for Microsoft Filesets (ATP, Defender) {pull}42309[42309] *Auditbeat* diff --git a/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc b/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc index 49b4c9e4cb1d..66835ad5bc50 100644 --- a/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc @@ -77,6 +77,10 @@ A predefined URL towards the Oauth2 service for Microsoft. The URL should always A list of included scopes, should use .default unless different is specified. +*`var.initial_interval`*:: + +An initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. Following restarts will fetch events starting from the last event read. It defaults to `55m`. + [float] ==== 365 Defender ECS fields @@ -153,6 +157,10 @@ The secret related to the client ID. A predefined URL towards the Oauth2 service for Microsoft. The URL should always be the same with the exception of the Tenant ID that needs to be added to the full URL. +*`var.initial_interval`*:: + +An initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. Following restarts will fetch events starting from the last event read. It defaults to `5m`. + [float] ==== Defender ATP ECS fields From c306e25db94ff5e046681b6a5af398cf31f6cd84 Mon Sep 17 00:00:00 2001 From: DumbBoi Date: Tue, 4 Mar 2025 12:05:14 +0500 Subject: [PATCH 4/4] fix sqs logging --- x-pack/filebeat/input/awss3/sqs_s3_event.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/input/awss3/sqs_s3_event.go b/x-pack/filebeat/input/awss3/sqs_s3_event.go index cb39376f7b47..980b750b390a 100644 --- a/x-pack/filebeat/input/awss3/sqs_s3_event.go +++ b/x-pack/filebeat/input/awss3/sqs_s3_event.go @@ -255,7 +255,7 @@ func (r sqsProcessingResult) Done() { return } p.metrics.sqsMessagesDeletedTotal.Inc() - p.log.Errorf("failed processing SQS message (message was deleted): %w", processingErr) + p.log.Errorf("failed processing SQS message (message was deleted): %v", processingErr.Error()) return }