[Filebeat] Add Initial Interval for Microsoft Filesets

CHANGELOG.next.asciidoc

@@ -431,6 +431,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Add metrics for number of events and pages published by HTTPJSON input. {issue}42340[42340] {pull}42442[42442]
 - Add `etw` input fallback to attach an already existing session. {pull}42847[42847]
 - Update CEL mito extensions to v1.17.0. {pull}42851[42851]
+- Add Initial Interval for Microsoft Filesets (ATP, Defender) {pull}42309[42309]
 
 *Auditbeat*
 

x-pack/filebeat/input/awss3/sqs_s3_event.go

@@ -255,7 +255,7 @@ func (r sqsProcessingResult) Done() {
 			return
 		}
 		p.metrics.sqsMessagesDeletedTotal.Inc()
-		p.log.Errorf("failed processing SQS message (message was deleted): %w", processingErr)
+		p.log.Errorf("failed processing SQS message (message was deleted): %v", processingErr.Error())
 		return
 	}
 

x-pack/filebeat/module/microsoft/_meta/docs.asciidoc

@@ -77,6 +77,10 @@ A predefined URL towards the Oauth2 service for Microsoft. The URL should always
 
 A list of included scopes, should use .default unless different is specified.
 
+*`var.initial_interval`*::
+
+An initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. Following restarts will fetch events starting from the last event read. It defaults to `55m`.
+
 [float]
 ==== 365 Defender ECS fields
 
@@ -153,6 +157,10 @@ The secret related to the client ID.
 
 A predefined URL towards the Oauth2 service for Microsoft. The URL should always be the same with the exception of the Tenant ID that needs to be added to the full URL.
 
+*`var.initial_interval`*::
+
+An initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. Following restarts will fetch events starting from the last event read. It defaults to `5m`.
+
 [float]
 ==== Defender ATP ECS fields
 

x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml

@@ -25,7 +25,7 @@ request.transforms:
   - set:
       target: "url.params.$filter"
       value: 'lastUpdateTime gt [[formatDate .cursor.lastUpdateTime "2006-01-02T15:04:05.9999999Z"]]'
-      default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-5m")) "2006-01-02T15:04:05.9999999Z"]]'
+      default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-{{.initial_interval}}")) "2006-01-02T15:04:05.9999999Z"]]'
 
 response.split:
   target: body.value

x-pack/filebeat/module/microsoft/defender_atp/manifest.yml

@@ -9,6 +9,8 @@ var:
     default: [defender-atp, forwarded]
   - name: oauth2
   - name: proxy_url
+  - name: initial_interval
+    default: 5m
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/atp.yml

x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml

@@ -20,7 +20,7 @@ request.transforms:
   - set:
       target: "url.params.$filter"
       value: 'lastUpdateTime gt [[.cursor.lastUpdateTime]]'
-      default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-55m")) "2006-01-02T15:04:05.9999999Z"]]'
+      default: 'lastUpdateTime gt [[formatDate (now (parseDuration "-{{.initial_interval}}")) "2006-01-02T15:04:05.9999999Z"]]'
 response.split:
   target: body.value
   ignore_empty_value: true

x-pack/filebeat/module/microsoft/m365_defender/manifest.yml

@@ -9,6 +9,8 @@ var:
     default: [m365-defender, forwarded]
   - name: oauth2
   - name: proxy_url
+  - name: initial_interval
+    default: 55m
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/defender.yml