Merge branch 'main' into poc

.github/workflows/pre-commit.yml

@@ -7,6 +7,7 @@ on:
       - main
       - 7.1*
       - 8.*
+      - 9.*
 
 permissions:
   contents: read

CHANGELOG.next.asciidoc

@@ -82,6 +82,7 @@ otherwise no tag is added. {issue}42208[42208] {pull}42403[42403]
 - Add support for specifying a custom endpoint for GCP service clients. {issue}40848[40848] {pull}40918[40918]
 - Fix incorrect handling of types in SQL module. {issue}40090[40090] {pull}41607[41607]
 - Remove kibana.settings metricset since the API was removed in 8.0 {issue}30592[30592] {pull}42937[42937]
+- Removed support for the Enterprise Search module {pull}42915[42915]
 
 *Osquerybeat*
 

auditbeat/auditbeat.reference.yml

@@ -80,6 +80,9 @@ auditbeat.modules:
   - /usr/sbin
   - /etc
 
+  # Detect changes to files included in subdirectories. Disabled by default.
+  recursive: false
+
   # List of regular expressions to filter out notifications for unwanted files.
   # Wrap in single quotes to workaround YAML escaping rules. By default no files
   # are ignored.
@@ -88,6 +91,10 @@ auditbeat.modules:
   - '~$'
   - '/\.git($|/)'
 
+  # If recursive is set to true, subdirectories can also be excluded here by
+  #  specifying them.
+  # - '(?i)/tmp/'
+
   # List of regular expressions used to explicitly include files. When configured,
   # Auditbeat will ignore files unless they match a pattern.
   #include_files:
@@ -115,9 +122,6 @@ auditbeat.modules:
   # Default is sha1.
   hash_types: [sha1]
 
-  # Detect changes to files included in subdirectories. Disabled by default.
-  recursive: false
-
   # Set to true to publish fields with null values in events.
   #keep_null: false
 

auditbeat/docs/modules/file_integrity.asciidoc

@@ -19,7 +19,7 @@ This module uses features of the operating system to monitor file changes in
 realtime. When the module starts it creates a subscription with the OS to
 receive notifications of changes to the specified files or directories. Upon
 receiving notification of a change the module will read the file's metadata
-and the compute a hash of the file's contents.
+and then compute a hash of the file's contents.
 
 At startup this module will perform an initial scan of the configured files
 and directories to generate baseline data for the monitored paths and detect
@@ -62,6 +62,7 @@ Linux.
   - /sbin
   - /usr/sbin
   - /etc
+  recursive: false
   exclude_files:
   - '(?i)\.sw[nop]$'
   - '~$'
@@ -71,7 +72,6 @@ Linux.
   scan_rate_per_sec: 50 MiB
   max_file_size: 100 MiB
   hash_types: [sha1]
-  recursive: false
 ----
 
 This module also supports the
@@ -88,6 +88,12 @@ the watch target. If the link does not resolve to a valid target, the symbolic
 link itself will be watched; if the symlink target becomes valid after module
 start up this will not be picked up by the file system watches.
 
+*`recursive`*:: By default, the watches set to the paths specified in
+`paths` are not recursive. This means that only changes to the contents
+of this directories are watched. If `recursive` is set to `true`, the
+`file_integrity` module will watch for changes on this directory and all
+its subdirectories.
+
 *`exclude_files`*:: A list of regular expressions used to filter out events
 for unwanted files. The expressions are matched against the full path of every
 file and directory. When used in conjunction with `include_files`, file paths need
@@ -96,6 +102,8 @@ By default, no files are excluded. See <<regexp-support>>
 for a list of supported regexp patterns. It is recommended to wrap regular
 expressions in single quotation marks to avoid issues with YAML escaping
 rules.
+If `recursive` is set to true, subdirectories can also be excluded here by
+specifying them.
 
 *`include_files`*:: A list of regular expressions used to specify which files to
 select. When configured, only files matching the pattern will be monitored.
@@ -144,12 +152,6 @@ when a file event involves the file. Files that are not targets of the specific
 file parser are only sniffed to examine whether analysis should proceed. This will
 usually only involve reading a small number of bytes.
 
-*`recursive`*:: By default, the watches set to the paths specified in
-`paths` are not recursive. This means that only changes to the contents
-of this directories are watched. If `recursive` is set to `true`, the
-`file_integrity` module will watch for changes on this directories and all
-their subdirectories.
-
 *`backend`*:: (*Linux only*) Select the backend which will be used to
 source events. Valid values: `auto`, `fsnotify`, `kprobes`, `ebpf`. Default: `fsnotify`.
 

auditbeat/module/file_integrity/_meta/config.yml.tmpl

@@ -27,22 +27,37 @@
 {{ end -}}
 
 {{- if .Reference }}
+  # Detect changes to files included in subdirectories. Disabled by default.
+  recursive: false
+
   # List of regular expressions to filter out notifications for unwanted files.
   # Wrap in single quotes to workaround YAML escaping rules. By default no files
   # are ignored.
   {{- if eq .GOOS "darwin" }}
   exclude_files:
   - '\.DS_Store$'
   - '\.swp$'
+
+  # If recursive is set to true, subdirectories can also be excluded here by
+  #  specifying them.
+  # - '(?i)/tmp/'
   {{- else if eq .GOOS "windows" }}
   exclude_files:
   - '(?i)\.lnk$'
   - '(?i)\.swp$'
+
+  # If recursive is set to true, subdirectories can also be excluded here by
+  #  specifying them.
+  #  - '(?i)\\temp\\'
   {{- else }}
   exclude_files:
   - '(?i)\.sw[nop]$'
   - '~$'
   - '/\.git($|/)'
+
+  # If recursive is set to true, subdirectories can also be excluded here by
+  #  specifying them.
+  # - '(?i)/tmp/'
   {{- end }}
 
   # List of regular expressions used to explicitly include files. When configured,
@@ -80,9 +95,6 @@
   # Default is sha1.
   hash_types: [sha1]
 
-  # Detect changes to files included in subdirectories. Disabled by default.
-  recursive: false
-
   # Set to true to publish fields with null values in events.
   #keep_null: false
 

auditbeat/module/file_integrity/_meta/docs.asciidoc

@@ -12,7 +12,7 @@ This module uses features of the operating system to monitor file changes in
 realtime. When the module starts it creates a subscription with the OS to
 receive notifications of changes to the specified files or directories. Upon
 receiving notification of a change the module will read the file's metadata
-and the compute a hash of the file's contents.
+and then compute a hash of the file's contents.
 
 At startup this module will perform an initial scan of the configured files
 and directories to generate baseline data for the monitored paths and detect
@@ -55,6 +55,7 @@ Linux.
   - /sbin
   - /usr/sbin
   - /etc
+  recursive: false
   exclude_files:
   - '(?i)\.sw[nop]$'
   - '~$'
@@ -64,7 +65,6 @@ Linux.
   scan_rate_per_sec: 50 MiB
   max_file_size: 100 MiB
   hash_types: [sha1]
-  recursive: false
 ----
 
 This module also supports the
@@ -81,6 +81,12 @@ the watch target. If the link does not resolve to a valid target, the symbolic
 link itself will be watched; if the symlink target becomes valid after module
 start up this will not be picked up by the file system watches.
 
+*`recursive`*:: By default, the watches set to the paths specified in
+`paths` are not recursive. This means that only changes to the contents
+of this directories are watched. If `recursive` is set to `true`, the
+`file_integrity` module will watch for changes on this directory and all
+its subdirectories.
+
 *`exclude_files`*:: A list of regular expressions used to filter out events
 for unwanted files. The expressions are matched against the full path of every
 file and directory. When used in conjunction with `include_files`, file paths need
@@ -89,6 +95,8 @@ By default, no files are excluded. See <<regexp-support>>
 for a list of supported regexp patterns. It is recommended to wrap regular
 expressions in single quotation marks to avoid issues with YAML escaping
 rules.
+If `recursive` is set to true, subdirectories can also be excluded here by
+specifying them.
 
 *`include_files`*:: A list of regular expressions used to specify which files to
 select. When configured, only files matching the pattern will be monitored.
@@ -137,12 +145,6 @@ when a file event involves the file. Files that are not targets of the specific
 file parser are only sniffed to examine whether analysis should proceed. This will
 usually only involve reading a small number of bytes.
 
-*`recursive`*:: By default, the watches set to the paths specified in
-`paths` are not recursive. This means that only changes to the contents
-of this directories are watched. If `recursive` is set to `true`, the
-`file_integrity` module will watch for changes on this directories and all
-their subdirectories.
-
 *`backend`*:: (*Linux only*) Select the backend which will be used to
 source events. Valid values: `auto`, `fsnotify`, `kprobes`, `ebpf`. Default: `fsnotify`.
 

dev-tools/mage/module_changeset.go

@@ -91,6 +91,12 @@ func shouldIgnore(file string) bool {
 			return true
 		}
 	}
+
+	// if the file has been removed, we should ignore it
+	if _, err := os.Stat(file); os.IsNotExist(err) {
+		return true
+	}
+
 	return false
 }
 

go.mod

@@ -153,15 +153,15 @@ require (
 	cloud.google.com/go v0.116.0
 	cloud.google.com/go/compute v1.29.0
 	cloud.google.com/go/redis v1.17.2
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
-	github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.2.1
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
+	github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.3.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/consumption/armconsumption v1.1.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.8.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/costmanagement/armcostmanagement v1.1.1
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.8.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Azure/go-autorest/autorest/adal v0.9.24
 	github.com/aerospike/aerospike-client-go/v7 v7.7.1
@@ -246,14 +246,14 @@ require (
 	github.com/Azure/azure-amqp-common-go/v4 v4.2.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
-	github.com/Azure/go-amqp v1.0.5 // indirect
+	github.com/Azure/go-amqp v1.3.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect