diff --git a/metricbeat/docs/modules/windows.asciidoc b/metricbeat/docs/modules/windows.asciidoc index 9a9552be72c0..0d92481e9509 100644 --- a/metricbeat/docs/modules/windows.asciidoc +++ b/metricbeat/docs/modules/windows.asciidoc @@ -55,24 +55,28 @@ metricbeat.modules: - module: windows metricsets: ["wmi"] - period: 60s + period: 10m wmi: - include_null: false # Exclude fields with null values from the output - include_queries: false # Do not include the query string in the output - include_empty_string: false # Exclude fields with empty string values from the output - warning_threshold: 30s # Maximum time to wait for a query result before logging a warning (defaults to period) + # Do not include the query string in the output + include_queries: false + # Exclude properties with null values from the output + include_null_properties: false + # Exclude properties with empty string values from the output + include_empty_string_properties: false + # Maximum time to wait for a query result before logging a warning (defaults to period) + warning_threshold: 10m # Default WMI namespace for all queries (if not specified per query) # Uncomment to override the default, which is "root\\cimv2". # namespace: "root\\cimv2" queries: - - class: Win32_OperatingSystem # FROM: Class to fetch - fields: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties + - class: Win32_OperatingSystem # FROM: Class to fetch + properties: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties - FreePhysicalMemory - FreeSpaceInPagingFiles - FreeVirtualMemory - LocalDateTime - NumberOfUsers - where: "" # Optional WHERE clause to filter query results + where: "" # Optional WHERE clause to filter query results # Override the WMI namespace for this specific query (optional). # If set, this takes precedence over the default namespace above. # namespace: "root\\cimv2" # Overrides the metric diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 0d7ceb34057c..5f1288ef455a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1093,24 +1093,28 @@ metricbeat.modules: - module: windows metricsets: ["wmi"] - period: 60s + period: 10m wmi: - include_null: false # Exclude fields with null values from the output - include_queries: false # Do not include the query string in the output - include_empty_string: false # Exclude fields with empty string values from the output - warning_threshold: 30s # Maximum time to wait for a query result before logging a warning (defaults to period) + # Do not include the query string in the output + include_queries: false + # Exclude properties with null values from the output + include_null_properties: false + # Exclude properties with empty string values from the output + include_empty_string_properties: false + # Maximum time to wait for a query result before logging a warning (defaults to period) + warning_threshold: 10m # Default WMI namespace for all queries (if not specified per query) # Uncomment to override the default, which is "root\\cimv2". # namespace: "root\\cimv2" queries: - - class: Win32_OperatingSystem # FROM: Class to fetch - fields: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties + - class: Win32_OperatingSystem # FROM: Class to fetch + properties: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties - FreePhysicalMemory - FreeSpaceInPagingFiles - FreeVirtualMemory - LocalDateTime - NumberOfUsers - where: "" # Optional WHERE clause to filter query results + where: "" # Optional WHERE clause to filter query results # Override the WMI namespace for this specific query (optional). # If set, this takes precedence over the default namespace above. # namespace: "root\\cimv2" # Overrides the metric diff --git a/metricbeat/module/windows/_meta/config.reference.yml b/metricbeat/module/windows/_meta/config.reference.yml index d033f457c48f..bbbc123e0c86 100644 --- a/metricbeat/module/windows/_meta/config.reference.yml +++ b/metricbeat/module/windows/_meta/config.reference.yml @@ -20,24 +20,28 @@ - module: windows metricsets: ["wmi"] - period: 60s + period: 10m wmi: - include_null: false # Exclude fields with null values from the output - include_queries: false # Do not include the query string in the output - include_empty_string: false # Exclude fields with empty string values from the output - warning_threshold: 30s # Maximum time to wait for a query result before logging a warning (defaults to period) + # Do not include the query string in the output + include_queries: false + # Exclude properties with null values from the output + include_null_properties: false + # Exclude properties with empty string values from the output + include_empty_string_properties: false + # Maximum time to wait for a query result before logging a warning (defaults to period) + warning_threshold: 10m # Default WMI namespace for all queries (if not specified per query) # Uncomment to override the default, which is "root\\cimv2". # namespace: "root\\cimv2" queries: - - class: Win32_OperatingSystem # FROM: Class to fetch - fields: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties + - class: Win32_OperatingSystem # FROM: Class to fetch + properties: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties - FreePhysicalMemory - FreeSpaceInPagingFiles - FreeVirtualMemory - LocalDateTime - NumberOfUsers - where: "" # Optional WHERE clause to filter query results + where: "" # Optional WHERE clause to filter query results # Override the WMI namespace for this specific query (optional). # If set, this takes precedence over the default namespace above. # namespace: "root\\cimv2" # Overrides the metric diff --git a/metricbeat/module/windows/_meta/config.yml b/metricbeat/module/windows/_meta/config.yml index 45a9a709d725..3b4352db4c36 100644 --- a/metricbeat/module/windows/_meta/config.yml +++ b/metricbeat/module/windows/_meta/config.yml @@ -19,13 +19,13 @@ # - module: windows # metricsets: # - wmi -# period: 60s +# period: 10m # wmi: -# warning_threshold: 30s +# warning_threshold: 10m # # namespace: "root\\cimv2" # queries: # - class: Win32_OperatingSystem # FROM: Class to fetch -# fields: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties +# properties: # SELECT: Properties to retrieve for this WMI class. Omit the setting to fetch all properties # - FreePhysicalMemory # - FreeSpaceInPagingFiles # - FreeVirtualMemory diff --git a/metricbeat/module/windows/wmi/_meta/data.json b/metricbeat/module/windows/wmi/_meta/data.json index f8f6205791d3..7bfda4bff677 100644 --- a/metricbeat/module/windows/wmi/_meta/data.json +++ b/metricbeat/module/windows/wmi/_meta/data.json @@ -20,7 +20,6 @@ "LocalDateTime": "2024-12-12T15:46:39.62Z", "NumberOfUsers": 1, "class": "Win32_OperatingSystem", - "host": "localhost", "namespace": "root\\cimv2" } } diff --git a/metricbeat/module/windows/wmi/_meta/docs.asciidoc b/metricbeat/module/windows/wmi/_meta/docs.asciidoc index 94dc73b67f0b..f4ca4dc47cbe 100644 --- a/metricbeat/module/windows/wmi/_meta/docs.asciidoc +++ b/metricbeat/module/windows/wmi/_meta/docs.asciidoc @@ -1,44 +1,143 @@ -The `wmi` metricset of the Windows module reads metrics via Windows Management Instrumentation link:https://learn.microsoft.com/en-us/windows/win32/wmisdk/about-wmi[(WMI)], a core management technology in the Windows Operating system. +The `wmi` metricset of the Windows module collects metrics via link:https://learn.microsoft.com/en-us/windows/win32/wmisdk/about-wmi[Windows Management Instrumentation (WMI)], a core management technology in the Windows Operating system. By leveraging WMI Query Language (WQL), this metricset allows you to extract detailed system information and metrics to monitor the health and performance of Windows Systems. -This metricset leverages the link:https://github.com/microsoft/wmi[Microsoft WMI], library a -convenient wrapper around the link:https://github.com/go-ole[GO-OLE] library which allows to -invoke the WMI Api. +This metricset leverages the link:https://github.com/microsoft/wmi[Microsoft WMI] library, a +convenient wrapper around the link:https://github.com/go-ole[Go-OLE] library. This allows invoking the +link:https://learn.microsoft.com/en-us/windows/win32/wmisdk/scripting-api-for-wmi[Scripting API for WMI]. +[float] === WMI Query Language (WQL) Support This metricset supports the execution of link:https://learn.microsoft.com/en-us/windows/win32/wmisdk/wql-sql-for-wmi[WQL] queries, a SQL-like query language for retrieving information from WMI namespaces. -As of now, we only support and execute queries with `SELECT`, `FROM` and `WHERE` clauses. +Currently, the metricset supports queries with `SELECT`, `FROM` and `WHERE` clauses. + +NOTE: When working with WMI queries, it is the user's responsibility to ensure that queries are safe, efficient, and do not cause unintended side effects. A notorious example of a problematic WMI class is Win32_Product. Read more in link:https://support.microsoft.com/kb/974524[Windows Documentation]. + +[float] +[[wmi-arbitrator-and-query-execution]] +==== WMI Arbitrator and Query Execution + +Query execution is managed by the underlying WMI Framework, specifically the link:https://learn.microsoft.com/en-us/troubleshoot/windows-server/system-management-components/new-wmi-arbitrator-behavior-in-windows-server[WMI Arbitrator]. +The Arbitrator is responsible for: + +- Scheduling and controlling query execution +- Throttling or stopping queries based on system resource availability and conditions +There is no way to directly stop a query once it has started. To prevent Metricbeat from waiting indefinitely for a query to return a result or fail, Metricbeat has a timeout mechanism that stops waiting for query results after a specified timeout. This is controlled by the `wmi.warning_threshold` setting. + +NOTE: While Metricbeat stops waiting for the result, the underlying WMI query may continue running until the WMI Arbitrator decides to stop execution. + + +[float] === Configuration [source,yaml] ---- - module: windows metricsets: ["wmi"] - period: 60s - namespace: "root\\cimv2" # Namespace - queries: - - class: Win32_OperatingSystem - fields: - - FreePhysicalMemory - - FreeSpaceInPaginFiles - - NumberOfUsers - # Where Clasue - where: "" + period: 10m + wmi: + namespace: "root\\cimv2" # Default Namespace + warning_threshold: 10m + include_queries: true + include_null_properties: false + include_empty_strings_properties: false + queries: + - class: Win32_OperatingSystem + properties: + - FreePhysicalMemory + - FreeSpaceInPagingFiles + - NumberOfUsers + where: "" + - class: Win32_PowerPlan + properties: [] + where: "IsActive = True" + namespace: "root\\cimv2\\power" # Overwrites the module namespace in this query +---- + +*`wmi.namespace`*:: +The default WMI namespace used for queries. This can be overridden per query. +The default is `root\cimv2`. + +*`wmi.warning_threshold`*:: The time threshold after which Metricbeat will stop +waiting for the query result and return control to the main flow of the program. +A warning is logged indicating that the query execution has exceeded the threshold. +The default is equal to the period. See <> +for more details. + +*`wmi.include_queries`*:: If set to `true` the metricset includes the query in the output document. The default value is `false`. + +*`wmi.include_null_properties`*:: If set to `true` the metricset includes the properties that have null value in the output document. +properties that have a `null` value in the output document. The default value is `false`. + +*`wmi.include_empty_string_properties`*:: A boolean option that causes the metricset to include +the properties that are empty string. The default value is `false`. + +*`wmi.queries`*:: The list of queries to execute. The list cannot be empty. See <> for the format of the queries. + +[float] +[[query-configuration]] +==== Query Configuration + +Each item in the `queries` list specifies a wmi query to perform. + +*`class`*:: The wmi class. In the query it specifies the `FROM` clause. Required + +*`properties`*:: List of properties to return. In the query it specifies the `SELECT` clause. Set it to the empty list (default value) to retrieve all available properties. + +*`where`*:: The where clause. In the query it specifies the `WHERE` clause. Read more about the format link:https://learn.microsoft.com/en-us/windows/win32/wmisdk/where-clause[in the Windows Documentation]. + +*`namespace`*:: The WMI Namespace for this particular query (it overwrites the metricset's `namespace` value) + +[float] +===== Example + +Example WQL Query: + +[source,sql] +---- +SELECT Name, ProcessId, WorkingSetSize +FROM Win32_Process +WHERE Name = 'lsass.exe' AND WorkingSetSize > 104857600 ---- +Equivalent YAML Configuration: + +[source,yaml] +---- +- class: Win32_Process + properties: + - Name + - ProcessId + - WorkingSetSize + where: "Name = 'lsass.exe' AND WorkingSetSize > 104857600" +---- + + +[float] +=== Best Practices + +- Test your queries in isolation using the `Get-CimInstance` PowerShell cmdlet or the WMI Explorer. + +- Ensure that `wmi.warning_threshold` is **less than or equal to** the module's `period`. + This prevents starting intentionally multiple executions of the same query. + +- Set up alerts in Metricbeat logs for timeouts and empty query results. If a query frequently times out or returns no data, investigate the cause to prevent missing critical information. + +- [Advanced] Collect WMI-Activity Operational Logs to correlate with Metricbeat WMI warnings. + + [float] === Compatibility This module has been tested on the following platform: - Operating System: Microsoft Windows Server 2019 Datacenter -- Architecture: x86 +- Architecture: x64 Other Windows versions and architectures may also work but have not been explicitly tested. diff --git a/metricbeat/module/windows/wmi/config.go b/metricbeat/module/windows/wmi/config.go index ad0e4d02df9a..d1f642a6c20c 100644 --- a/metricbeat/module/windows/wmi/config.go +++ b/metricbeat/module/windows/wmi/config.go @@ -31,32 +31,37 @@ import ( ) type Config struct { - IncludeQueries bool `config:"wmi.include_queries"` // Determines if the query string should be included in the output document - IncludeNull bool `config:"wmi.include_null"` // Specifies whether to include fields with nil values in the final document - IncludeEmptyString bool `config:"wmi.include_empty_string"` // Specifies whether to include fields with empty string values in the final document - Host string `config:"wmi.host"` // Hostname or IP address of the remote WMI server - User string `config:"wmi.username"` // Username for authentication on the remote WMI server - Password string `config:"wmi.password"` // Password for authentication on the remote WMI server - Namespace string `config:"wmi.namespace"` // Default WMI namespace for executing queries, used if not overridden by individual query configurations - Queries []QueryConfig `config:"wmi.queries"` // List of WMI query configurations - WarningThreshold time.Duration `config:"wmi.warning_threshold"` // Maximum duration to wait for query results before logging a warning. The query will continue running in WMI but will no longer be awaited - NamespaceQueryIndex map[string][]QueryConfig // Internal structure indexing queries by namespace to reduce the number of WMI connections required per execution + IncludeQueries bool `config:"wmi.include_queries"` // Determines if the query string should be included in the output document + IncludeNullProperties bool `config:"wmi.include_null_properties"` // Specifies whether to include properties with nil values in the final document + IncludeEmptyStringProperties bool `config:"wmi.include_empty_string_properties"` // Specifies whether to include properties with empty string values in the final document + Namespace string `config:"wmi.namespace"` // Default WMI namespace for executing queries, used if not overridden by individual query configurations + Queries []QueryConfig `config:"wmi.queries"` // List of WMI query configurations + WarningThreshold time.Duration `config:"wmi.warning_threshold"` // Maximum duration to wait for query results before logging a warning. The query will continue running in WMI but will no longer be awaited + NamespaceQueryIndex map[string][]QueryConfig // Internal structure indexing queries by namespace to reduce the number of WMI connections required per execution + // Remote WMI Parameters + // These parameters are intentionally hidden to discourage their use. + // If you need access, please open a support ticket to request exposure. + Host string // Hostname of the remote WMI server + Domain string // Domain of the remote WMI Server + User string // Username for authentication on the remote WMI server + Password string // Password for authentication on the remote WMI server } type QueryConfig struct { - QueryStr string // The compiled query string generated internally (not user-configurable) - Class string `config:"class"` // WMI class to query (used in the FROM clause) - Fields []string `config:"fields"` // List of properties to retrieve (used in the SELECT clause). If omitted, all properties of the class are fetched - Where string `config:"where"` // Custom WHERE clause to filter query results. The provided string is used directly in the query - Namespace string `config:"namespace"` // WMI namespace for the query. This takes precedence over the globally configured namespace + QueryStr string // The compiled query string generated internally (not user-configurable) + Class string `config:"class"` // WMI class to query (used in the FROM clause) + Properties []string `config:"properties"` // List of properties to retrieve (used in the SELECT clause). If omitted, all properties of the class are fetched + Where string `config:"where"` // Custom WHERE clause to filter query results. The provided string is used directly in the query + Namespace string `config:"namespace"` // WMI namespace for the query. This takes precedence over the globally configured namespace } func NewDefaultConfig() Config { return Config{ - IncludeQueries: false, - IncludeNull: false, - Host: "localhost", - Namespace: WMIDefaultNamespace, + IncludeQueries: false, + IncludeNullProperties: false, + IncludeEmptyStringProperties: false, + Host: "localhost", + Namespace: WMIDefaultNamespace, } } @@ -72,11 +77,11 @@ func (c *Config) ValidateConnectionParameters() error { func (qc *QueryConfig) compileQuery() { // Let us normalize the case where the array is ['*'] // To the Empty Array - if len(qc.Fields) == 1 && qc.Fields[0] == "*" { - qc.Fields = []string{} + if len(qc.Properties) == 1 && qc.Properties[0] == "*" { + qc.Properties = []string{} } - query := wmiquery.NewWmiQueryWithSelectList(qc.Class, qc.Fields, []string{}...) + query := wmiquery.NewWmiQueryWithSelectList(qc.Class, qc.Properties, []string{}...) queryStr := query.String() // Concatenating the where clause manually, because the library supports only a subset of where clauses // while we want to leverage all filtering capabilities diff --git a/metricbeat/module/windows/wmi/config_test.go b/metricbeat/module/windows/wmi/config_test.go index 443e05e5e35e..079c835027a4 100644 --- a/metricbeat/module/windows/wmi/config_test.go +++ b/metricbeat/module/windows/wmi/config_test.go @@ -30,7 +30,8 @@ func TestNewDefaultConfig(t *testing.T) { cfg := NewDefaultConfig() assert.False(t, cfg.IncludeQueries, "IncludeQueries should default to false") - assert.False(t, cfg.IncludeNull, "IncludeNull should default to false") + assert.False(t, cfg.IncludeNullProperties, "IncludeNullProperties should default to false") + assert.False(t, cfg.IncludeEmptyStringProperties, "IncludeEmptyStringProperties should default to false") assert.Equal(t, WMIDefaultNamespace, cfg.Namespace, "Namespace should default to WMIDefaultNamespace") assert.Empty(t, cfg.Queries, "Queries should default to an empty slice") } @@ -88,9 +89,9 @@ func TestCompileQueries(t *testing.T) { config: Config{ Queries: []QueryConfig{ { - Class: "Win32_Process", - Fields: []string{"Name", "ID"}, - Where: "Name LIKE 'chrome%'", + Class: "Win32_Process", + Properties: []string{"Name", "ID"}, + Where: "Name LIKE 'chrome%'", }, }, }, @@ -119,50 +120,50 @@ func TestCompileQueries(t *testing.T) { // TestQueryCompile ensures individual query compilation works correctly. func TestQueryCompile(t *testing.T) { tests := []struct { - name string - queryConfig QueryConfig - expectedQuery string - expectedFieldsLength int + name string + queryConfig QueryConfig + expectedQuery string + expectedPropertiesLength int }{ { name: "Simple query with WHERE clause", queryConfig: QueryConfig{ - Class: "Win32_Process", - Fields: []string{"Name", "ProcessId"}, - Where: "Name = 'notepad.exe'", + Class: "Win32_Process", + Properties: []string{"Name", "ProcessId"}, + Where: "Name = 'notepad.exe'", }, - expectedQuery: "SELECT Name,ProcessId FROM Win32_Process WHERE Name = 'notepad.exe'", - expectedFieldsLength: 2, + expectedQuery: "SELECT Name,ProcessId FROM Win32_Process WHERE Name = 'notepad.exe'", + expectedPropertiesLength: 2, }, { - name: "Query with multiple fields and no WHERE clause", + name: "Query with multiple properties and no WHERE clause", queryConfig: QueryConfig{ - Class: "Win32_Service", - Fields: []string{"Name", "State", "StartMode"}, - Where: "", + Class: "Win32_Service", + Properties: []string{"Name", "State", "StartMode"}, + Where: "", }, - expectedQuery: "SELECT Name,State,StartMode FROM Win32_Service", - expectedFieldsLength: 3, + expectedQuery: "SELECT Name,State,StartMode FROM Win32_Service", + expectedPropertiesLength: 3, }, { - name: "Query with wildcard (*) for fields", + name: "Query with empty list for properties and Where", queryConfig: QueryConfig{ - Class: "Win32_ComputerSystem", - Fields: []string{}, - Where: "Manufacturer = 'Dell'", + Class: "Win32_ComputerSystem", + Properties: []string{}, + Where: "Manufacturer = 'Dell'", }, - expectedQuery: "SELECT * FROM Win32_ComputerSystem WHERE Manufacturer = 'Dell'", - expectedFieldsLength: 0, + expectedQuery: "SELECT * FROM Win32_ComputerSystem WHERE Manufacturer = 'Dell'", + expectedPropertiesLength: 0, }, { name: "Query with wildcard (*) and no WHERE clause", queryConfig: QueryConfig{ - Class: "Win32_BIOS", - Fields: []string{"*"}, - Where: "", + Class: "Win32_BIOS", + Properties: []string{"*"}, + Where: "", }, - expectedQuery: "SELECT * FROM Win32_BIOS", - expectedFieldsLength: 0, // The normalization process make sure that ['*'] and [] are the same case + expectedQuery: "SELECT * FROM Win32_BIOS", + expectedPropertiesLength: 0, // The normalization process make sure that ['*'] and [] are the same case }, } @@ -170,7 +171,7 @@ func TestQueryCompile(t *testing.T) { t.Run(tt.name, func(t *testing.T) { tt.queryConfig.compileQuery() assert.Equal(t, tt.expectedQuery, tt.queryConfig.QueryStr, "QueryStr should match the expected query string") - assert.Equal(t, tt.expectedFieldsLength, len(tt.queryConfig.Fields)) + assert.Equal(t, tt.expectedPropertiesLength, len(tt.queryConfig.Properties)) }) } } diff --git a/metricbeat/module/windows/wmi/utils.go b/metricbeat/module/windows/wmi/utils.go index e43f59567b67..f6bb3feaaf1b 100644 --- a/metricbeat/module/windows/wmi/utils.go +++ b/metricbeat/module/windows/wmi/utils.go @@ -48,9 +48,54 @@ func ConvertSint64(v string) (interface{}, error) { return strconv.ParseInt(v, 10, 64) } +const WMI_DATETIME_LAYOUT string = "20060102150405.999999" +const TIMEZONE_LAYOUT string = "-07:00" + +// The CIMDateFormat is defined as "yyyymmddHHMMSS.mmmmmmsUUU". +// Example: "20231224093045.123456+000" +// More information: https://learn.microsoft.com/en-us/windows/win32/wmisdk/cim-datetime +// +// The "yyyyMMddHHmmSS.mmmmmm" part can be parsed using time.Parse, but Go's time package does not support parsing the "sUUU" +// part (the sign and minute offset from UTC). +// +// Here, "s" represents the sign (+ or -), and "UUU" represents the UTC offset in minutes. +// +// The approach for handling this is: +// 1. Extract the sign ('+' or '-') from the string. +// 2. Normalize the offset from minutes to the standard `hh:mm` format. +// 3. Concatenate the "yyyyMMddHHmmSS.mmmmmm" part with the normalized offset. +// 4. Parse the combined string using time.Parse to return a time.Date object. func ConvertDatetime(v string) (interface{}, error) { - layout := "20060102150405.999999-0700" - return time.Parse(layout, v+"0") + if len(v) != 25 { + return nil, fmt.Errorf("datetime is invalid: the datetime is expected to be exactly 25 characters long, got: %s", v) + } + + // Extract the sign (either '+' or '-') + utcOffsetSign := v[21] + if utcOffsetSign != '+' && utcOffsetSign != '-' { + return nil, fmt.Errorf("datetime is invalid: the offset sign is expected to be either + or -") + } + + // Extract UTC offset (last 3 characters) + utcOffsetStr := v[22:] + utcOffset, err := strconv.ParseInt(utcOffsetStr, 10, 16) + if err != nil { + return nil, fmt.Errorf("datetime is invalid: error parsing UTC offset: %w", err) + } + offsetHours := utcOffset / 60 + offsetMinutes := utcOffset % 60 + + // Build the complete date string including the UTC offset in the format yyyyMMddHHmmss.mmmmmm+hh:mm + // Concatenate the date string with the offset formatted as "+hh:mm" + dateString := fmt.Sprintf("%s%c%02d:%02d", v[:21], utcOffsetSign, offsetHours, offsetMinutes) + + // Parse the combined datetime string using the defined layout + date, err := time.Parse(WMI_DATETIME_LAYOUT+TIMEZONE_LAYOUT, dateString) + if err != nil { + return nil, fmt.Errorf("datetime is invalid: error parsing the final datetime: %w", err) + } + + return date, err } func ConvertString(v string) (interface{}, error) { @@ -59,8 +104,8 @@ func ConvertString(v string) (interface{}, error) { // Function that determines if a given value requires additional conversion // This holds true for strings that encode uint64, sint64 and datetime format -func RequiresExtraConversion(fieldValue interface{}) bool { - stringValue, isString := fieldValue.(string) +func RequiresExtraConversion(propertyValue interface{}) bool { + stringValue, isString := propertyValue.(string) if !isString { return false } @@ -85,11 +130,11 @@ func getPropertyType(property *ole.IDispatch) (base.WmiType, error) { return base.WmiType(value.(int32)), nil } -// Returns the "raw" SWbemProperty containing type information for a given field. +// Returns the "raw" SWbemProperty containing type information for a given property. // // The microsoft/wmi library does not have a function that given an instance and a property name // returns the wmi.wmiProperty object. This function mimics the behavior of the `GetSystemProperty` -// method in the wmi.wmiInstance struct and applies it on the Properties_ field +// method in the wmi.wmiInstance struct and applies it on the Properties_ property // https://github.com/microsoft/wmi/blob/v0.25.2/pkg/wmiinstance/WmiInstance.go#L87 // // Note: We are not instantiating a wmi.wmiProperty because of this issue diff --git a/metricbeat/module/windows/wmi/utils_test.go b/metricbeat/module/windows/wmi/utils_test.go index 54f66e4d90a3..281708475c93 100644 --- a/metricbeat/module/windows/wmi/utils_test.go +++ b/metricbeat/module/windows/wmi/utils_test.go @@ -58,69 +58,71 @@ func TestExecuteGuardedQueryInstances(t *testing.T) { func Test_RequiresExtraConversion(t *testing.T) { tests := []struct { - name string - fieldValue interface{} - expected bool - description string + name string + propertyValue interface{} + expected bool + description string }{ { - name: "Valid numeric string - ends with a digit", - fieldValue: "12345", - expected: true, - description: "Should require conversion as the string ends with a digit", + name: "Valid numeric string - ends with a digit", + propertyValue: "12345", + expected: true, + description: "Should require conversion as the string ends with a digit", }, { - name: "Empty string", - fieldValue: "", - expected: false, - description: "Should not require conversion as the string is empty", + name: "Empty string", + propertyValue: "", + expected: false, + description: "Should not require conversion as the string is empty", }, { - name: "Non-numeric string - no digits", - fieldValue: "abcdef", - expected: false, - description: "Should not require conversion as the string does not end with a digit", + name: "Non-numeric string - no digits", + propertyValue: "abcdef", + expected: false, + description: "Should not require conversion as the string does not end with a digit", }, { - name: "Mixed string - ends with a digit. Let us fetch the type", - fieldValue: "abc123", - expected: true, - description: "Should require conversion as the string ends with a digit", + name: "Mixed string - ends with a digit. Let us fetch the type", + propertyValue: "abc123", + expected: true, + description: "Should require conversion as the string ends with a digit", }, { - name: "String ending with a non-digit", - fieldValue: "123abc", - expected: false, - description: "Should not require conversion as the string ends with a non-digit", + name: "String ending with a non-digit", + propertyValue: "123abc", + expected: false, + description: "Should not require conversion as the string ends with a non-digit", }, { - name: "Nil input", - fieldValue: nil, - expected: false, - description: "Should not require conversion as the input is nil", + name: "Nil input", + propertyValue: nil, + expected: false, + description: "Should not require conversion as the input is nil", }, { - name: "Non-string input", - fieldValue: 12345, - expected: false, - description: "Should not require conversion as the input is not a string", + name: "Non-string input", + propertyValue: 12345, + expected: false, + description: "Should not require conversion as the input is not a string", }, { - name: "Datetime input - requires a conversion", - fieldValue: "20240925192747.000000+000", - expected: true, - description: "Should not require conversion as the input is not a string", + name: "Datetime input - requires a conversion", + propertyValue: "20240925192747.000000+000", + expected: true, + description: "Should not require conversion as the input is not a string", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := RequiresExtraConversion(tt.fieldValue) + result := RequiresExtraConversion(tt.propertyValue) assert.Equal(t, tt.expected, result, tt.description) }) } } +const TEST_DATE_FORMAT string = "2006-01-02T15:04:05.999999-07:00" + func Test_ConversionFunctions(t *testing.T) { tests := []struct { name string @@ -170,8 +172,16 @@ func Test_ConversionFunctions(t *testing.T) { { name: "ConvertDatetime - valid input", conversion: ConvertDatetime, - input: "20231224093045.123456-000", - expected: mustParseTime("20060102150405.999999-0700", "20231224093045.123456-0000"), + input: "20231224093045.123456+000", + expected: mustParseTime(TEST_DATE_FORMAT, "2023-12-24T09:30:45.123456+00:00"), + expectErr: false, + description: "Should convert string to time.Time", + }, + { + name: "ConvertDatetime - valid input - timezone set", + conversion: ConvertDatetime, + input: "20231224093045.123456-690", + expected: mustParseTime(TEST_DATE_FORMAT, "2023-12-24T09:30:45.123456-11:30"), expectErr: false, description: "Should convert string to time.Time", }, diff --git a/metricbeat/module/windows/wmi/wmi.go b/metricbeat/module/windows/wmi/wmi.go index 26f9b8a30404..6bf56568d167 100644 --- a/metricbeat/module/windows/wmi/wmi.go +++ b/metricbeat/module/windows/wmi/wmi.go @@ -88,14 +88,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } // This function handles the skip conditions -func (m *MetricSet) shouldSkipNilOrEmptyValue(fieldValue interface{}) bool { - if fieldValue == nil { - if !m.config.IncludeNull { - return true // Skip if it's nil and IncludeNull is false +func (m *MetricSet) shouldSkipNilOrEmptyValue(propertyValue interface{}) bool { + if propertyValue == nil { + if !m.config.IncludeNullProperties { + return true // Skip if it's nil and IncludeNullProperties is false } - } else if stringValue, ok := fieldValue.(string); ok { - if len(stringValue) == 0 && !m.config.IncludeEmptyString { - return true // Skip if it's an empty string and IncludeEmptyString is false + } else if stringValue, ok := propertyValue.(string); ok { + if len(stringValue) == 0 && !m.config.IncludeEmptyStringProperties { + return true // Skip if it's an empty string and IncludeEmptyStringProperties is false } } return false @@ -113,7 +113,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // for each unique WMI namespace. This minimizes the number of session creations for namespace, queries := range m.config.NamespaceQueryIndex { - session, err := sm.GetSession(namespace, m.config.Host, "", m.config.User, m.config.Password) + session, err := sm.GetSession(namespace, m.config.Host, m.config.Domain, m.config.User, m.config.Password) if err != nil { return fmt.Errorf("could not initialize session %w", err) @@ -140,7 +140,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { } if len(rows) == 0 { - m.Logger().Warnf("The query '%s' did not produce results. This can be expected, but it can also be the result of querying an invalid field. Make sure all required fields do exist or check the WMI-Activity Operational Log for more information.", query) + m.Logger().Warnf("The query '%s' did not return any results. While this can be expected in case of a too strict WHERE clause, it may also indicate an invalid query. Ensure the query is correct or check the WMI-Activity Operational Log for further details.", query) } defer wmi.CloseAllInstances(rows) @@ -150,35 +150,41 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { MetricSetFields: mapstr.M{ "class": queryConfig.Class, "namespace": namespace, - "host": m.config.Host, + // Remote WMI is intentionally hidden, this will always be localhost + // "host": m.config.Host, }, } + // Remote WMI is intentionally hidden, this will always be the empty string + // if m.config.Domain != "" { + // event.MetricSetFields.Put("domain", m.config.Domain) + // } + if m.config.IncludeQueries { event.MetricSetFields.Put("query", query) } // Get only the required properties - properties := queryConfig.Fields + properties := queryConfig.Properties - // If the Fields array is empty we retrieve all fields - if len(queryConfig.Fields) == 0 { + // If the Properties array is empty we retrieve all properties available in the class + if len(queryConfig.Properties) == 0 { properties = instance.GetClass().GetPropertiesNames() } - for _, fieldName := range properties { - fieldValue, err := instance.GetProperty(fieldName) + for _, propertyName := range properties { + propertyValue, err := instance.GetProperty(propertyName) if err != nil { m.Logger().Error("Unable to get propery by name: %v", err) continue } - if m.shouldSkipNilOrEmptyValue(fieldValue) { + if m.shouldSkipNilOrEmptyValue(propertyValue) { continue } // The default case, we simply return what we got - finalValue := fieldValue + finalValue := propertyValue // The script API of WMI returns strings for uint64, sint64, datetime // Link: https://learn.microsoft.com/en-us/windows/win32/wmisdk/querying-wmi @@ -186,31 +192,31 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // // Example: in the query: SELECT * FROM Win32_OperatingSystem // FreePhysicalMemory is a string, but it should be an uint64 - if RequiresExtraConversion(fieldValue) { - convertFun, ok := conversionTable[fieldName] + if RequiresExtraConversion(propertyValue) { + convertFun, ok := conversionTable[propertyName] // If the function is not found let us fetch it and cache it if !ok { - convertFun, err = GetConvertFunction(instance, fieldName, m.Logger()) + convertFun, err = GetConvertFunction(instance, propertyName, m.Logger()) if err != nil { - m.Logger().Warn("Skipping addition of field %s: Unable to retrieve the conversion function: %v", fieldName, err) + m.Logger().Warn("Skipping addition of property %s: Unable to retrieve the conversion function: %v", propertyName, err) continue } - conversionTable[fieldName] = convertFun + conversionTable[propertyName] = convertFun } // Perform the conversion at this point it's safe to cast to string. - fieldValueString, ok := fieldValue.(string) + propertyValueString, ok := propertyValue.(string) if !ok { - m.Logger().Warn("Skipping addition of field %s: expected a string found %v", fieldName, fieldValue) + m.Logger().Warn("Skipping addition of property %s: expected a string found %v", propertyName, propertyValue) continue } - convertedValue, err := convertFun(fieldValueString) + convertedValue, err := convertFun(propertyValueString) if err != nil { - m.Logger().Warn("Skipping addition of field %s. Cannot convert: %v", fieldName, err) + m.Logger().Warn("Skipping addition of property %s. Cannot convert: %v", propertyName, err) continue } finalValue = convertedValue } - event.MetricSetFields.Put(fieldName, finalValue) + event.MetricSetFields.Put(propertyName, finalValue) } report.Event(event) } diff --git a/metricbeat/module/windows/wmi/wmi_test.go b/metricbeat/module/windows/wmi/wmi_test.go index e9cb2e35be2e..5382fd45ed77 100644 --- a/metricbeat/module/windows/wmi/wmi_test.go +++ b/metricbeat/module/windows/wmi/wmi_test.go @@ -28,54 +28,54 @@ import ( func TestShouldSkipNilOrEmptyValue(t *testing.T) { tests := []struct { key string - fieldValue interface{} + propertyValue interface{} includeNull bool includeEmptyString bool expectedShouldSkip bool }{ - // Test Case 1: fieldValue is nil, and IncludeNull is false + // Test Case 1: propertyValue is nil, and IncludeNullProperties is false { - key: "Skip nil value with IncludeNull false", - fieldValue: nil, + key: "Skip nil value with IncludeNullProperties false", + propertyValue: nil, includeNull: false, includeEmptyString: false, - expectedShouldSkip: true, // Should skip because IncludeNull is false + expectedShouldSkip: true, // Should skip because IncludeNullProperties is false }, - // Test Case 2: fieldValue is an empty string, and IncludeEmptyString is false + // Test Case 2: propertyValue is an empty string, and IncludeEmptyStringProperties is false { - key: "Skip Empty string with IncludeEmptyString false", - fieldValue: "", + key: "Skip Empty string with IncludeEmptyStringProperties false", + propertyValue: "", includeNull: false, includeEmptyString: false, - expectedShouldSkip: true, // Should skip because IncludeEmptyString is false + expectedShouldSkip: true, // Should skip because IncludeEmptyStringProperties is false }, - // Test Case 3: fieldValue is nil, and IncludeNull is true + // Test Case 3: propertyValue is nil, and IncludeNullProperties is true { - key: "Don't skip Nil value with IncludeNull true", - fieldValue: nil, + key: "Don't skip Nil value with IncludeNullProperties true", + propertyValue: nil, includeNull: true, includeEmptyString: false, - expectedShouldSkip: false, // Should not skip because IncludeNull is true + expectedShouldSkip: false, // Should not skip because IncludeNullProperties is true }, - // Test Case 4: fieldValue is a non-empty string, and IncludeEmptyString is false + // Test Case 4: propertyValue is a non-empty string, and IncludeEmptyStringProperties is false { - key: "Don't skip Non-empty string with IncludeEmptyString false", - fieldValue: "non-empty", + key: "Don't skip Non-empty string with IncludeEmptyStringProperties false", + propertyValue: "non-empty", includeNull: false, includeEmptyString: false, expectedShouldSkip: false, // Should not skip because the string is non-empty }, - // Test Case 5: fieldValue is a non-empty string, and IncludeEmptyString is true + // Test Case 5: propertyValue is a non-empty string, and IncludeEmptyStringProperties is true { - key: "Non-empty string with IncludeEmptyString true", - fieldValue: "non-empty", + key: "Non-empty string with IncludeEmptyStringProperties true", + propertyValue: "non-empty", includeNull: true, includeEmptyString: true, - expectedShouldSkip: false, // Should not skip because IncludeEmptyString is true + expectedShouldSkip: false, // Should not skip because IncludeEmptyStringProperties is true }, } @@ -84,16 +84,16 @@ func TestShouldSkipNilOrEmptyValue(t *testing.T) { // Arrange: Create a MetricSet with the configuration based on the test case config := Config{ - IncludeNull: test.includeNull, - IncludeEmptyString: test.includeEmptyString, + IncludeNullProperties: test.includeNull, + IncludeEmptyStringProperties: test.includeEmptyString, } metricSet := &MetricSet{ config: config, } - // Act: Call shouldSkipNilOrEmptyValue with the test case fieldValue - result := metricSet.shouldSkipNilOrEmptyValue(test.fieldValue) + // Act: Call shouldSkipNilOrEmptyValue with the test case propertyValue + result := metricSet.shouldSkipNilOrEmptyValue(test.propertyValue) // Assert: Check if the result matches the expected result assert.Equal(t, test.expectedShouldSkip, result) diff --git a/metricbeat/modules.d/windows.yml.disabled b/metricbeat/modules.d/windows.yml.disabled index c917800c1915..172ade104749 100644 --- a/metricbeat/modules.d/windows.yml.disabled +++ b/metricbeat/modules.d/windows.yml.disabled @@ -22,13 +22,13 @@ # - module: windows # metricsets: # - wmi -# period: 60s +# period: 10m # wmi: -# warning_threshold: 30s +# warning_threshold: 10m # # namespace: "root\\cimv2" # queries: # - class: Win32_OperatingSystem # FROM: Class to fetch -# fields: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties +# properties: # SELECT: Properties to retrieve for this WMI class. Omit the setting to fetch all properties # - FreePhysicalMemory # - FreeSpaceInPagingFiles # - FreeVirtualMemory diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 40aeb4742c17..a051e1135dd8 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1715,24 +1715,28 @@ metricbeat.modules: - module: windows metricsets: ["wmi"] - period: 60s + period: 10m wmi: - include_null: false # Exclude fields with null values from the output - include_queries: false # Do not include the query string in the output - include_empty_string: false # Exclude fields with empty string values from the output - warning_threshold: 30s # Maximum time to wait for a query result before logging a warning (defaults to period) + # Do not include the query string in the output + include_queries: false + # Exclude properties with null values from the output + include_null_properties: false + # Exclude properties with empty string values from the output + include_empty_string_properties: false + # Maximum time to wait for a query result before logging a warning (defaults to period) + warning_threshold: 10m # Default WMI namespace for all queries (if not specified per query) # Uncomment to override the default, which is "root\\cimv2". # namespace: "root\\cimv2" queries: - - class: Win32_OperatingSystem # FROM: Class to fetch - fields: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties + - class: Win32_OperatingSystem # FROM: Class to fetch + properties: # SELECT: Fields to retrieve for this WMI class. Omit the setting to fetch all properties - FreePhysicalMemory - FreeSpaceInPagingFiles - FreeVirtualMemory - LocalDateTime - NumberOfUsers - where: "" # Optional WHERE clause to filter query results + where: "" # Optional WHERE clause to filter query results # Override the WMI namespace for this specific query (optional). # If set, this takes precedence over the default namespace above. # namespace: "root\\cimv2" # Overrides the metric