diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 83026eeb964e..a9b6439adf24 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -46,6 +46,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix high IO and handling of a corrupted registry log file. {pull}35893[35893] - Filebeat, when running with Elastic-Agent, reports status for Filestream input. {pull}40121[40121] - Implement Elastic Agent status and health reporting for Winlog Filebeat input. {pull}40163[40163] +- Fix filestream's registry GC: registry entries will never be removed if clean_inactive is set to "-1". {pull}40258[40258] *Heartbeat* @@ -55,6 +56,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Setting period for counter cache for Prometheus remote_write at least to 60sec {pull}38553[38553] - Add support of Graphite series 1.1.0+ tagging extension for statsd module. {pull}39619[39619] - Remove fallback to the node limit for the `kubernetes.pod.cpu.usage.limit.pct` and `kubernetes.pod.memory.usage.limit.pct` metrics calculation +- Add support for Kibana status metricset in v8 format {pull}40275[40275] *Osquerybeat* @@ -178,6 +180,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix behavior of pagetypeinfo metrics {pull}39985[39985] - Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787] - Set GCP metrics config period to the default (60s) when the value is below the minimum allowed period. {issue}30434[30434] {pull}40020[40020] +- Add GCP 'instance_id' resource label in ECS cloud fields. {issue}40033[40033] {pull}40062[40062] - Fix missing metrics from CloudWatch when include_linked_accounts set to false. {issue}40071[40071] {pull}40135[40135] - Update beat module with apm-server monitoring metrics fields {pull}40127[40127] diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl index 68ff3d22e07b..84c63cf11e98 100644 --- a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl +++ b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl @@ -432,7 +432,7 @@ filebeat.inputs: # Files for the modification data is older than clean_inactive the state from the registry is removed # By default this is disabled. - #clean_inactive: 0 + #clean_inactive: -1 # Removes the state for files which cannot be found on disk anymore immediately #clean_removed: true diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index ff308012ed1f..9da15b975c23 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -839,7 +839,7 @@ filebeat.inputs: # Files for the modification data is older than clean_inactive the state from the registry is removed # By default this is disabled. - #clean_inactive: 0 + #clean_inactive: -1 # Removes the state for files which cannot be found on disk anymore immediately #clean_removed: true diff --git a/filebeat/input/filestream/config.go b/filebeat/input/filestream/config.go index f6e1ca03f4c6..1cb8fa5da979 100644 --- a/filebeat/input/filestream/config.go +++ b/filebeat/input/filestream/config.go @@ -33,12 +33,14 @@ import ( type config struct { Reader readerConfig `config:",inline"` - ID string `config:"id"` - Paths []string `config:"paths"` - Close closerConfig `config:"close"` - FileWatcher *conf.Namespace `config:"prospector"` - FileIdentity *conf.Namespace `config:"file_identity"` - CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"` + ID string `config:"id"` + Paths []string `config:"paths"` + Close closerConfig `config:"close"` + FileWatcher *conf.Namespace `config:"prospector"` + FileIdentity *conf.Namespace `config:"file_identity"` + + // -1 means that registry will never be cleaned + CleanInactive time.Duration `config:"clean_inactive" validate:"min=-1"` CleanRemoved bool `config:"clean_removed"` HarvesterLimit uint32 `config:"harvester_limit" validate:"min=0"` IgnoreOlder time.Duration `config:"ignore_older"` @@ -98,7 +100,7 @@ func defaultConfig() config { Reader: defaultReaderConfig(), Paths: []string{}, Close: defaultCloserConfig(), - CleanInactive: 0, + CleanInactive: -1, CleanRemoved: true, HarvesterLimit: 0, IgnoreOlder: 0, diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index f1f9b764600a..c2efe2c50cdd 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -75,10 +75,11 @@ func Plugin(log *logp.Logger, store loginp.StateStore) input.Plugin { Info: "filestream input", Doc: "The filestream input collects logs from the local filestream service", Manager: &loginp.InputManager{ - Logger: log, - StateStore: store, - Type: pluginName, - Configure: configure, + Logger: log, + StateStore: store, + Type: pluginName, + Configure: configure, + DefaultCleanTimeout: -1, }, } } diff --git a/filebeat/input/filestream/internal/input-logfile/clean.go b/filebeat/input/filestream/internal/input-logfile/clean.go index 36f429f3f861..706ddee11897 100644 --- a/filebeat/input/filestream/internal/input-logfile/clean.go +++ b/filebeat/input/filestream/internal/input-logfile/clean.go @@ -130,5 +130,7 @@ func checkCleanResource(started, now time.Time, resource *resource) bool { reference = started } - return reference.Add(ttl).Before(now) && resource.stored + // if ttl is negative, we never delete the entry + // else check for time elapsed + return ttl >= 0 && reference.Add(ttl).Before(now) && resource.stored } diff --git a/filebeat/input/filestream/internal/input-logfile/clean_test.go b/filebeat/input/filestream/internal/input-logfile/clean_test.go index e677d8f4c6d2..765317c39652 100644 --- a/filebeat/input/filestream/internal/input-logfile/clean_test.go +++ b/filebeat/input/filestream/internal/input-logfile/clean_test.go @@ -159,4 +159,24 @@ func TestGCStore(t *testing.T) { want := map[string]state{} checkEqualStoreState(t, want, backend.snapshot()) }) + + t.Run("state never removed with ttl=-1", func(t *testing.T) { + + // keep started as a large value + started := time.Now().Add(-1 * time.Hour * 24 * 356) // cleanup process is running for a while already + + initState := map[string]state{ + "test::key": { + TTL: -1, + Updated: started, + }, + } + + backend := createSampleStore(t, initState) + store := testOpenStore(t, "test", backend) + defer store.Release() + + gcStore(logp.NewLogger("test"), started, store) + checkEqualStoreState(t, initState, backend.snapshot()) + }) } diff --git a/filebeat/input/filestream/internal/input-logfile/manager.go b/filebeat/input/filestream/internal/input-logfile/manager.go index 3eb2f951036d..c65ccb5e3089 100644 --- a/filebeat/input/filestream/internal/input-logfile/manager.go +++ b/filebeat/input/filestream/internal/input-logfile/manager.go @@ -94,9 +94,6 @@ type StateStore interface { func (cim *InputManager) init() error { cim.initOnce.Do(func() { - if cim.DefaultCleanTimeout <= 0 { - cim.DefaultCleanTimeout = 30 * time.Minute - } log := cim.Logger.With("input_type", cim.Type) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 08bb77cc8acb..1564bb64718a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -44450,13 +44450,73 @@ alias to: service.version *`kibana.status.status.overall.state`*:: + -- -Kibana overall state. +Kibana overall state (v7 format). type: keyword -- +*`kibana.status.status.overall.level`*:: ++ +-- +Kibana overall level (v8 format). + + +type: keyword + +-- + +*`kibana.status.status.overall.summary`*:: ++ +-- +Kibana overall state in a human-readable format. + + +type: text + +-- + +*`kibana.status.status.core.elasticsearch.level`*:: ++ +-- +Kibana Elasticsearch client's status + + +type: keyword + +-- + +*`kibana.status.status.core.elasticsearch.summary`*:: ++ +-- +Kibana Elasticsearch client's status in a human-readable format. + + +type: text + +-- + +*`kibana.status.status.core.savedObjects.level`*:: ++ +-- +Kibana Saved Objects client's status + + +type: keyword + +-- + +*`kibana.status.status.core.savedObjects.summary`*:: ++ +-- +Kibana Saved Objects client's status in a human-readable format. + + +type: text + +-- + [float] === metrics diff --git a/metricbeat/mb/testing/modules.go b/metricbeat/mb/testing/modules.go index 1dcc9b075b8d..8c6e09df537f 100644 --- a/metricbeat/mb/testing/modules.go +++ b/metricbeat/mb/testing/modules.go @@ -57,13 +57,13 @@ package testing import ( "context" - "sync" "testing" "time" + "github.com/elastic/go-concert/timed" + "github.com/elastic/beats/v7/metricbeat/mb" conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/mapstr" ) type TestModule struct { @@ -132,25 +132,6 @@ func NewMetricSetsWithRegistry(t testing.TB, config interface{}, registry *mb.Re return metricsets } -func NewReportingMetricSet(t testing.TB, config interface{}) mb.ReportingMetricSet { - metricSet := NewMetricSet(t, config) - - reportingMetricSet, ok := metricSet.(mb.ReportingMetricSet) - if !ok { - t.Fatal("MetricSet does not implement ReportingMetricSet") - } - - return reportingMetricSet -} - -// ReportingFetch runs the given reporting metricset and returns all of the -// events and errors that occur during that period. -func ReportingFetch(metricSet mb.ReportingMetricSet) ([]mapstr.M, []error) { - r := &capturingReporter{} - metricSet.Fetch(r) - return r.events, r.errs -} - // NewReportingMetricSetV2 returns a new ReportingMetricSetV2 instance. Then // you can use ReportingFetchV2 to perform a Fetch operation with the MetricSet. func NewReportingMetricSetV2(t testing.TB, config interface{}) mb.ReportingMetricSetV2 { @@ -186,7 +167,7 @@ func NewReportingMetricSetV2Error(t testing.TB, config interface{}) mb.Reporting // NewReportingMetricSetV2Errors returns an array of new ReportingMetricSetV2 instances. func NewReportingMetricSetV2Errors(t testing.TB, config interface{}) []mb.ReportingMetricSetV2Error { metricSets := NewMetricSets(t, config) - var reportingMetricSets []mb.ReportingMetricSetV2Error + reportingMetricSets := make([]mb.ReportingMetricSetV2Error, 0, len(metricSets)) for _, metricSet := range metricSets { rMS, ok := metricSet.(mb.ReportingMetricSetV2Error) if !ok { @@ -259,6 +240,41 @@ func ReportingFetchV2Error(metricSet mb.ReportingMetricSetV2Error) ([]mb.Event, return r.events, r.errs } +// PeriodicReportingFetchV2Error runs the given metricset and returns +// the first batch of events or errors that occur during that period. +// +// `period` is the time between each fetch. +// `timeout` is the maximum time to wait for the first event. +// +// The function tries to fetch the metrics every `period` until it gets +// the first batch of metrics or the `timeout` is reached. +func PeriodicReportingFetchV2Error(metricSet mb.ReportingMetricSetV2Error, period time.Duration, timeout time.Duration) ([]mb.Event, []error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + r := &CapturingReporterV2{} + _ = timed.Periodic(ctx, period, func() error { + // Fetch the metrics and store them in the + // reporter. + if err := metricSet.Fetch(r); err != nil { + r.errs = append(r.errs, err) + return err + } + + if len(r.events) > 0 { + // We have metrics, stop the periodic + // and return the metrics. + cancel() + } + + // No metrics yet, retry again + // in the next period. + return nil + }) + + return r.events, r.errs +} + // ReportingFetchV2WithContext runs the given reporting metricset and returns all of the // events and errors that occur during that period. func ReportingFetchV2WithContext(metricSet mb.ReportingMetricSetV2WithContext) ([]mb.Event, []error) { @@ -270,71 +286,6 @@ func ReportingFetchV2WithContext(metricSet mb.ReportingMetricSetV2WithContext) ( return r.events, r.errs } -// NewPushMetricSet instantiates a new PushMetricSet using the given -// configuration. The ModuleFactory and MetricSetFactory are obtained from the -// global Registry. -func NewPushMetricSet(t testing.TB, config interface{}) mb.PushMetricSet { - metricSet := NewMetricSet(t, config) - - pushMetricSet, ok := metricSet.(mb.PushMetricSet) - if !ok { - t.Fatal("MetricSet does not implement PushMetricSet") - } - - return pushMetricSet -} - -type capturingReporter struct { - events []mapstr.M - errs []error - done chan struct{} -} - -func (r *capturingReporter) Event(event mapstr.M) bool { - r.events = append(r.events, event) - return true -} - -func (r *capturingReporter) ErrorWith(err error, meta mapstr.M) bool { - r.events = append(r.events, meta) - r.errs = append(r.errs, err) - return true -} - -func (r *capturingReporter) Error(err error) bool { - r.errs = append(r.errs, err) - return true -} - -func (r *capturingReporter) Done() <-chan struct{} { - return r.done -} - -// RunPushMetricSet run the given push metricset for the specific amount of time -// and returns all of the events and errors that occur during that period. -func RunPushMetricSet(duration time.Duration, metricSet mb.PushMetricSet) ([]mapstr.M, []error) { - r := &capturingReporter{done: make(chan struct{})} - - // Run the metricset. - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - metricSet.Run(r) - }() - - // Let it run for some period, then stop it by closing the done channel. - time.AfterFunc(duration, func() { - close(r.done) - }) - - // Wait for the PushMetricSet to completely stop. - wg.Wait() - - // Return all events and errors that were collected. - return r.events, r.errs -} - // NewPushMetricSetV2 instantiates a new PushMetricSetV2 using the given // configuration. The ModuleFactory and MetricSetFactory are obtained from the // global Registry. @@ -428,16 +379,16 @@ func (r *CapturingPushReporterV2) capture(waitEvents int) []mb.Event { // BlockingCapture blocks until waitEvents n of events are captured func (r *CapturingPushReporterV2) BlockingCapture(waitEvents int) []mb.Event { - var events []mb.Event - for { - select { - case e := <-r.eventsC: - events = append(events, e) - if waitEvents > 0 && len(events) >= waitEvents { - return events - } + events := make([]mb.Event, 0, waitEvents) + + for e := range r.eventsC { + events = append(events, e) + if waitEvents > 0 && len(events) >= waitEvents { + return events } } + + return events } // RunPushMetricSetV2 run the given push metricset for the specific amount of diff --git a/metricbeat/module/kibana/fields.go b/metricbeat/module/kibana/fields.go index 670132dd90ea..2a1db1d08e31 100644 --- a/metricbeat/module/kibana/fields.go +++ b/metricbeat/module/kibana/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKibana returns asset data. // This is the base64 encoded zlib format compressed contents of module/kibana. func AssetKibana() string { - return "eJzsWk+v27gRv79PMfAll0ToO+SwPhQtsgu0KBIU6QY9FIVAS2OLDUVqOaTzvJ++oP7YskRKtCzvPiziw8ODpPnNjzPDmeGfd/AVT1v4yndMsicAw43ALWz+UT/YPAHkSJnmleFKbuHPTwAAzUsoVW4FPgFQobRJMyX3/LCFPRPknmoUyAi3cHDAhMZweaAt/GdDJDZvYVMYU23++wSw5yhy2tbY70CyEjtGKRlmqH4BYE6VQ9PKVu2TvmBf2PASybCyOr/ppJngjHpPK2aKLWz+cpbYjMAaJolGqpQkTN2nScleIrFb8XogY5CkpJBCJ2FpiRY/QAevKCmxVPqU7DViymW6OxlcpGgOqlNZaZUhUWIrN273XcmF4It0XmP57Nd90VIrkFUJ8V8xFbzk5h6dfsTEP+hMycxqjbKeGhIzN4UWDXkGKTBujcRzJ0No0prtPa6eAw8YQVEiFMuT53JhfA2lh8Dv7wJ+HwZ+vg/52QPd2RCPznJCqSrNUbDTPf4YYk1NiN631nDBf2UukBKWGX7ElUhMAEfS4bl4BJkr2Egqvf8fwMiH3hHT+ItFMpQYZZhYVmpqhAHACD/n1OaURWmh0+KDuejqFT16aOkcqGJH1OywKJqu1bHjYaJSW8vzSB2E+sgzTFqBa6Cb+xwUjAzPCJnOiiQTlgzqxEPmK56+KZ2Pa1QjkrJAceqTcD9PM9j92qawRYQWEUo0mmeU9L49N4Y7NKz3fDjKoIGmOYag+nCj3mraXH1RdUSdW1yNSqasNEEmQslDUHRYPObJTBHqI1fv/+R932HvhWI+0j2EH364AWEYj9oKT5uyPBod3vdQ/B6KEQidlFQ5rpgXHdwfNynuGRdW41A4EDjn8vWCmR0aOELOlWRlTYTUlTPXSiq1K/+YGeW1+7HbRVrBi3WbB0zmoK1856hMuvMQ68xBMzht9wHRlhqXZJjMEL58+fuPXiXu71pKRlidEi5zfFmq5RMrEdS+1faGgEuDWjLhge0UFoqGmXzxqBxWcGRGM0mV0iZlea6R/BEYr9V19ajf0FlrHVdOgVf9ETVdLymXKPWhnOeJZBUVym/NnVIC2VD9lYp/F2gK1GAK7NTtLBc5cAJ2Rm+e+Qn4ctBSZzrDIhOm8MF2GoXKmIifFOfmYOikTsjF6wH1OAX1dqThFeaf11NMLF2vvZcQmcpFURxC2Xi4PIfQEv3yK/lBs8ajRo/a7gmPw8Xrj03t82ofk+zHepem/+Sxw1+hKswHjl96jehZsc74hhFHwo96bQL/N1MmWKtyjWn/zrVsNiijqpvy61+S0XNORqubMvpY/HNTlxajVIKZvdLl3QD3EmnOs4Li9+xchI5Xx1q8Wyt9qHoTfyUsS5jfAXXeLKosy7LwdtE9lsuUNFqJNIQBs+6FYR+SSiZHk+iW4dZc0geGS3fEu46X6Rur0vshL9018xn5/lE/+1LABbpgYp/Ob/G9XwXleRlM9IE/TNl6rqGy5Q61a6kywVEa6KmAkuUIRtUlrukREvikDIIpmIGdVt8INUHGJBDKHEorDK8EAnH3L5OoLF0hGtUdEfY+rpGBEaFbuRhQMsO3dQtiCjzV8BrfWUJAMmwnOBWY92ET/xKsOSaNLnAzlvpnA9etpm5aZkTebph15gCPac1O6c7u96jpTix8aRrppTDjezPRolN3Da5ByK3F88CUm/EfwE9ODzg9zbkCcAnNhSHMlMyHSj30/Af3fYb+nDVL7ecCoe5h3UysF+1uzuGFMCeQygDPBZcHR/zyvtLqyHPXiTYztS7qIRBtpeTykIw4zLV3o9scN3gmwgAAP9qmiT9boEe8YAQ7RNmyAOJu1SWYSyPIyGosUZrxoM6r0Hy0hfObU3ccbiY+FXDr8f+gysoazPvq4MiERdBYaSSUxoXdOUJbNxjVjKoec9zQBlmnQDaeLXfNpC5NO+RArp4P9noKBTLhTKvkFg/MbCEkHOGOn+sJXA+ACaEy5lxjVFfQ3Oz3o/db8d+N/d8cb8cAdqd4ysFrj78d8Y/sJQElcqCKZVgzurb/J5Vj8j/qBvU2whHBojgznAi6X2psNxt7Rg6Ws8EFrbWaIoDPbTtXH/Lf1Bxdrf09l8XWMtSlxe1upzX96zfU2FON4cX88LLcWsyaiS5H/AKe690cW+a/zy1Ec5F9obMGd9aWWOOvzf2585CaAjLdjfVKx/UdvSUEPrIXXtryBgLeq3Fps2Za5oyf+kjwoUZ6Q6NzoBiXNNs4TVFOSWVfcWIiDQ+gAvSGsdl1XgH0wY5SLh5GpOmjomj8YtFinnqn1QpMGvjxrA1u8N58pvevGqH1f+/N7cd3jz/+8S/CX+1BmSPmp9weMiSNo6PJ33TK0UbRsDte7bSjjptEuTwrRH31eH3vt+i1soD3/RVmcbfxsYEbzweIuoUYsY0GK1T1D42WnsLwXhV47s4HKYX2QyM4dV2a8xWvy86YR9zG6nSvBvM7yVF8wdMd9Vu1aYtCRO/2SKZX5P4fAAD//3pp2g4=" + return "eJzsW0uP3LgRvs+vKMzFG8AjZA5GsnMIEngXSBDYCbxr5BAEQrVU3eKaIrUssj29vz6gHj16UI9Wq3eNhftgDCTxq49VZD1Y9AN8otMTfBI7VHgHYIWV9AT3/ywf3N8BpMSJEYUVWj3BX+4AAKqXkOvUSboD4EwbGyda7cXhCfYo2T81JAmZnuDggZmsFerAT/Dfe2Z5/xruM2uL+//dAewFyZSfSuwHUJhTwyhmi5bLFwD2VHg0o11RP2kPbA+2Iie2mBfnN81olAK59bRAmz3B/V/PI+4HYBWTyBAXWjHF/tMox+eF2PXwciJDkCjnMYF+hOM1UsIADbzmKKdcm1O0N0SxUPHuZGmVoDmoRmRhdELMkSv8vP13uZBSrJLZxQrpr/mippYRFhGLXyiWIhf2GplhxCg86USrxBlDqtwaihK/hVZNeQZpZN6GWKR+DJONS7bXmHoOfEQJmiOpMY0e85Xrqz+6D/zmKuA348CP1yE/BqAbHdLRa05qXcQpSTxdY48+1tSGaH3rrJDiF/QLKcLEiiNtRGICeCEdkcpbkOnALqTS+vsGjELoDTFDPztiy5HVFuW6UFMi9AAG+Kng2qescguNlBDMi6xW0OObhs6eKDySwcOq1dQVh8fDRKR2TqQLZTCZo0goqgd0gS7Oc0giW5EwoUmyKJGOLZkoQOYTnT5rkw5jVDUkxpHg1Cbhf4FksPnVSWGNCDUi5GSNSDhqfXtODHdksfW8P8tRBU1zHINqww1yq2l1tYfqI5nU0WZUEu2UHWUitTqMDu0Hj3kyU4TayMWbPwbfN9h7qTFEuoXw7bcXIPTXo3EykKasX40e7+tS/LoUFyA0o5ROaUO/6OF+v05xj0I6Q/3BIwvnHL6eKXF9BS8Y50OydnbBqI4xt3IqpSl/nx7lS7djc4q0gRXLNA9QpWCcevBUJs15WGrMXjI4rfce0ZqaUGxRJQQfP/7ju6AQ/+9WQgZYjRChUnpeK+U95gR6X0t7xSCUJaNQBmAbgZnmvidfPSuPNToza1BxoY2NMU0NcXgFLpfqs3oyr/gstVxXXkBQ/JEMd0vKNUJDKOd9orDgTIe1udNaEvbFd0T8JyObkQGbUSNu54RMQTDgGb16FiYQ8kFrjekVSyhtFoJtJEqdoFy+Kc7JQd9IzSC/Xg9khi6odSINX6D/+XKCieNu7b2GyJQvWsRhzBv3y3MYK9Fffrk4GKwsas0g7Z6wOLxY/baufV7sbZz9UO5a9x/ddvobRIX5hRMevcXq2TDOhKaxjEQYtauC8DdTKtgqcg1p/8axbHZRLopuOix/jUdPBVujL/Low+Efqri0GqWQaPfa5FcDXEuk6meNDr/m5GKsvTqUEjxaaUOVh/gbYTmm9Aqo82FR4TBJxo+LrtFcopU1WsZjGDBrXujnIbFCNdhEl0y35BLfcLk0Ld5trMyfsYivh3zJrjGk5Otn/RhyAS/QGcp9PH/E92YTlMd1MIsb/jCl67mEyuU7Mj6lSqQgZaElAnJMCawuQ1yVI0TwXlsCm6GFndGfmQxDggqYVAq5k1YUkoCF/xMVaccdRKubFmHr4xIZkJl85WJBq4RelymIzehUwht6cExAbHEnBWeUtmGjcAlWtUkXB7gZTf27gmuqqYvKjIW3G2aN2cNDY/AU79x+T4avxKLnKpFeCzO8N7N46NRdgy4I+1o8HdlyM/YD+N7LAS+n6iuAUFBdGKJEq7QvNEAv3LhvMwz7rFlqP2YEZQ7rd2JZtPs9Ry+EBYPSFkQqhTp44i/vC6OPIvWZaLVTy6A+BmKcUkIdogGHufRucJvjAsssUADAd65K4s8aaBHPkGFHpGoWwMJXXRK9GyFkZygnZYeTOleh6eAI51en7jlcTHxqwW3H/63OC2cpbYuDI0pHYKgwxKSsX3bnFVqbwepqVuWcl02t53UywuFuuWonNW7aI4/46vnFXm6hEU84kyr54gHtE4wNXmCOH8sNXE4ApdQJetNY3QQ0v/vD6O1U/Ddj/3fP2zOA3Wk55dFrj78e8Xf4HIGWKXCBCZWMuvp/r1OKfuJmUq8XGGI0KM5MZwHdjyW2340tJY+Gs94Fra2SIoAPdTpXNvkvSo46tX/gsthWinpJcZvbaVX++pkMtUTTeDHfvyy3FbNqo6sBvxHLtW6OrbPfhxqiusi+0li9O2trtPG36v7ceUpVAJnOxlqho3tHbw2Bd/gscpdfQCB4NS6uaqZ1xvi+jQRvS6RXPOgDLTFJdYxTBeWYdfKJJjZSvwE1Qq+/NpvMawS9d6KUypsRqfKoRTR+duQojYPbagMmFfxw144e8F7c0/uhRKjt33pzefvu9u2fcBH+xTbKPLEw5brJEFWGXkz+oi5HvYr62fFm3Y5y3UTa+1kpy6vH21u/Ri+FEXxz/FOdhP0hrNYeJ0lH6sfU7TiV6PDN8c8XcWKX5zg4hq1YWXruO/o1ahIKEDKXo3owhCnuJNUUJxkm2lDUvZZ9GwV2o1LSjkojLaRxhjdR5yTBq9TLeKT0X7uffBp6I+3+4EVALeNi7XYI3kS5k/wuVm44v1xda7yr4IbREBbdQV5wiA4b5PRvKyktgeMn1RD4nzOjlMa6IQs4NTWat6Mod8+Qx7K2ynSlBvN9pEV8IVAbtQu1aY3Cgsrtlkw75P4fAAD//xNN+qo=" } diff --git a/metricbeat/module/kibana/status/_meta/fields.yml b/metricbeat/module/kibana/status/_meta/fields.yml index 8f0993e7cdf0..c078ef6c46a7 100644 --- a/metricbeat/module/kibana/status/_meta/fields.yml +++ b/metricbeat/module/kibana/status/_meta/fields.yml @@ -23,7 +23,31 @@ - name: status.overall.state type: keyword description: > - Kibana overall state. + Kibana overall state (v7 format). + - name: status.overall.level + type: keyword + description: > + Kibana overall level (v8 format). + - name: status.overall.summary + type: text + description: > + Kibana overall state in a human-readable format. + - name: status.core.elasticsearch.level + type: keyword + description: > + Kibana Elasticsearch client's status + - name: status.core.elasticsearch.summary + type: text + description: > + Kibana Elasticsearch client's status in a human-readable format. + - name: status.core.savedObjects.level + type: keyword + description: > + Kibana Saved Objects client's status + - name: status.core.savedObjects.summary + type: text + description: > + Kibana Saved Objects client's status in a human-readable format. - name: metrics type: group description: > diff --git a/metricbeat/module/kibana/status/_meta/testdata/8.16.0.json b/metricbeat/module/kibana/status/_meta/testdata/8.16.0.json new file mode 100644 index 000000000000..8c18a52d3ed2 --- /dev/null +++ b/metricbeat/module/kibana/status/_meta/testdata/8.16.0.json @@ -0,0 +1,932 @@ +{ + "name": "kibana", + "uuid": "5b2de169-2785-441b-ae8c-186a1936b17d", + "version": { + "number": "8.16.0", + "build_hash": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", + "build_number": 9007199254740991, + "build_snapshot": false, + "build_flavor": "traditional", + "build_date": "2024-07-16T17:38:42.062Z" + }, + "status": { + "overall": { + "level": "available", + "summary": "All services and plugins are available" + }, + "core": { + "elasticsearch": { + "level": "available", + "summary": "Elasticsearch is available", + "meta": { + "warningNodes": [], + "incompatibleNodes": [] + } + }, + "savedObjects": { + "level": "available", + "summary": "SavedObjects service has completed migrations and is available", + "meta": { + "migratedIndices": { + "migrated": 0, + "skipped": 0, + "patched": 7 + } + } + } + }, + "plugins": { + "alerting": { + "level": "available", + "summary": "Alerting is (probably) ready", + "reported": true + }, + "triggersActionsUi": { + "level": "available", + "summary": "All services and plugins are available" + }, + "transform": { + "level": "available", + "summary": "All services and plugins are available" + }, + "stackConnectors": { + "level": "available", + "summary": "All services and plugins are available" + }, + "searchPlayground": { + "level": "available", + "summary": "All services and plugins are available" + }, + "stackAlerts": { + "level": "available", + "summary": "All services and plugins are available" + }, + "ruleRegistry": { + "level": "available", + "summary": "All services and plugins are available" + }, + "cases": { + "level": "available", + "summary": "All services and plugins are available" + }, + "timelines": { + "level": "available", + "summary": "All services and plugins are available" + }, + "sessionView": { + "level": "available", + "summary": "All services and plugins are available" + }, + "kubernetesSecurity": { + "level": "available", + "summary": "All services and plugins are available" + }, + "threatIntelligence": { + "level": "available", + "summary": "All services and plugins are available" + }, + "observabilityShared": { + "level": "available", + "summary": "All services and plugins are available" + }, + "metricsDataAccess": { + "level": "available", + "summary": "All services and plugins are available" + }, + "logsShared": { + "level": "available", + "summary": "All services and plugins are available" + }, + "aiops": { + "level": "available", + "summary": "All services and plugins are available" + }, + "links": { + "level": "available", + "summary": "All services and plugins are available" + }, + "discover": { + "level": "available", + "summary": "All services and plugins are available" + }, + "reporting": { + "level": "available", + "summary": "All services and plugins are available" + }, + "canvas": { + "level": "available", + "summary": "All services and plugins are available" + }, + "logsExplorer": { + "level": "available", + "summary": "All services and plugins are available" + }, + "exploratoryView": { + "level": "available", + "summary": "All services and plugins are available" + }, + "observability": { + "level": "available", + "summary": "All services and plugins are available" + }, + "slo": { + "level": "available", + "summary": "All services and plugins are available" + }, + "ml": { + "level": "available", + "summary": "All services and plugins are available" + }, + "searchInferenceEndpoints": { + "level": "available", + "summary": "All services and plugins are available" + }, + "observabilityAIAssistantApp": { + "level": "available", + "summary": "All services and plugins are available" + }, + "elasticAssistant": { + "level": "available", + "summary": "All services and plugins are available" + }, + "osquery": { + "level": "available", + "summary": "All services and plugins are available" + }, + "infra": { + "level": "available", + "summary": "All services and plugins are available" + }, + "upgradeAssistant": { + "level": "available", + "summary": "All services and plugins are available" + }, + "monitoring": { + "level": "available", + "summary": "All services and plugins are available" + }, + "logstash": { + "level": "available", + "summary": "All services and plugins are available" + }, + "uptime": { + "level": "available", + "summary": "All services and plugins are available" + }, + "synthetics": { + "level": "available", + "summary": "All services and plugins are available" + }, + "observabilityOnboarding": { + "level": "available", + "summary": "All services and plugins are available" + }, + "datasetQuality": { + "level": "available", + "summary": "All services and plugins are available" + }, + "dataQuality": { + "level": "available", + "summary": "All services and plugins are available" + }, + "observabilityLogsExplorer": { + "level": "available", + "summary": "All services and plugins are available" + }, + "apm": { + "level": "available", + "summary": "All services and plugins are available" + }, + "ux": { + "level": "available", + "summary": "All services and plugins are available" + }, + "indexManagement": { + "level": "available", + "summary": "All services and plugins are available" + }, + "rollup": { + "level": "available", + "summary": "All services and plugins are available" + }, + "remoteClusters": { + "level": "available", + "summary": "All services and plugins are available" + }, + "crossClusterReplication": { + "level": "available", + "summary": "All services and plugins are available" + }, + "indexLifecycleManagement": { + "level": "available", + "summary": "All services and plugins are available" + }, + "enterpriseSearch": { + "level": "available", + "summary": "All services and plugins are available" + }, + "observabilityAiAssistantManagement": { + "level": "available", + "summary": "All services and plugins are available" + }, + "esql": { + "level": "available", + "summary": "All services and plugins are available" + }, + "cloudSecurityPosture": { + "level": "available", + "summary": "All services and plugins are available" + }, + "cloudDefend": { + "level": "available", + "summary": "All services and plugins are available" + }, + "securitySolution": { + "level": "available", + "summary": "All services and plugins are available" + }, + "securitySolutionEss": { + "level": "available", + "summary": "All services and plugins are available" + }, + "discoverEnhanced": { + "level": "available", + "summary": "All services and plugins are available" + }, + "fleet": { + "level": "available", + "summary": "Fleet is available", + "reported": true + }, + "translations": { + "level": "available", + "summary": "All services are available" + }, + "searchConnectors": { + "level": "available", + "summary": "All services are available" + }, + "runtimeFields": { + "level": "available", + "summary": "All services are available" + }, + "assetsDataAccess": { + "level": "available", + "summary": "All services are available" + }, + "monitoringCollection": { + "level": "available", + "summary": "All services are available" + }, + "licenseApiGuard": { + "level": "available", + "summary": "All services are available" + }, + "fieldsMetadata": { + "level": "available", + "summary": "All services are available" + }, + "urlForwarding": { + "level": "available", + "summary": "All services are available" + }, + "unifiedHistogram": { + "level": "available", + "summary": "All services are available" + }, + "uiActions": { + "level": "available", + "summary": "All services are available" + }, + "share": { + "level": "available", + "summary": "All services are available" + }, + "screenshotMode": { + "level": "available", + "summary": "All services are available" + }, + "savedObjectsFinder": { + "level": "available", + "summary": "All services are available" + }, + "noDataPage": { + "level": "available", + "summary": "All services are available" + }, + "kibanaUtils": { + "level": "available", + "summary": "All services are available" + }, + "kibanaReact": { + "level": "available", + "summary": "All services are available" + }, + "ftrApis": { + "level": "available", + "summary": "All services are available" + }, + "fieldFormats": { + "level": "available", + "summary": "All services are available" + }, + "expressions": { + "level": "available", + "summary": "All services are available" + }, + "esUiShared": { + "level": "available", + "summary": "All services are available" + }, + "discoverShared": { + "level": "available", + "summary": "All services are available" + }, + "customIntegrations": { + "level": "available", + "summary": "All services are available" + }, + "contentManagement": { + "level": "available", + "summary": "All services are available" + }, + "bfetch": { + "level": "available", + "summary": "All services are available" + }, + "globalSearch": { + "level": "available", + "summary": "All services and plugins are available" + }, + "globalSearchProviders": { + "level": "available", + "summary": "All services and plugins are available" + }, + "features": { + "level": "available", + "summary": "All services and plugins are available" + }, + "mapsEms": { + "level": "available", + "summary": "All services and plugins are available" + }, + "customBranding": { + "level": "available", + "summary": "All services and plugins are available" + }, + "devTools": { + "level": "available", + "summary": "All services and plugins are available" + }, + "inspector": { + "level": "available", + "summary": "All services and plugins are available" + }, + "banners": { + "level": "available", + "summary": "All services and plugins are available" + }, + "usageCollection": { + "level": "available", + "summary": "All services and plugins are available" + }, + "cloud": { + "level": "available", + "summary": "All services and plugins are available" + }, + "guidedOnboarding": { + "level": "available", + "summary": "All services and plugins are available" + }, + "telemetryCollectionManager": { + "level": "available", + "summary": "All services and plugins are available" + }, + "telemetryCollectionXpack": { + "level": "available", + "summary": "All services and plugins are available" + }, + "kibanaUsageCollection": { + "level": "available", + "summary": "All services and plugins are available" + }, + "newsfeed": { + "level": "available", + "summary": "All services and plugins are available" + }, + "screenshotting": { + "level": "available", + "summary": "All services and plugins are available" + }, + "dataViews": { + "level": "available", + "summary": "All services and plugins are available" + }, + "cloudExperiments": { + "level": "available", + "summary": "All services and plugins are available" + }, + "home": { + "level": "available", + "summary": "All services and plugins are available" + }, + "searchprofiler": { + "level": "available", + "summary": "All services and plugins are available" + }, + "painlessLab": { + "level": "available", + "summary": "All services and plugins are available" + }, + "grokdebugger": { + "level": "available", + "summary": "All services and plugins are available" + }, + "management": { + "level": "available", + "summary": "All services and plugins are available" + }, + "spaces": { + "level": "available", + "summary": "All services and plugins are available" + }, + "security": { + "level": "available", + "summary": "All services and plugins are available" + }, + "snapshotRestore": { + "level": "available", + "summary": "All services and plugins are available" + }, + "lists": { + "level": "available", + "summary": "All services and plugins are available" + }, + "encryptedSavedObjects": { + "level": "available", + "summary": "All services and plugins are available" + }, + "entityManager": { + "level": "available", + "summary": "All services and plugins are available" + }, + "cloudLinks": { + "level": "available", + "summary": "All services and plugins are available" + }, + "telemetry": { + "level": "available", + "summary": "All services and plugins are available" + }, + "licenseManagement": { + "level": "available", + "summary": "All services and plugins are available" + }, + "files": { + "level": "available", + "summary": "All services and plugins are available" + }, + "filesManagement": { + "level": "available", + "summary": "All services and plugins are available" + }, + "eventLog": { + "level": "available", + "summary": "All services and plugins are available" + }, + "actions": { + "level": "available", + "summary": "All services and plugins are available" + }, + "observabilityAIAssistant": { + "level": "available", + "summary": "All services and plugins are available" + }, + "investigate": { + "level": "available", + "summary": "All services and plugins are available" + }, + "notifications": { + "level": "available", + "summary": "All services and plugins are available" + }, + "cloudDataMigration": { + "level": "available", + "summary": "All services and plugins are available" + }, + "aiAssistantManagementSelection": { + "level": "available", + "summary": "All services and plugins are available" + }, + "advancedSettings": { + "level": "available", + "summary": "All services and plugins are available" + }, + "telemetryManagementSection": { + "level": "available", + "summary": "All services and plugins are available" + }, + "console": { + "level": "available", + "summary": "All services and plugins are available" + }, + "searchNotebooks": { + "level": "available", + "summary": "All services and plugins are available" + }, + "searchHomepage": { + "level": "available", + "summary": "All services and plugins are available" + }, + "data": { + "level": "available", + "summary": "All services and plugins are available" + }, + "logsDataAccess": { + "level": "available", + "summary": "All services and plugins are available" + }, + "apmDataAccess": { + "level": "available", + "summary": "All services and plugins are available" + }, + "fileUpload": { + "level": "available", + "summary": "All services and plugins are available" + }, + "ingestPipelines": { + "level": "available", + "summary": "All services and plugins are available" + }, + "ecsDataQualityDashboard": { + "level": "available", + "summary": "All services and plugins are available" + }, + "unifiedDocViewer": { + "level": "available", + "summary": "All services and plugins are available" + }, + "savedObjects": { + "level": "available", + "summary": "All services and plugins are available" + }, + "savedObjectsTaggingOss": { + "level": "available", + "summary": "All services and plugins are available" + }, + "savedObjectsTagging": { + "level": "available", + "summary": "All services and plugins are available" + }, + "globalSearchBar": { + "level": "available", + "summary": "All services and plugins are available" + }, + "savedObjectsManagement": { + "level": "available", + "summary": "All services and plugins are available" + }, + "unifiedSearch": { + "level": "available", + "summary": "All services and plugins are available" + }, + "navigation": { + "level": "available", + "summary": "All services and plugins are available" + }, + "graph": { + "level": "available", + "summary": "All services and plugins are available" + }, + "presentationPanel": { + "level": "available", + "summary": "All services and plugins are available" + }, + "embeddable": { + "level": "available", + "summary": "All services and plugins are available" + }, + "uiActionsEnhanced": { + "level": "available", + "summary": "All services and plugins are available" + }, + "embeddableEnhanced": { + "level": "available", + "summary": "All services and plugins are available" + }, + "imageEmbeddable": { + "level": "available", + "summary": "All services and plugins are available" + }, + "urlDrilldown": { + "level": "available", + "summary": "All services and plugins are available" + }, + "savedSearch": { + "level": "available", + "summary": "All services and plugins are available" + }, + "presentationUtil": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionShape": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionRevealImage": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionRepeatImage": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionMetric": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionImage": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionError": { + "level": "available", + "summary": "All services and plugins are available" + }, + "controls": { + "level": "available", + "summary": "All services and plugins are available" + }, + "esqlDataGrid": { + "level": "available", + "summary": "All services and plugins are available" + }, + "dataViewFieldEditor": { + "level": "available", + "summary": "All services and plugins are available" + }, + "dataViewEditor": { + "level": "available", + "summary": "All services and plugins are available" + }, + "kibanaOverview": { + "level": "available", + "summary": "All services and plugins are available" + }, + "dataViewManagement": { + "level": "available", + "summary": "All services and plugins are available" + }, + "charts": { + "level": "available", + "summary": "All services and plugins are available" + }, + "watcher": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visualizations": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeXy": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeVislib": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeVega": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeTimeseries": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeTimelion": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeTagcloud": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeTable": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeMetric": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeHeatmap": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeMarkdown": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visDefaultEditor": { + "level": "available", + "summary": "All services and plugins are available" + }, + "inputControlVis": { + "level": "available", + "summary": "All services and plugins are available" + }, + "eventAnnotation": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionXY": { + "level": "available", + "summary": "All services and plugins are available" + }, + "dashboard": { + "level": "available", + "summary": "All services and plugins are available" + }, + "dashboardEnhanced": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionTagcloud": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionPartitionVis": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypePie": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionMetricVis": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionLegacyMetricVis": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionHeatmap": { + "level": "available", + "summary": "All services and plugins are available" + }, + "expressionGauge": { + "level": "available", + "summary": "All services and plugins are available" + }, + "lens": { + "level": "available", + "summary": "All services and plugins are available" + }, + "maps": { + "level": "available", + "summary": "All services and plugins are available" + }, + "dataVisualizer": { + "level": "available", + "summary": "All services and plugins are available" + }, + "eventAnnotationListing": { + "level": "available", + "summary": "All services and plugins are available" + }, + "visTypeGauge": { + "level": "available", + "summary": "All services and plugins are available" + }, + "licensing": { + "level": "available", + "summary": "License fetched", + "reported": true + }, + "taskManager": { + "level": "available", + "summary": "Task Manager is healthy", + "reported": true + } + } + }, + "metrics": { + "last_updated": "2024-07-17T09:35:11.129Z", + "collection_interval_in_millis": 5000, + "os": { + "platform": "darwin", + "platformRelease": "darwin-23.5.0", + "load": { + "1m": 12.6708984375, + "5m": 10.1025390625, + "15m": 10.5244140625 + }, + "memory": { + "total_in_bytes": 34359738368, + "free_in_bytes": 82526208, + "used_in_bytes": 34277212160 + }, + "uptime_in_millis": 2116272000 + }, + "process": { + "memory": { + "heap": { + "total_in_bytes": 739229696, + "used_in_bytes": 613658752, + "size_limit": 4345298944 + }, + "resident_set_size_in_bytes": 763396096, + "array_buffers_in_bytes": 15459091, + "external_in_bytes": 18984913 + }, + "pid": 94536, + "event_loop_delay": 29.786111, + "event_loop_delay_histogram": { + "min": 9.09312, + "max": 29.786111, + "mean": 10.76752587473, + "exceeds": 0, + "stddev": 1.65098471681065, + "fromTimestamp": "2024-07-17T09:35:06.129Z", + "lastUpdatedAt": "2024-07-17T09:35:11.128Z", + "percentiles": { + "50": 10.387455, + "75": 10.870783, + "95": 13.459455, + "99": 17.580031 + } + }, + "event_loop_utilization": { + "active": 1574.23844694346, + "idle": 3425.01509500295, + "utilization": 0.31489470052574 + }, + "uptime_in_millis": 57391843.684875 + }, + "processes": [ + { + "memory": { + "heap": { + "total_in_bytes": 739229696, + "used_in_bytes": 613658752, + "size_limit": 4345298944 + }, + "resident_set_size_in_bytes": 763396096, + "array_buffers_in_bytes": 15459091, + "external_in_bytes": 18984913 + }, + "pid": 94536, + "event_loop_delay": 29.786111, + "event_loop_delay_histogram": { + "min": 9.09312, + "max": 29.786111, + "mean": 10.76752587473, + "exceeds": 0, + "stddev": 1.65098471681065, + "fromTimestamp": "2024-07-17T09:35:06.129Z", + "lastUpdatedAt": "2024-07-17T09:35:11.128Z", + "percentiles": { + "50": 10.387455, + "75": 10.870783, + "95": 13.459455, + "99": 17.580031 + } + }, + "event_loop_utilization": { + "active": 1574.23844694346, + "idle": 3425.01509500295, + "utilization": 0.31489470052574 + }, + "uptime_in_millis": 57391843.684875 + } + ], + "response_times": { + "avg_in_millis": 15.8129032258065, + "max_in_millis": 863 + }, + "concurrent_connections": 6, + "requests": { + "disconnects": 0, + "total": 312, + "statusCodes": { + "200": 31, + "302": 2, + "304": 277 + }, + "status_codes": { + "200": 31, + "302": 2, + "304": 277 + } + }, + "elasticsearch_client": { + "totalActiveSockets": 2, + "totalIdleSockets": 8, + "totalQueuedRequests": 0 + } + } +} diff --git a/metricbeat/module/kibana/status/_meta/testdata/8.16.0.json-expected.json b/metricbeat/module/kibana/status/_meta/testdata/8.16.0.json-expected.json new file mode 100644 index 000000000000..46821f87b51f --- /dev/null +++ b/metricbeat/module/kibana/status/_meta/testdata/8.16.0.json-expected.json @@ -0,0 +1,48 @@ +[ + { + "event": { + "dataset": "kibana.status", + "duration": 115000, + "module": "kibana" + }, + "kibana": { + "status": { + "metrics": { + "concurrent_connections": 6, + "requests": { + "disconnects": 0, + "total": 312 + } + }, + "name": "kibana", + "status": { + "overall": { + "level": "available", + "summary": "All services and plugins are available" + }, + "core": { + "elasticsearch": { + "level": "available", + "summary": "Elasticsearch is available" + }, + "savedObjects": { + "level": "available", + "summary": "SavedObjects service has completed migrations and is available" + } + } + } + } + }, + "metricset": { + "name": "status", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:55555", + "id": "5b2de169-2785-441b-ae8c-186a1936b17d", + "name": "kibana", + "type": "kibana", + "version": "8.16.0" + } + } +] diff --git a/metricbeat/module/kibana/status/data.go b/metricbeat/module/kibana/status/data.go index 3f975323a9e1..e033a7412103 100644 --- a/metricbeat/module/kibana/status/data.go +++ b/metricbeat/module/kibana/status/data.go @@ -38,7 +38,19 @@ var ( }), "status": c.Dict("status", s.Schema{ "overall": c.Dict("overall", s.Schema{ - "state": c.Str("state"), + "state": c.Str("state"), + "level": c.Str("level"), + "summary": c.Str("summary"), + }), + "core": c.Dict("core", s.Schema{ + "elasticsearch": c.Dict("elasticsearch", s.Schema{ + "level": c.Str("level"), + "summary": c.Str("summary"), + }), + "savedObjects": c.Dict("savedObjects", s.Schema{ + "level": c.Str("level"), + "summary": c.Str("summary"), + }), }), }), "metrics": c.Dict("metrics", s.Schema{ diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 9830e468e647..86576d13becb 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2541,7 +2541,7 @@ filebeat.inputs: # Files for the modification data is older than clean_inactive the state from the registry is removed # By default this is disabled. - #clean_inactive: 0 + #clean_inactive: -1 # Removes the state for files which cannot be found on disk anymore immediately #clean_removed: true diff --git a/x-pack/filebeat/input/websocket/input.go b/x-pack/filebeat/input/websocket/input.go index 69810edd944f..b4c1b8408664 100644 --- a/x-pack/filebeat/input/websocket/input.go +++ b/x-pack/filebeat/input/websocket/input.go @@ -5,15 +5,18 @@ package websocket import ( + "bytes" "context" "errors" "fmt" + "io" "net/url" "reflect" "time" "github.com/google/cel-go/cel" "github.com/gorilla/websocket" + "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/structpb" v2 "github.com/elastic/beats/v7/filebeat/input/v2" @@ -109,7 +112,15 @@ func (i input) run(env v2.Context, src *source, cursor map[string]interface{}, p headers := formHeader(cfg) c, resp, err := websocket.DefaultDialer.DialContext(ctx, url, headers) if resp != nil && resp.Body != nil { - log.Debugw("websocket connection response", "body", resp.Body) + var buf bytes.Buffer + if log.Core().Enabled(zapcore.DebugLevel) { + const limit = 1e4 + io.CopyN(&buf, resp.Body, limit) + } + if n, _ := io.Copy(io.Discard, resp.Body); n != 0 && buf.Len() != 0 { + buf.WriteString("... truncated") + } + log.Debugw("websocket connection response", "body", &buf) resp.Body.Close() } if err != nil { @@ -119,41 +130,26 @@ func (i input) run(env v2.Context, src *source, cursor map[string]interface{}, p } defer c.Close() - done := make(chan error) - - go func() { - defer close(done) - for { - _, message, err := c.ReadMessage() - if err != nil { - metrics.errorsTotal.Inc() - if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { - log.Errorw("websocket connection closed", "error", err) - } else { - log.Errorw("failed to read websocket data", "error", err) - } - done <- err - return - } - metrics.receivedBytesTotal.Add(uint64(len(message))) - state["response"] = message - log.Debugw("received websocket message", logp.Namespace("websocket"), string(message)) - err = i.processAndPublishData(ctx, metrics, prg, ast, state, cursor, pub, log) - if err != nil { - metrics.errorsTotal.Inc() - log.Errorw("failed to process and publish data", "error", err) - done <- err - return + for { + _, message, err := c.ReadMessage() + if err != nil { + metrics.errorsTotal.Inc() + if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { + log.Errorw("websocket connection closed", "error", err) + } else { + log.Errorw("failed to read websocket data", "error", err) } + return err + } + metrics.receivedBytesTotal.Add(uint64(len(message))) + state["response"] = message + log.Debugw("received websocket message", logp.Namespace("websocket"), string(message)) + err = i.processAndPublishData(ctx, metrics, prg, ast, state, cursor, pub, log) + if err != nil { + metrics.errorsTotal.Inc() + log.Errorw("failed to process and publish data", "error", err) + return err } - }() - - // blocks until done is closed, context is cancelled or an error is received - select { - case err := <-done: - return err - case <-ctx.Done(): - return ctx.Err() } } diff --git a/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go index ccab4e3690dd..4970e481b8f9 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs_integration_test.go @@ -8,6 +8,7 @@ package sqs import ( "testing" + "time" "github.com/stretchr/testify/assert" @@ -20,7 +21,7 @@ func TestFetch(t *testing.T) { config := mtest.GetConfigForTest(t, "sqs", "300s") metricSet := mbtest.NewReportingMetricSetV2Error(t, config) - events, errs := mbtest.ReportingFetchV2Error(metricSet) + events, errs := mbtest.PeriodicReportingFetchV2Error(metricSet, 1*time.Minute, 8*time.Minute) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } diff --git a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go index 01dad3c22cbe..028f3d713e6a 100644 --- a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go @@ -100,7 +100,15 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i // common.Mapstr seems to not work as expected when deleting keys so I have to iterate over all results to add // the ones I want for k, v := range s.timeSeries.Resource.Labels { - if k == TimeSeriesResponsePathForECSAvailabilityZone || k == TimeSeriesResponsePathForECSInstanceID || k == TimeSeriesResponsePathForECSAccountID { + + // We are omitting some labels here because they are added separately for services with additional metadata logic. + // However, we explicitly include the instance_id label to ensure it is not missed for services without additional metadata logic. + if k == TimeSeriesResponsePathForECSInstanceID { + _, _ = ecs.Put(ECSCloudInstanceIDKey, v) + continue + } + + if k == TimeSeriesResponsePathForECSAvailabilityZone || k == TimeSeriesResponsePathForECSAccountID { continue }