From 820380efa325ac0b52d5ce52fa08a73328ad5fe1 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Wed, 7 Feb 2024 05:17:03 +0000 Subject: [PATCH 1/2] Filter out time series with no valid data point - From a prod test failure, we observed that there may be time series data in a metric that contain no valid data points. It's unclear why they come to be at the moment - This PR attempts to not count time seires when there is no valid data point in the time series --- tests/gamma/csm_observability_test.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/gamma/csm_observability_test.py b/tests/gamma/csm_observability_test.py index bd5a03bc..4aa56bb2 100644 --- a/tests/gamma/csm_observability_test.py +++ b/tests/gamma/csm_observability_test.py @@ -429,7 +429,10 @@ def query_metrics( view=monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL, retry=retry_settings, ) - time_series = list(response) + time_series = [] + for series in response: + if self.is_legit_time_series(series): + time_series.append(series) self.assertLen( time_series, @@ -447,6 +450,22 @@ def query_metrics( results[metric] = metric_time_series return results + def is_legit_time_series( + self, series: monitoring_v3.types.TimeSeries + ) -> bool: + for point in series.points: + # Test flake: we found some time series with no actual data point + # in prod test runs. + # Here we will only include time series with actual data in it. + if ( + point.value.distribution_value.count > 0 + or point.value.double_value > 0 + ): + return True + logger.error("Warning: found time_series with no valid data point") + logger.error(series) + return False + def assertAtLeastOnePointWithinRange( self, points: list[monitoring_v3.types.Point], From d5659570a412448139b5ea6d065dc7b7669c1f50 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Wed, 7 Feb 2024 21:34:03 +0000 Subject: [PATCH 2/2] Review changes - some pythonistic fixes --- tests/gamma/csm_observability_test.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/gamma/csm_observability_test.py b/tests/gamma/csm_observability_test.py index 4aa56bb2..af116ecf 100644 --- a/tests/gamma/csm_observability_test.py +++ b/tests/gamma/csm_observability_test.py @@ -429,10 +429,7 @@ def query_metrics( view=monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL, retry=retry_settings, ) - time_series = [] - for series in response: - if self.is_legit_time_series(series): - time_series.append(series) + time_series = list(filter(self.is_legit_time_series, response)) self.assertLen( time_series, @@ -450,20 +447,19 @@ def query_metrics( results[metric] = metric_time_series return results + @classmethod def is_legit_time_series( - self, series: monitoring_v3.types.TimeSeries + cls, series: monitoring_v3.types.TimeSeries ) -> bool: for point in series.points: # Test flake: we found some time series with no actual data point # in prod test runs. # Here we will only include time series with actual data in it. - if ( - point.value.distribution_value.count > 0 - or point.value.double_value > 0 - ): + if point.value.distribution_value.count or point.value.double_value: return True - logger.error("Warning: found time_series with no valid data point") - logger.error(series) + logger.warning( + "Warning: found time_series with no valid data point\n%s", series + ) return False def assertAtLeastOnePointWithinRange(