[CSM O11y test] Filter out time series with no valid data point

tests/gamma/csm_observability_test.py

@@ -429,7 +429,10 @@ def query_metrics(
                 view=monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,
                 retry=retry_settings,
             )
-            time_series = list(response)
+            time_series = []
+            for series in response:
+                if self.is_legit_time_series(series):
+                    time_series.append(series)
 
             self.assertLen(
                 time_series,
@@ -447,6 +450,22 @@ def query_metrics(
             results[metric] = metric_time_series
         return results
 
+    def is_legit_time_series(
+        self, series: monitoring_v3.types.TimeSeries
+    ) -> bool:
+        for point in series.points:
+            # Test flake: we found some time series with no actual data point
+            # in prod test runs.
+            # Here we will only include time series with actual data in it.
+            if (
+                point.value.distribution_value.count > 0
+                or point.value.double_value > 0
+            ):
+                return True
+        logger.error("Warning: found time_series with no valid data point")
+        logger.error(series)
+        return False
+
     def assertAtLeastOnePointWithinRange(
         self,
         points: list[monitoring_v3.types.Point],