[CSM O11y test] Filter out time series with no valid data point (#32)

- From a prod test failure, we observed that there may be time series
data in a metric that contain no valid data points. It's unclear why
they come to be at the moment
- This PR attempts to not count time series when there is no valid data
point in the time series

Can't really do the filtering at the query level because the monitoring
API does not support filtering by data points:
https://cloud.google.com/monitoring/api/v3/filters

tests/gamma/csm_observability_test.py

@@ -429,7 +429,7 @@ def query_metrics(
                 view=monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,
                 retry=retry_settings,
             )
-            time_series = list(response)
+            time_series = list(filter(self.is_legit_time_series, response))
 
             self.assertLen(
                 time_series,
@@ -447,6 +447,21 @@ def query_metrics(
             results[metric] = metric_time_series
         return results
 
+    @classmethod
+    def is_legit_time_series(
+        cls, series: monitoring_v3.types.TimeSeries
+    ) -> bool:
+        for point in series.points:
+            # Test flake: we found some time series with no actual data point
+            # in prod test runs.
+            # Here we will only include time series with actual data in it.
+            if point.value.distribution_value.count or point.value.double_value:
+                return True
+        logger.warning(
+            "Warning: found time_series with no valid data point\n%s", series
+        )
+        return False
+
     def assertAtLeastOnePointWithinRange(
         self,
         points: list[monitoring_v3.types.Point],