[COST-2127] - Increase context timeout to 120 seconds (#126)

* Increase prometheus query context timeout to 120 seconds
* make context configurable between 10 and 180 seconds

api/v1beta1/kokumetricsconfig_types.go

@@ -162,6 +162,13 @@ type UploadSpec struct {
 // PrometheusSpec defines the desired state of PrometheusConfig object in the KokuMetricsConfigSpec.
 type PrometheusSpec struct {
 
+	// ContextTimeout is a field of KokuMetricsConfig to represent how long a query to prometheus should run in seconds before timing out.
+	// The default is 120 seconds.
+	// +kubebuilder:validation:Minimum=10
+	// +kubebuilder:validation:Maximum=180
+	// +kubebuilder:default=120
+	ContextTimeout *int64 `json:"context_timeout,omitempty"`
+
 	// FOR DEVELOPMENT ONLY.
 	// SvcAddress is a field of KokuMetricsConfig to represent the thanos-querier address.
 	// The default is `https://thanos-querier.openshift-monitoring.svc:9091`.
@@ -374,6 +381,9 @@ type PrometheusStatus struct {
 	// PrometheusConnected is a field of KokuMetricsConfigStatus to represent if prometheus can be queried.
 	PrometheusConnected bool `json:"prometheus_connected"`
 
+	//ContextTimeout is a field of KokuMetricsConfigState to represent how long a query to prometheus should run in seconds before timing out.
+	ContextTimeout *int64 `json:"context_timeout,omitempty"`
+
 	// ConnectionError is a field of KokuMetricsConfigStatus to represent errors during prometheus test query.
 	ConnectionError string `json:"prometheus_connection_error,omitempty"`
 

collector/prometheus.go

@@ -34,11 +34,12 @@ var (
 )
 
 type PromCollector struct {
-	PromConn   prometheusConnection
-	PromCfg    *PrometheusConfig
-	TimeSeries *promv1.Range
-	Log        logr.Logger
-	InCluster  bool
+	PromConn       prometheusConnection
+	PromCfg        *PrometheusConfig
+	TimeSeries     *promv1.Range
+	ContextTimeout *int64
+	Log            logr.Logger
+	InCluster      bool
 }
 
 type prometheusConnection interface {
@@ -181,8 +182,13 @@ func (c *PromCollector) GetPromConn(kmCfg *kokumetricscfgv1beta1.KokuMetricsConf
 
 func (c *PromCollector) getQueryResults(queries *querys, results *mappedResults) error {
 	log := c.Log.WithValues("kokumetricsconfig", "getQueryResults")
+	timeout := int64(120)
+	if c.ContextTimeout != nil {
+		timeout = *c.ContextTimeout
+	}
+	log.Info(fmt.Sprintf("prometheus query timeout set to: %d seconds", timeout))
 	for _, query := range *queries {
-		ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
 		defer cancel()
 
 		queryResult, warnings, err := c.PromConn.QueryRange(ctx, query.QueryString, *c.TimeSeries)

collector/prometheus_test.go

@@ -21,6 +21,7 @@ import (
 )
 
 var trueDef = true
+var defaultContextTimeout int64 = 90
 
 type mappedMockPromResult map[string]*mockPromResult
 type mockPromResult struct {
@@ -184,8 +185,9 @@ func TestGetQueryResultsSuccess(t *testing.T) {
 
 func TestGetQueryResultsError(t *testing.T) {
 	col := PromCollector{
-		TimeSeries: &promv1.Range{},
-		Log:        testLogger,
+		ContextTimeout: &defaultContextTimeout,
+		TimeSeries:     &promv1.Range{},
+		Log:            testLogger,
 	}
 	getQueryResultsErrorsTests := []struct {
 		name         string

config/crd/bases/koku-metrics-cfg.openshift.io_kokumetricsconfigs.yaml

@@ -106,6 +106,15 @@ spec:
                 description: PrometheusConfig is a field of KokuMetricsConfig to represent
                   the configuration of Prometheus connection.
                 properties:
+                  context_timeout:
+                    default: 120
+                    description: ContextTimeout is a field of KokuMetricsConfig to
+                      represent how long a query to prometheus should run in seconds
+                      before timing out. The default is 120 seconds.
+                    format: int64
+                    maximum: 180
+                    minimum: 10
+                    type: integer
                   service_address:
                     default: https://thanos-querier.openshift-monitoring.svc:9091
                     description: FOR DEVELOPMENT ONLY. SvcAddress is a field of KokuMetricsConfig
@@ -642,6 +651,12 @@ spec:
                     description: ConfigError is a field of KokuMetricsConfigStatus
                       to represent errors during prometheus configuration.
                     type: string
+                  context_timeout:
+                    description: ContextTimeout is a field of KokuMetricsConfigState
+                      to represent how long a query to prometheus should run in seconds
+                      before timing out.
+                    format: int64
+                    type: integer
                   last_query_start_time:
                     description: LastQueryStartTime is a field of KokuMetricsConfigStatus
                       to represent the last time queries were started.

controllers/kokumetricsconfig_controller.go

@@ -148,6 +148,7 @@ func ReflectSpec(r *KokuMetricsConfigReconciler, kmCfg *kokumetricscfgv1beta1.Ko
 
 	StringReflectSpec(r, kmCfg, &kmCfg.Spec.PrometheusConfig.SvcAddress, &kmCfg.Status.Prometheus.SvcAddress, kokumetricscfgv1beta1.DefaultPrometheusSvcAddress)
 	kmCfg.Status.Prometheus.SkipTLSVerification = kmCfg.Spec.PrometheusConfig.SkipTLSVerification
+	kmCfg.Status.Prometheus.ContextTimeout = kmCfg.Spec.PrometheusConfig.ContextTimeout
 }
 
 // GetClientset returns a clientset based on rest.config
@@ -534,6 +535,7 @@ func collectPromStats(r *KokuMetricsConfigReconciler, kmCfg *kokumetricscfgv1bet
 		Step:  time.Minute,
 	}
 	r.promCollector.TimeSeries = &timeRange
+	r.promCollector.ContextTimeout = kmCfg.Spec.PrometheusConfig.ContextTimeout
 
 	if kmCfg.Status.Prometheus.LastQuerySuccessTime.UTC().Format(promCompareFormat) == t.Format(promCompareFormat) {
 		log.Info("reports already generated for range", "start", timeRange.Start, "end", timeRange.End)

controllers/kokumetricsconfig_controller_test.go

@@ -41,6 +41,8 @@ var (
 	badAuthUserSecretName       = "baduser"
 	falseValue            bool  = false
 	trueValue             bool  = true
+	defaultContextTimeout int64 = 120
+	diffContextTimeout    int64 = 10
 	defaultUploadCycle    int64 = 360
 	defaultCheckCycle     int64 = 1440
 	defaultUploadWait     int64 = 0
@@ -70,6 +72,7 @@ var (
 				CheckCycle:     &defaultCheckCycle,
 			},
 			PrometheusConfig: kokumetricscfgv1beta1.PrometheusSpec{
+				ContextTimeout:      &defaultContextTimeout,
 				SkipTLSVerification: &trueValue,
 				SvcAddress:          "https://thanos-querier.openshift-monitoring.svc:9091",
 			},
@@ -100,6 +103,7 @@ var (
 				CheckCycle:     &defaultCheckCycle,
 			},
 			PrometheusConfig: kokumetricscfgv1beta1.PrometheusSpec{
+				ContextTimeout:      &diffContextTimeout,
 				SkipTLSVerification: &trueValue,
 				SvcAddress:          "https://thanos-querier.openshift-monitoring.svc:9091",
 			},
@@ -434,6 +438,8 @@ var _ = Describe("KokuMetricsConfigController - CRD Handling", func() {
 
 			Expect(fetched.Status.APIURL).To(Equal(unauthorizedTS.URL))
 
+			Expect(fetched.Status.Prometheus.ContextTimeout).To(Equal(&diffContextTimeout))
+
 			Expect(fetched.Status.Source.SourceDefined).To(BeNil())
 			Expect(fetched.Status.Source.LastSourceCheckTime.IsZero()).To(BeTrue())
 			Expect(fetched.Status.Source.SourceError).To(Equal(""))
@@ -461,6 +467,8 @@ var _ = Describe("KokuMetricsConfigController - CRD Handling", func() {
 
 			Expect(fetched.Status.APIURL).To(Equal(unauthorizedTS.URL))
 
+			Expect(fetched.Status.Prometheus.ContextTimeout).To(Equal(&diffContextTimeout))
+
 			Expect(fetched.Status.Source.SourceDefined).To(BeNil())
 			Expect(fetched.Status.Source.LastSourceCheckTime.IsZero()).To(BeTrue())
 			Expect(fetched.Status.Source.SourceError).To(Equal(""))
@@ -493,6 +501,7 @@ var _ = Describe("KokuMetricsConfigController - CRD Handling", func() {
 			Expect(fetched.Status.APIURL).To(Equal(validTS.URL))
 			Expect(fetched.Status.ClusterID).To(Equal(clusterID))
 			Expect(fetched.Status.OperatorCommit).To(Equal(GitCommit))
+			Expect(fetched.Status.Prometheus.ContextTimeout).To(Equal(&defaultContextTimeout))
 			Expect(*fetched.Status.Source.SourceDefined).To(BeFalse())
 			Expect(fetched.Status.Source.SourceError).ToNot(Equal(""))
 			Expect(fetched.Status.Upload.UploadWait).NotTo(BeNil())