Skip to content

Commit

Permalink
Alerting: Add math node to the converted Prometheus rules (grafana#10…
Browse files Browse the repository at this point in the history 
…1097)
  • Loading branch information
@alexander-akhmetov
alexander-akhmetov authored Feb 22, 2025
1 parent 5a6d9a9 commit 9dac0c9
Show file tree
Hide file tree
Showing 3 changed files with 308 additions and 40 deletions.
99 changes: 59 additions & 40 deletions pkg/services/ngalert/prom/convert.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package prom

import (
"encoding/json"
"fmt"
"time"

Expand All @@ -20,6 +19,13 @@ const (
ruleUIDLabel = "__grafana_alert_rule_uid__"
)

const (
queryRefID = "query"
prometheusMathRefID = "prometheus_math"
thresholdRefID = "threshold"
)

// Config defines the configuration options for the Prometheus to Grafana rules converter.
type Config struct {
DatasourceUID string
DatasourceType string
Expand All @@ -31,6 +37,7 @@ type Config struct {
AlertRules RulesConfig
}

// RulesConfig contains configuration that applies to either recording or alerting rules.
type RulesConfig struct {
IsPaused bool
}
Expand All @@ -43,14 +50,17 @@ var (
FromTimeRange: &defaultTimeRange,
EvaluationOffset: &defaultEvaluationOffset,
ExecErrState: models.ErrorErrState,
NoDataState: models.NoData,
NoDataState: models.OK,
}
)

type Converter struct {
cfg Config
}

// NewConverter creates a new Converter instance with the provided configuration.
// It validates the configuration and returns an error if any required fields are missing
// or if the configuration is invalid.
func NewConverter(cfg Config) (*Converter, error) {
if cfg.DatasourceUID == "" {
return nil, fmt.Errorf("datasource UID is required")
Expand Down Expand Up @@ -166,15 +176,28 @@ func (p *Converter) convertRule(orgID int64, namespaceUID, group string, rule Pr
forInterval = time.Duration(*rule.For)
}

queryNode, err := createAlertQueryNode(p.cfg.DatasourceUID, p.cfg.DatasourceType, rule.Expr, *p.cfg.FromTimeRange, *p.cfg.EvaluationOffset)
var query []models.AlertQuery
var title string
var isPaused bool
var record *models.Record
var err error

isRecordingRule := rule.Record != ""
query, err = p.createQuery(rule.Expr, isRecordingRule)
if err != nil {
return models.AlertRule{}, err
}

var title string
if rule.Record != "" {
if isRecordingRule {
record = &models.Record{
From: queryRefID,
Metric: rule.Record,
}

isPaused = p.cfg.RecordingRules.IsPaused
title = rule.Record
} else {
isPaused = p.cfg.AlertRules.IsPaused
title = rule.Alert
}

Expand All @@ -192,62 +215,58 @@ func (p *Converter) convertRule(orgID int64, namespaceUID, group string, rule Pr
OrgID: orgID,
NamespaceUID: namespaceUID,
Title: title,
Data: []models.AlertQuery{queryNode},
Condition: "A",
Data: query,
Condition: query[len(query)-1].RefID,
NoDataState: p.cfg.NoDataState,
ExecErrState: p.cfg.ExecErrState,
Annotations: rule.Annotations,
Labels: labels,
For: forInterval,
RuleGroup: group,
IsPaused: isPaused,
Record: record,
Metadata: models.AlertRuleMetadata{
PrometheusStyleRule: &models.PrometheusStyleRule{
OriginalRuleDefinition: string(originalRuleDefinition),
},
},
}

if rule.Record != "" {
result.Record = &models.Record{
From: "A",
Metric: rule.Record,
}
result.IsPaused = p.cfg.RecordingRules.IsPaused
} else {
result.IsPaused = p.cfg.AlertRules.IsPaused
}

return result, nil
}

func createAlertQueryNode(datasourceUID, datasourceType, expr string, fromTimeRange, evaluationOffset time.Duration) (models.AlertQuery, error) {
modelData := map[string]interface{}{
"datasource": map[string]interface{}{
"type": datasourceType,
"uid": datasourceUID,
},
"expr": expr,
"instant": true,
"range": false,
"refId": "A",
// createQuery constructs the alert query nodes for a given Prometheus rule expression.
// It returns a slice of AlertQuery that represent the evaluation steps for the rule.
//
// For recording rules it generates a single query node that
// executes the PromQL query in the configured datasource.
//
// For alerting rules, it generates three query nodes:
// 1. Query Node (query): Executes the PromQL query using the configured datasource.
// 2. Math Node (prometheus_math): Applies a math expression "is_number($query) || is_nan($query) || is_inf($query)".
// 3. Threshold Node (threshold): Gets the result from the math node and checks that it's greater than 0.
//
// This is needed to ensure that we keep the Prometheus behaviour, where any returned result
// is considered alerting, and only when the query returns no data is the alert treated as normal.
func (p *Converter) createQuery(expr string, isRecordingRule bool) ([]models.AlertQuery, error) {
queryNode, err := createQueryNode(p.cfg.DatasourceUID, p.cfg.DatasourceType, expr, *p.cfg.FromTimeRange, *p.cfg.EvaluationOffset)
if err != nil {
return nil, err
}

if datasourceType == datasources.DS_LOKI {
modelData["queryType"] = "instant"
if isRecordingRule {
return []models.AlertQuery{queryNode}, nil
}

modelJSON, err := json.Marshal(modelData)
mathNode, err := createMathNode()
if err != nil {
return models.AlertQuery{}, err
return nil, err
}

return models.AlertQuery{
DatasourceUID: datasourceUID,
Model: modelJSON,
RefID: "A",
RelativeTimeRange: models.RelativeTimeRange{
From: models.Duration(fromTimeRange + evaluationOffset),
To: models.Duration(0 + evaluationOffset),
},
}, nil
thresholdNode, err := createThresholdNode()
if err != nil {
return nil, err
}

return []models.AlertQuery{queryNode, mathNode, thresholdNode}, nil
}
122 changes: 122 additions & 0 deletions pkg/services/ngalert/prom/convert_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package prom

import (
"encoding/json"
"fmt"
"testing"
"time"
Expand All @@ -10,6 +11,8 @@ import (
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"

"github.com/grafana/grafana/pkg/expr"
"github.com/grafana/grafana/pkg/expr/mathexp"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/util"
Expand Down Expand Up @@ -124,6 +127,9 @@ func TestPrometheusRulesToGrafana(t *testing.T) {

if promRule.Record != "" {
require.Equal(t, promRule.Record, grafanaRule.Title)
require.NotNil(t, grafanaRule.Record)
require.Equal(t, grafanaRule.Record.From, queryRefID)
require.Equal(t, promRule.Record, grafanaRule.Record.Metric)
} else {
require.Equal(t, promRule.Alert, grafanaRule.Title)
}
Expand Down Expand Up @@ -198,6 +204,122 @@ func TestPrometheusRulesToGrafanaWithDuplicateRuleNames(t *testing.T) {
require.Equal(t, "alert (3)", group.Rules[3].Title)
}

func TestCreateMathNode(t *testing.T) {
node, err := createMathNode()
require.NoError(t, err)

require.Equal(t, expr.DatasourceUID, node.DatasourceUID)
require.Equal(t, string(expr.QueryTypeMath), node.QueryType)
require.Equal(t, "prometheus_math", node.RefID)

var model map[string]interface{}
err = json.Unmarshal(node.Model, &model)
require.NoError(t, err)

require.Equal(t, "prometheus_math", model["refId"])
require.Equal(t, string(expr.QueryTypeMath), model["type"])
require.Equal(t, "is_number($query) || is_nan($query) || is_inf($query)", model["expression"])

ds := model["datasource"].(map[string]interface{})
require.Equal(t, expr.DatasourceUID, ds["name"])
require.Equal(t, expr.DatasourceType, ds["type"])
require.Equal(t, expr.DatasourceUID, ds["uid"])
}

func TestCreateThresholdNode(t *testing.T) {
node, err := createThresholdNode()
require.NoError(t, err)

require.Equal(t, expr.DatasourceUID, node.DatasourceUID)
require.Equal(t, string(expr.QueryTypeThreshold), node.QueryType)
require.Equal(t, "threshold", node.RefID)

var model map[string]interface{}
err = json.Unmarshal(node.Model, &model)
require.NoError(t, err)

require.Equal(t, "threshold", model["refId"])
require.Equal(t, string(expr.QueryTypeThreshold), model["type"])

ds := model["datasource"].(map[string]interface{})
require.Equal(t, expr.DatasourceUID, ds["name"])
require.Equal(t, expr.DatasourceType, ds["type"])
require.Equal(t, expr.DatasourceUID, ds["uid"])

conditions := model["conditions"].([]interface{})
require.Len(t, conditions, 1)

condition := conditions[0].(map[string]interface{})
evaluator := condition["evaluator"].(map[string]interface{})
require.Equal(t, string(expr.ThresholdIsAbove), evaluator["type"])
require.Equal(t, []interface{}{float64(0)}, evaluator["params"])
}

func TestPrometheusRulesToGrafana_NodesInRules(t *testing.T) {
cfg := Config{
DatasourceUID: "datasource-uid",
DatasourceType: datasources.DS_PROMETHEUS,
}
converter, err := NewConverter(cfg)
require.NoError(t, err)

t.Run("alert rule should have math and threshold nodes", func(t *testing.T) {
group := PrometheusRuleGroup{
Name: "test",
Rules: []PrometheusRule{
{
Alert: "alert1",
Expr: "up == 0",
},
},
}

result, err := converter.PrometheusRulesToGrafana(1, "namespace", group)
require.NoError(t, err)
require.Len(t, result.Rules, 1)
require.Len(t, result.Rules[0].Data, 3)

// First node should be query
require.Equal(t, "query", result.Rules[0].Data[0].RefID)

// Second node should be math
require.Equal(t, "prometheus_math", result.Rules[0].Data[1].RefID)
require.Equal(t, string(expr.QueryTypeMath), result.Rules[0].Data[1].QueryType)
// Check that the math expression is valid
var model map[string]interface{}
err = json.Unmarshal(result.Rules[0].Data[1].Model, &model)
require.NoError(t, err)
require.Equal(t, "is_number($query) || is_nan($query) || is_inf($query)", model["expression"])
// The math expression should be parsed successfully
_, err = mathexp.New(model["expression"].(string))
require.NoError(t, err)

// Third node should be threshold
require.Equal(t, "threshold", result.Rules[0].Data[2].RefID)
require.Equal(t, string(expr.QueryTypeThreshold), result.Rules[0].Data[2].QueryType)
})

t.Run("recording rule should only have query node", func(t *testing.T) {
group := PrometheusRuleGroup{
Name: "test",
Rules: []PrometheusRule{
{
Record: "metric",
Expr: "sum(rate(http_requests_total[5m]))",
},
},
}

result, err := converter.PrometheusRulesToGrafana(1, "namespace", group)
require.NoError(t, err)
require.Len(t, result.Rules, 1)
require.Len(t, result.Rules[0].Data, 1)

// Should only have query node
require.Equal(t, "query", result.Rules[0].Data[0].RefID)
})
}

func TestPrometheusRulesToGrafana_UID(t *testing.T) {
orgID := int64(1)
namespace := "some-namespace"
Expand Down
Loading

0 comments on commit 9dac0c9

Please sign in to comment.