diff --git a/README.md b/README.md index 78e3581b4..eecdf230a 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ While most of my infrastructure and workloads are self-hosted I do rely upon the | [Cloudflare](https://www.cloudflare.com/) | Domains and tunnel | Free | | [GitHub](https://github.com/) | Hosting this repository and continuous integration/deployments | Free | | [Migadu](https://migadu.com/) | Email hosting | $19/yr | +| [Pushover](https://pushover.net/) | Kubernetes Alerts and application notifications | $5 (one time) | | | | Total: ~$10/mo | --- diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/templates/network_policy.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/templates/network_policy.yaml index f20fbeeff..e024de4f0 100644 --- a/kubernetes/apps/monitoring/kube-prometheus-stack/templates/network_policy.yaml +++ b/kubernetes/apps/monitoring/kube-prometheus-stack/templates/network_policy.yaml @@ -17,5 +17,6 @@ specs: dns: - matchPattern: "*" - toFQDNs: + - matchName: "api.pushover.net" - matchName: "discord.com" - matchName: "" diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/values.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/values.yaml index 6b5d93a1f..6cc24b9d2 100644 --- a/kubernetes/apps/monitoring/kube-prometheus-stack/values.yaml +++ b/kubernetes/apps/monitoring/kube-prometheus-stack/values.yaml @@ -51,7 +51,8 @@ kube-prometheus-stack: route: receiver: email-cluster group_by: - - namespace + - alertname + - job continue: false routes: - receiver: "null" @@ -61,6 +62,9 @@ kube-prometheus-stack: matchers: - alertname = Watchdog repeat_interval: 15m + - receiver: pushover + matchers: + - severity = "critical" - receiver: discord-csp matchers: - alertname = CSPAlert @@ -68,9 +72,11 @@ kube-prometheus-stack: matchers: - instance =~ ":[0-9]+" - instance =~ ":[0-9]+" + - instance =~ ":[0-9]+" - receiver: email-nas matchers: - instance =~ ":[0-9]+" + - instance =~ ":[0-9]+" group_wait: 1m group_interval: 10m repeat_interval: 12h @@ -99,10 +105,18 @@ kube-prometheus-stack: url: "" http_config: follow_redirects: true + - name: "pushover" + pushover_configs: + - html: true + message: '{{ template "homelab.pushover.message" . }}' + priority: 1 + title: '{{ template "homelab.pushover.title" . }}' + token: "" + url_title: View in Alertmanager + user_key: "" templates: - '/etc/alertmanager/config/*.tmpl' templateFiles: - # https://github.com/prometheus/alertmanager/issues/3310 discord_msg.tmpl: |- {{ define "homelab.discord.title" }} Fire! {{ .GroupLabels.SortedPairs.Values | join " " }} @@ -121,6 +135,31 @@ kube-prometheus-stack: Also {{ .Alerts.Resolved | len }} resolved alerts. {{ end }} {{ end }} + pushover_msg.tmpl: |- + {{ define "homelab.pushover.title" }} + [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] + {{ .CommonLabels.alertname }} + {{ end }} + {{ define "homelab.pushover.message" }} + {{- range .Alerts }} + {{- if ne .Annotations.description "" }} + {{ .Annotations.description }} + {{- else if ne .Annotations.summary "" }} + {{ .Annotations.summary }} + {{- else if ne .Annotations.message "" }} + {{ .Annotations.message }} + {{- else }} + Alert description not available + {{- end }} + {{- if gt (len .Labels.SortedPairs) 0 }} + + {{- range .Labels.SortedPairs }} + {{ .Name }}: {{ .Value }} + {{- end }} + + {{- end }} + {{- end }} + {{ end }} alertmanagerSpec: replicas: 2 podMetadata: