#88 Collect Consul, Vault, HAProxy, Prometheus, Grafana metrics (#100)

Author: vityaman

backend/.gitignore

@@ -37,5 +37,8 @@ build/
 *.srl
 *.jks
 
+# Secrets
+*.token.txt
+
 # Java
 *.hprof

backend/haproxy/config/haproxy.cfg

@@ -15,6 +15,12 @@ defaults
 
   default-server check inter 10s fall 2 rise 1
 
+frontend prometheus
+  bind :8405
+  mode http
+  http-request use-service prometheus-exporter if { path /metrics }
+  no log
+
 frontend internal
   bind :8445 ssl crt /usr/local/etc/haproxy/itmo-dating-backend.pem
   bind :8446 ssl crt /usr/local/etc/haproxy/itmo-dating-backend.pem

backend/prometheus/Dockerfile

@@ -2,6 +2,8 @@ FROM prom/prometheus:latest
 
 COPY ./prometheus.yml /etc/prometheus/prometheus.yml
 
+COPY ./vault.token.txt /etc/prometheus/vault.token.txt
+
 CMD [ \
     "--config.file=/etc/prometheus/prometheus.yml", \
     "--storage.tsdb.retention.size=500MB", \

backend/prometheus/prometheus.yml

@@ -65,3 +65,46 @@ scrape_configs:
           - database-primary-exporter.dating.se.ifmo.ru
         type: 'A'
         port: 9187
+
+  - job_name: consul
+    dns_sd_configs:
+      - names:
+          - consul-exporter.dating.se.ifmo.ru
+        type: 'A'
+        port: 9107
+
+  - job_name: vault
+    metrics_path: /v1/sys/metrics
+    params:
+      format: [ 'prometheus' ]
+    authorization:
+      credentials_file: /etc/prometheus/vault.token.txt
+    dns_sd_configs:
+      - names:
+          - vault.dating.se.ifmo.ru
+        type: 'A'
+        port: 8200
+    scheme: https
+    tls_config:
+      insecure_skip_verify: true
+
+  - job_name: haproxy
+    dns_sd_configs:
+      - names:
+          - haproxy.dating.se.ifmo.ru
+        type: 'A'
+        port: 8405
+
+  - job_name: prometheus
+    dns_sd_configs:
+      - names:
+          - prometheus.dating.se.ifmo.ru
+        type: 'A'
+        port: 9090
+
+  - job_name: grafana
+    dns_sd_configs:
+      - names:
+          - grafana.dating.se.ifmo.ru
+        type: 'A'
+        port: 3000

backend/vault/config/vault.hcl

@@ -10,6 +10,10 @@ listener "tcp" {
     tls_min_version     = "tls13"
 }
 
+telemetry {
+  prometheus_retention_time = "12h"
+}
+
 storage "consul" {
   address           = "server.dc1.consul:8501"
   max_parallel      = 32

compose.yml

@@ -141,6 +141,14 @@ services:
       - consul-data:/opt/consul/data
     tty: true
     hostname: server.dc1.consul
+  consul-exporter:
+    image: prom/consul-exporter
+    command: --consul.server=server.dc1.consul:8500
+    tty: true
+    hostname: consul-exporter.dating.se.ifmo.ru
+    depends_on:
+      consul:
+        condition: service_started
   vault:
     image: ghcr.io/secs-dev/itmo-dating-vault:latest
     build: