feat(localhost): deploy prometheus/grafana stack

plays/localhost.yml

@@ -41,9 +41,17 @@
       tags: hosts
     - role: n8n
       tags: n8n
-    - role: nginx
-      tags: nginx
     - role: crowdsec
       tags: crowdsec
     - role: metabase
-      tags: metabase
+      tags: metabase
+    - role: prometheus
+      tags: prometheus
+    - role: loki
+      tags: loki
+    - role: promtail
+      tags: promtail
+    - role: grafana
+      tags: grafana
+    - role: nginx
+      tags: nginx

roles/alertmanager/README.md

@@ -12,7 +12,7 @@ alertmanager_hostname: alertmanager01
 alertmanager_description: Alertmanager
 alertmanager_web_external_url: https://monitoring.example.com/alertmanager
 alertmanager_proxy_basic_auth_username: alertmanager
-alertmanager_proxy_basic_auth_password: "{{ vault_alertmanager_proxy_basic_auth_password }}"
+alertmanager_proxy_basic_auth_password: # default: "{{ vault_alertmanager_proxy_basic_auth_password }}"
 alertmanager_nginx_data_dir: /usr/share/nginx/proxies # default: "{{ nginx_data_dir }}/proxies"
 alertmanager_config:
   route:

roles/alertmanager/tasks/main.yml

@@ -1,12 +1,4 @@
 ---
-- name: Make sure a {{ role_name }} password is configured
-  ansible.builtin.assert:
-    that:
-      - alertmanager_proxy_basic_auth_password is defined
-      - alertmanager_proxy_basic_auth_password | length >= 8
-    fail_msg: >
-      alertmanager_proxy_basic_auth_password is not defined or length is < 8
-
 - name: Include {{ role_name }} tasks
   ansible.builtin.include_tasks: "{{ role_name }}.yml"
   when: alertmanager_image is defined
@@ -15,7 +7,7 @@
 
 - name: Include {{ role_name }} nginx config tasks
   ansible.builtin.include_tasks: "{{ role_name }}_nginx_config.yml"
-  when: alertmanager_image is defined
+  when: alertmanager_proxy_basic_auth_password is defined
   tags:
     - alertmanager
     - alertmanager_nginx_config

roles/bigbluebutton_exporter/README.md

@@ -14,7 +14,7 @@ bigbluebutton_exporter_description: Host metric for server1 # default: BigBlueBu
 bigbluebutton_exporter_nginx_data_dir: /usr/share/nginx/proxies # default: "{{ nginx_data_dir }}/proxies"
 bigbluebutton_exporter_requires_package: python2-passlib # default: python3-passlib
 bigbluebutton_exporter_proxy_basic_auth_username: exporter # default: bigbluebutton-exporter
-bigbluebutton_exporter_proxy_basic_auth_password: "{{ vault_bigbluebutton_exporter_proxy_basic_auth_password }}"
+bigbluebutton_exporter_proxy_basic_auth_password: # default: "{{ vault_bigbluebutton_exporter_proxy_basic_auth_password }}"
 ```
 
 Ensure the nginx proxy includes the bigbluebutton-exporter config:

roles/bigbluebutton_exporter/defaults/main.yml

@@ -1,5 +1,6 @@
 ---
 bigbluebutton_exporter_requires_package: python3-passlib
 bigbluebutton_exporter_description: BigBlueButton exporter
-bigbluebutton_exporter_proxy_basic_auth_username: bigbluebutton-exporter
 bigbluebutton_exporter_nginx_data_dir: "{{ nginx_data_dir }}/proxies"
+bigbluebutton_exporter_proxy_basic_auth_username: bigbluebutton-exporter
+bigbluebutton_exporter_proxy_basic_auth_password: "{{ vault_bigbluebutton_exporter_proxy_basic_auth_password }}"

roles/bigbluebutton_exporter/tasks/main.yml

@@ -7,7 +7,7 @@
 
 - name: Include {{ role_name }} nginx config tasks
   ansible.builtin.include_tasks: "{{ role_name }}_nginx_config.yml"
-  when: bigbluebutton_exporter_image is defined
+  when: bigbluebutton_exporter_proxy_basic_auth_password is defined
   tags:
     - bigbluebutton_exporter
     - bigbluebutton_exporter_nginx_config

roles/cadvisor/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 cadvisor_requires_package: python3-passlib
 cadvisor_description: cAdvisor {{ inventory_hostname_short }}
+cadvisor_nginx_data_dir: "{{ nginx_data_dir }}/proxies"
 cadvisor_proxy_basic_auth_username: cadvisor
 cadvisor_proxy_basic_auth_password: "{{ vault_cadvisor_proxy_basic_auth_password }}"
-cadvisor_nginx_data_dir: "{{ nginx_data_dir }}/proxies"

roles/cadvisor/tasks/main.yml

@@ -7,7 +7,7 @@
 
 - name: Include {{ role_name }} nginx config tasks
   ansible.builtin.include_tasks: "{{ role_name }}_nginx_config.yml"
-  when: cadvisor_image is defined
+  when: cadvisor_proxy_basic_auth_password is defined
   tags:
     - cadvisor
     - cadvisor_nginx_config

roles/clean/defaults/main.yml

@@ -9,3 +9,9 @@ odoo_volume_name: "{{ odoo_hostname }}"
 postgres_volume_name: "{{ postgres_hostname }}"
 nginx_data_dir: "/usr/share/{{ nginx_hostname }}"
 metabase_config_map: [name: prod]
+loki_data_dir: /usr/share/{{ loki_hostname }}
+promtail_data_dir: /usr/share/{{ promtail_hostname }}
+grafana_data_dir: /usr/share/{{ grafana_hostname }}
+grafana_volume_name: "{{ grafana_hostname }}"
+prometheus_volume_name: "{{ prometheus_hostname }}"
+prometheus_data_dir: /usr/share/{{ prometheus_hostname }}

roles/clean/tasks/loki.yml

@@ -0,0 +1,11 @@
+---
+- name: Remove loki container {{ loki_hostname }}
+  community.docker.docker_container:
+    name: "{{ loki_hostname }}"
+    state: absent
+
+- name: Remove loki data dir
+  ansible.builtin.file:
+    path: "{{ loki_data_dir }}"
+    state: absent
+  when: '"loki_data" in ansible_run_tags'

roles/clean/tasks/main.yml

@@ -48,3 +48,5 @@
       - crowdsec
       - metabase
       - iam
+      - loki
+      - promtail

roles/clean/tasks/promtail.yml

@@ -0,0 +1,11 @@
+---
+- name: Remove promtail container {{ promtail_hostname }}
+  community.docker.docker_container:
+    name: "{{ promtail_hostname }}"
+    state: absent
+
+- name: Remove promtail data dir
+  ansible.builtin.file:
+    path: "{{ promtail_data_dir }}"
+    state: absent
+  when: '"promtail_data" in ansible_run_tags'

roles/grafana/README.md

@@ -14,7 +14,7 @@ grafana_description: Prometheus dashboard # default: Grafana
 grafana_data_dir: /usr/share/graf # default: "/usr/share/{{ grafana_hostname }}"
 grafana_volume_name: grafana_data01 # default: "{{ grafana_hostname }}"
 grafana_admin_user: admin
-grafana_admin_password: "{{ vault_grafana_admin_password }}"
+grafana_admin_password: # default: "{{ vault_grafana_admin_password }}"
 grafana_prometheus_hostname: prom01
 grafana_server_domain: "monitor.example.com"
 grafana_server_root_url: "https://monitor.example.com"

roles/grafana/defaults/main.yml

@@ -2,6 +2,7 @@
 grafana_description: Grafana
 grafana_data_dir: /usr/share/{{ grafana_hostname }}
 grafana_volume_name: "{{ grafana_hostname }}"
+grafana_admin_password: "{{ vault_grafana_admin_password }}"
 
 grafana_mail_enabled: "false"
 grafana_mail_from_name: Grafana

roles/loki/README.md

@@ -15,7 +15,7 @@ loki_nginx_data_dir: /usr/share/nginx/proxies # default: "{{ nginx_data_dir }}/p
 loki_data_dir: /usr/share/loki # default: "/usr/share/{{ loki_hostname }}"
 loki_requires_package: python2-passlib # default: python3-passlib
 loki_proxy_basic_auth_username: log # default: loki
-loki_proxy_basic_auth_password: "{{ vault_loki_proxy_basic_auth_password }}"
+loki_proxy_basic_auth_password: # default: "{{ vault_loki_proxy_basic_auth_password }}"
 ```
 
 Ensure the nginx proxy includes the loki config:

roles/loki/defaults/main.yml

@@ -1,6 +1,7 @@
 ---
 loki_requires_package: python3-passlib
 loki_description: Loki
-loki_proxy_basic_auth_username: loki
-loki_nginx_data_dir: "{{ nginx_data_dir }}/proxies"
 loki_data_dir: /usr/share/{{ loki_hostname }}
+loki_nginx_data_dir: "{{ nginx_data_dir }}/proxies"
+loki_proxy_basic_auth_username: loki
+loki_proxy_basic_auth_password: "{{ vault_loki_proxy_basic_auth_password }}"

roles/loki/tasks/loki.yml

@@ -22,7 +22,7 @@
     recreate: false
     volumes:
       - "{{ loki_data_dir }}:/etc/loki"
-    command: -config.file=/etc/loki/local-config.yml
+    command: -config.file=/etc/loki/local-config.yml -config.expand-env=true
     networks:
       - name: "{{ docker_network_name }}"
     log_driver: "{{ docker_log_driver }}"

roles/loki/tasks/main.yml

@@ -7,7 +7,7 @@
 
 - name: Include {{ role_name }} nginx config tasks
   ansible.builtin.include_tasks: "{{ role_name }}_nginx_config.yml"
-  when: loki_image is defined
+  when: loki_proxy_basic_auth_password is defined
   tags:
     - loki
     - loki_nginx_config

roles/loki/templates/local-config.yml

@@ -26,6 +26,9 @@ query_range:
         enabled: true
         max_size_mb: 100
 
+limits_config:
+  metric_aggregation_enabled: true
+
 schema_config:
   configs:
     - from: 2020-10-24
@@ -38,9 +41,25 @@ schema_config:
 
 pattern_ingester:
   enabled: true
+  metric_aggregation:
+    # enabled: true
+    loki_address: localhost:3100
 
 ruler:
   alertmanager_url: http://localhost:9093
 
 frontend:
   encoding: protobuf
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+analytics:
+ reporting_enabled: false

roles/mysqld_exporter/defaults/main.yml

@@ -2,6 +2,6 @@
 mysqld_exporter_requires_package: python3-passlib
 mysqld_exporter_description: MySQL Exporter {{ mysqld_exporter_server }}
 mysqld_exporter_nginx_data_dir: "{{ nginx_data_dir }}/proxies"
+mysqld_exporter_data_dir: /usr/share/{{ mysqld_exporter_hostname }}
 mysqld_exporter_proxy_basic_auth_username: mysqld-exporter
 mysqld_exporter_proxy_basic_auth_password: "{{ vault_mysqld_exporter_proxy_basic_auth_password }}"
-mysqld_exporter_data_dir: /usr/share/{{ mysqld_exporter_hostname }}

roles/mysqld_exporter/tasks/main.yml

@@ -7,7 +7,7 @@
 
 - name: Include {{ role_name }} nginx config tasks
   ansible.builtin.include_tasks: "{{ role_name }}_nginx_config.yml"
-  when: mysqld_exporter_image is defined
+  when: mysqld_exporter_proxy_basic_auth_password is defined
   tags:
     - mysqld_exporter
     - mysqld_exporter_nginx_config

roles/node_exporter/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 node_exporter_requires_package: python3-passlib
 node_exporter_description: Node Exporter {{ inventory_hostname_short }}
+node_exporter_nginx_data_dir: "{{ nginx_data_dir }}/proxies"
 node_exporter_proxy_basic_auth_username: node-exporter
 node_exporter_proxy_basic_auth_password: "{{ vault_node_exporter_proxy_basic_auth_password }}"
-node_exporter_nginx_data_dir: "{{ nginx_data_dir }}/proxies"

roles/node_exporter/tasks/main.yml

@@ -7,7 +7,7 @@
 
 - name: Include {{ role_name }} nginx config tasks
   ansible.builtin.include_tasks: "{{ role_name }}_nginx_config.yml"
-  when: node_exporter_image is defined
+  when: node_exporter_proxy_basic_auth_password is defined
   tags:
     - node_exporter
     - node_exporter_nginx_config

roles/postgres_exporter/defaults/main.yml

@@ -1,10 +1,10 @@
 ---
 postgres_exporter_requires_package: python3-passlib
 postgres_exporter_description: PostgreSQL Exporter {{ postgres_exporter_server }}
-postgres_exporter_nginx_data_dir: "{{ nginx_data_dir }}/proxies"
-postgres_exporter_proxy_basic_auth_username: postgres-exporter
-postgres_exporter_proxy_basic_auth_password: "{{ vault_postgres_exporter_proxy_basic_auth_password }}"
 postgres_exporter_data_dir: /usr/share/{{ postgres_exporter_hostname }}
 postgres_exporter_server: "{{ postgres_hostname }}"
 postgres_exporter_username: "{{ postgres_user }}"
 postgres_exporter_password: "{{ vault_postgres_password }}"
+postgres_exporter_nginx_data_dir: "{{ nginx_data_dir }}/proxies"
+postgres_exporter_proxy_basic_auth_username: postgres-exporter
+postgres_exporter_proxy_basic_auth_password: "{{ vault_postgres_exporter_proxy_basic_auth_password }}"

roles/postgres_exporter/tasks/main.yml

@@ -7,7 +7,7 @@
 
 - name: Include {{ role_name }} nginx config tasks
   ansible.builtin.include_tasks: "{{ role_name }}_nginx_config.yml"
-  when: postgres_exporter_image is defined
+  when: postgres_exporter_proxy_basic_auth_password is defined
   tags:
     - postgres_exporter
     - postgres_exporter_nginx_config

roles/promtail/defaults/main.yml

@@ -2,4 +2,4 @@
 promtail_description: Promtail
 promtail_loki_basic_auth_username: loki
 promtail_data_dir: /usr/share/{{ promtail_hostname }}
-promtail_state: started
+promtail_state: started

roles/promtail/templates/config.yml

@@ -8,7 +8,11 @@ positions:
   filename: /tmp/positions.yaml
 
 clients:
+{% if promtail_loki_basic_auth_password is defined -%}
   - url: https://{{ promtail_loki_basic_auth_username }}:{{ promtail_loki_basic_auth_password }}@{{ promtail_push_url }}
+{% else -%}
+  - url: {{ promtail_push_url }}
+{% endif -%}
 
 scrape_configs:
 

task

@@ -154,7 +154,17 @@ function test-localhost() {
     version
     if [ "$1" = "clean" ]; then
         echo "Cleanup localhost"
-        ansible-playbook -i test plays/clean.yml -t odoo,odoo_data,odoo_volume,postgres,postgres_data,postgres_volume,meilisearch,meilisearch_data,meilisearch_volume,pgadmin,pgadmin_data,pgadmin_volume,n8n,n8n_data,n8n_volume,nginx,nginx_data,nginx_volume,metabase,metabase_data,metabase_volume
+        ansible-playbook -i test plays/clean.yml -t "odoo,odoo_data,odoo_volume,\
+            postgres,postgres_data,postgres_volume,\
+            meilisearch,meilisearch_data,meilisearch_volume,\
+            pgadmin,pgadmin_data,pgadmin_volume,\
+            n8n,n8n_data,n8n_volume,\
+            nginx,nginx_data,nginx_volume,\
+            metabase,metabase_data,metabase_volume,\
+            prometheus,prometheus_data,prometheus_volume,\
+            grafana,grafana_data,grafana_volume,\
+            loki,loki_data,loki_volume,\
+            promtail,promtail_data,promatil_volume"
     else
         echo "Setup localhost"
         ansible-playbook -i test plays/localhost.yml --skip-tags check

test/host_vars/localhost.yml

@@ -3,7 +3,7 @@ ansible_become: true
 ufw_enabled: false
 hosts:
   - ip: 127.0.0.1
-    name: n8n.local odoo.local metabase.local
+    name: n8n.local odoo.local metabase.local grafana.local
   - ip: 10.101.16.2
     name: erp.brunner-innovation.swiss erp-dev.brunner-innovation.swiss gaia.mint-system.com
 
@@ -80,6 +80,9 @@ nginx_proxies:
           proxy_set_header Upgrade $http_upgrade;
           proxy_set_header Connection $connection_upgrade;
           include /etc/nginx/conf.d/proxy-params.conf;
+  - src_hostname: grafana.local
+    dest_hostname: grafana01
+    dest_port: 3000
 
 # https://hub.docker.com/r/mintsystem/odoo/
 odoo_revision: "18.0.20250106"
@@ -108,4 +111,26 @@ metabase_image: metabase/metabase:v0.51.12.1
 metabase_hostname: metabase01
 metabase_db_host: postgres01
 metabase_db_user: example
-metabase_db_pass: test
+metabase_db_pass: test
+
+# https://hub.docker.com/r/prom/prometheus
+prometheus_image: prom/prometheus:v2.54.1
+prometheus_hostname: prom01
+
+# https://hub.docker.com/r/grafana/loki/
+loki_image: grafana/loki:3.3.2
+loki_hostname: loki01
+
+# https://hub.docker.com/r/grafana/promtail/
+promtail_image: grafana/promtail:3.3.2
+promtail_hostname: promtail01
+promtail_push_url: http://loki01:3100/loki/api/v1/push
+
+# https://hub.docker.com/r/grafana/grafana/
+grafana_image: grafana/grafana:11.2.0
+grafana_hostname: grafana01
+grafana_admin_user: admin
+grafana_admin_password: admin
+grafana_prometheus_hostname: prom01
+grafana_server_domain: "grafana.local"
+grafana_server_root_url: "http://grafana.local"