clickhouse: Add initial clickhouse configuration

.env-example

@@ -18,3 +18,11 @@ AWS_SECRET_ACCESS_KEY=
 S3_ENDPOINT=
 S3_REGION=
 S3_BUCKET_NAME=
+
+# Clickhouse
+CLICKHOUSE_HOST=clickhouse
+CLICKHOUSE_HTTP_PORT=8123
+CLICKHOUSE_TCP_PORT=9000
+CLICKHOUSE_DB=default
+CLICKHOUSE_USER=default
+CLICKHOUSE_PASSWORD=

.pre-commit-config.yaml

@@ -14,6 +14,11 @@ repos:
   - id: mixed-line-ending
   - id: pretty-format-json
     args: [--autofix]
+- repo: https://github.com/sqlfluff/sqlfluff
+  rev: 3.2.2
+  hooks:
+  - id: sqlfluff-fix
+    args: ["--dialect", "clickhouse", "-e", "CP03,CP05"]
 
 # Hooks for validation
 - repo: https://github.com/pre-commit/pre-commit-hooks
@@ -28,3 +33,8 @@ repos:
   rev: 3.0.3
   hooks:
   - id: editorconfig-checker
+- repo: https://github.com/sqlfluff/sqlfluff
+  rev: 3.2.2
+  hooks:
+  - id: sqlfluff-lint
+    args: ["--dialect", "clickhouse", "-e", "CP03,CP05"]

Makefile

@@ -1,5 +1,5 @@
 # Define the list of all directories containing the compose files
-ALL_DIRS=webserver monitoring authentik rabbitmq parser user-ingest spectator/server
+ALL_DIRS=clickhouse webserver monitoring authentik rabbitmq parser user-ingest spectator/server
 
 # If specific directories are passed as arguments, use them; otherwise, use ALL_DIRS
 DIRS=$(if $(filter $(ALL_DIRS), $(MAKECMDGOALS)), $(filter $(ALL_DIRS), $(MAKECMDGOALS)), $(ALL_DIRS))

authentik/docker-compose.yaml

@@ -45,7 +45,7 @@ services:
     - ./custom-templates:/templates
     env_file: .env
     ports:
-    - "9000:9000"
+    - "9001:9000"
     networks:
     - default
     - monitoring

clickhouse/.dockerignore

@@ -0,0 +1,2 @@
+data/
+log/

clickhouse/.gitignore

@@ -0,0 +1,2 @@
+data/
+log/

clickhouse/README.md

@@ -0,0 +1,20 @@
+# Clickhouse
+
+### Migration
+
+#### Python-Dependencies
+
+- clickhouse-migrations (See: https://github.com/zifter/clickhouse-migrations)
+
+#### Run Migration
+
+1. Add a migration file in the `migrations` directory. (must be ascending order)
+2. Run the following command to apply the migrations.
+    ```bash
+    clickhouse-migrations \
+        --db-host $CLICKHOUSE_URL \
+        --db-user $CLICKHOUSE_USER \
+        --db-password $CLICKHOUSE_PASSWORD \
+        --db-name $CLICKHOUSE_DB \
+        --migrations-dir ./migrations
+    ```

clickhouse/docker-compose.yaml

@@ -0,0 +1,29 @@
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    restart: always
+    ports:
+    - "127.0.0.1:8123:8123"
+    - "127.0.0.1:9000:9000"
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    privileged: true
+    volumes:
+    - ./data:/var/lib/clickhouse
+    - ./log:/var/log/clickhouse-server
+    env_file: ../.env
+    environment:
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    networks:
+    - clickhouse
+
+networks:
+  clickhouse:
+    name: clickhouse