diff --git a/.env-example b/.env-example index 7e00b25..5becafb 100644 --- a/.env-example +++ b/.env-example @@ -18,3 +18,11 @@ AWS_SECRET_ACCESS_KEY= S3_ENDPOINT= S3_REGION= S3_BUCKET_NAME= + +# Clickhouse +CLICKHOUSE_HOST=clickhouse +CLICKHOUSE_HTTP_PORT=8123 +CLICKHOUSE_TCP_PORT=9000 +CLICKHOUSE_DB=default +CLICKHOUSE_USER=default +CLICKHOUSE_PASSWORD= diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 347e0a9..da1b534 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,11 @@ repos: - id: mixed-line-ending - id: pretty-format-json args: [--autofix] +- repo: https://github.com/sqlfluff/sqlfluff + rev: 3.2.2 + hooks: + - id: sqlfluff-fix + args: ["--dialect", "clickhouse", "-e", "CP03,CP05"] # Hooks for validation - repo: https://github.com/pre-commit/pre-commit-hooks @@ -28,3 +33,8 @@ repos: rev: 3.0.3 hooks: - id: editorconfig-checker +- repo: https://github.com/sqlfluff/sqlfluff + rev: 3.2.2 + hooks: + - id: sqlfluff-lint + args: ["--dialect", "clickhouse", "-e", "CP03,CP05"] diff --git a/Makefile b/Makefile index c240f50..178f6ba 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Define the list of all directories containing the compose files -ALL_DIRS=webserver monitoring authentik rabbitmq parser user-ingest spectator/server +ALL_DIRS=clickhouse webserver monitoring authentik rabbitmq parser user-ingest spectator/server # If specific directories are passed as arguments, use them; otherwise, use ALL_DIRS DIRS=$(if $(filter $(ALL_DIRS), $(MAKECMDGOALS)), $(filter $(ALL_DIRS), $(MAKECMDGOALS)), $(ALL_DIRS)) diff --git a/authentik/docker-compose.yaml b/authentik/docker-compose.yaml index d0cd5eb..8058dab 100644 --- a/authentik/docker-compose.yaml +++ b/authentik/docker-compose.yaml @@ -45,7 +45,7 @@ services: - ./custom-templates:/templates env_file: .env ports: - - "9000:9000" + - "9001:9000" networks: - default - monitoring diff --git a/clickhouse/.dockerignore b/clickhouse/.dockerignore new file mode 100644 index 0000000..ad2cb87 --- /dev/null +++ b/clickhouse/.dockerignore @@ -0,0 +1,2 @@ +data/ +log/ diff --git a/clickhouse/.gitignore b/clickhouse/.gitignore new file mode 100644 index 0000000..ad2cb87 --- /dev/null +++ b/clickhouse/.gitignore @@ -0,0 +1,2 @@ +data/ +log/ diff --git a/clickhouse/README.md b/clickhouse/README.md new file mode 100644 index 0000000..1669f90 --- /dev/null +++ b/clickhouse/README.md @@ -0,0 +1,20 @@ +# Clickhouse + +### Migration + +#### Python-Dependencies + +- clickhouse-migrations (See: https://github.com/zifter/clickhouse-migrations) + +#### Run Migration + +1. Add a migration file in the `migrations` directory. (must be ascending order) +2. Run the following command to apply the migrations. + ```bash + clickhouse-migrations \ + --db-host $CLICKHOUSE_URL \ + --db-user $CLICKHOUSE_USER \ + --db-password $CLICKHOUSE_PASSWORD \ + --db-name $CLICKHOUSE_DB \ + --migrations-dir ./migrations + ``` diff --git a/clickhouse/docker-compose.yaml b/clickhouse/docker-compose.yaml new file mode 100644 index 0000000..2168115 --- /dev/null +++ b/clickhouse/docker-compose.yaml @@ -0,0 +1,29 @@ +services: + clickhouse: + image: clickhouse/clickhouse-server + restart: always + ports: + - "127.0.0.1:8123:8123" + - "127.0.0.1:9000:9000" + ulimits: + nofile: + soft: 262144 + hard: 262144 + privileged: true + volumes: + - ./data:/var/lib/clickhouse + - ./log:/var/log/clickhouse-server + env_file: ../.env + environment: + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + healthcheck: + test: ["CMD", "clickhouse-client", "--query", "SELECT 1"] + interval: 10s + timeout: 5s + retries: 3 + networks: + - clickhouse + +networks: + clickhouse: + name: clickhouse