Initialize Tinybird for Insights

.gitignore

@@ -24,6 +24,7 @@ yarn-error.log*
 api-test/__pycache__/api_test.cpython-39-pytest-6.2.4.pyc
 
 # python
+.venv/
 **/*-venv
 **/venv
 **/venv*
@@ -50,3 +51,7 @@ api-test/__pycache__/api_test.cpython-39-pytest-6.2.4.pyc
 docker/volume
 
 services/libs/*/dist
+
+**/.tinyb
+**/.tinyenv
+services/libs/tinybird/.diff_tmp

backend/src/database/migrations/V1741006323__AddSlugToCollectionsAndInsightsProjects.sql

@@ -0,0 +1,37 @@
+ALTER TABLE collections ADD COLUMN slug TEXT;
+ALTER TABLE "insightsProjects" ADD COLUMN slug TEXT;
+
+CREATE OR REPLACE FUNCTION generate_slug(table_name TEXT,  input_text TEXT) RETURNS TEXT AS $$
+DECLARE
+    base_slug TEXT;
+    unique_slug TEXT;
+    counter INT := 1;
+    query TEXT;
+    slug_exists BOOLEAN;
+BEGIN
+    base_slug := lower(regexp_replace(input_text, '[^a-zA-Z0-9]+', '-', 'g'));
+    base_slug := regexp_replace(base_slug, '-$', '', 'g');
+    unique_slug := base_slug;
+
+    -- Ensure uniqueness by appending a counter if needed
+    LOOP
+        query := format('SELECT EXISTS (SELECT 1 FROM %I WHERE slug = $1)', table_name);
+        EXECUTE query INTO slug_exists USING unique_slug;
+
+        EXIT WHEN NOT slug_exists;
+        unique_slug := base_slug || '-' || counter;
+        counter := counter + 1;
+    END LOOP;
+
+    RETURN unique_slug;
+END;
+$$ LANGUAGE plpgsql;
+
+UPDATE collections SET slug = generate_slug('collections', name) WHERE name IS NOT NULL;
+UPDATE "insightsProjects" SET slug = generate_slug('insightsProjects', name) WHERE name IS NOT NULL;
+
+ALTER TABLE collections ALTER COLUMN slug SET NOT NULL;
+ALTER TABLE collections ADD CONSTRAINT idx_collections_slug_unique UNIQUE (slug);
+
+ALTER TABLE "insightsProjects" ALTER COLUMN slug SET NOT NULL;
+ALTER TABLE "insightsProjects" ADD CONSTRAINT "idx_insightsProjects_slug_unique" UNIQUE (slug);

scripts/.env.dist

@@ -0,0 +1,8 @@
+# Environment variables that override the defaults in the Docker Compose scaffold.yml file.
+# In order to use, make a copy of this file named .env and change the values as needed.
+
+# Sequin
+CROWD_SEQUIN_SERVER_HOST=192.168.122.31
+CROWD_SEQUIN_SERVER_PORT=7376
+CROWD_SEQUIN_SECRET_KEY_BASE=WyPLiGs0pvD6qJhKJICO4dauYPXfO/Yl782Zjtpew5qRBDp7CZvbWtQmY0eB13If
+CROWD_SEQUIN_VAULT_KEY=2Sig69bIpuSm2kv0VQfDekET2qy8qUZGI8v3/h3ASiY=

scripts/.gitignore

@@ -1,2 +1,5 @@
 db_dumps/*
-!db_dumps/.gitkeep
+!db_dumps/.gitkeep
+.env
+kafka-connect-http.zip
+scaffold/kafka-connect/kafka-connect-http/*

scripts/cli

@@ -536,7 +536,87 @@ function migrate_env() {
         -f /tmp/migrations/V1716382997__init-questdb.sql
 }
 
-function migrate_local() {
+function wait_for_tinybird() {
+    say "Waiting for Tinybird to get ready.."
+    until curl -s -f http://localhost:80/tokens >/dev/null 2>&1; do
+        yell "Tinybird is not ready yet. Retrying in 5 seconds.."
+        sleep 5
+    done
+    say "Tinybird is ready!"
+}
+
+function fetch_tinybird_tokens() {
+    wait_for_tinybird
+    read -r CROWD_TINYBIRD_WORKSPACE_ADMIN_TOKEN CROWD_TINYBIRD_USER_TOKEN < <(curl -s http://localhost:80/tokens | jq -r '[.workspace_admin_token, .user_token] | @tsv')
+    progress "Workspace Admin Token=$CROWD_TINYBIRD_WORKSPACE_ADMIN_TOKEN"
+    progress "User Token=$CROWD_TINYBIRD_USER_TOKEN"
+    export CROWD_TINYBIRD_WORKSPACE_ADMIN_TOKEN
+    export CROWD_TINYBIRD_USER_TOKEN
+}
+
+function run_tinybird_cli_command() {
+    fetch_tinybird_tokens
+    yell "Running Tinybird CLI command: sh -c '$@'"
+    docker run --rm --network "${PROJECT_NAME}-bridge" \
+        -e TB_TOKEN=$CROWD_TINYBIRD_WORKSPACE_ADMIN_TOKEN \
+        -v "$CLI_HOME/../services/libs/tinybird:/mnt/data:rw" \
+        -w /mnt/data \
+        tinybirdco/tinybird-cli-docker:latest \
+        sh -c "$@"
+}
+
+function migrate_tinybird_local() {
+    set +e +o pipefail
+
+    say "Applying Tinybird project!"
+    fetch_tinybird_tokens
+
+    if [ -z "$CROWD_TINYBIRD_WORKSPACE_ADMIN_TOKEN" ] ||
+        [ "$CROWD_TINYBIRD_WORKSPACE_ADMIN_TOKEN" = "null" ] ||
+        [ -z "$CROWD_TINYBIRD_USER_TOKEN" ] ||
+        [ "$CROWD_TINYBIRD_USER_TOKEN" = "null" ]; then
+        echo "Error: Could not fetch Tinybird tokens. Exiting."
+        return 1
+    fi
+
+    say "Pushing Tinybird project!"
+
+    MAX_RETRIES=5
+    RETRY_DELAY=5
+    attempt=1
+
+    while [ $attempt -le $MAX_RETRIES ]; do
+        whisper "Attempt $attempt/$MAX_RETRIES to deploy Tinybird project.."
+
+        OUTPUT=$(docker run --rm --network "${PROJECT_NAME}-bridge" \
+            -e TB_TOKEN=$CROWD_TINYBIRD_WORKSPACE_ADMIN_TOKEN \
+            -v $CLI_HOME/../services/libs/tinybird:/mnt/data:rw \
+            -w /mnt/data \
+            tinybirdco/tinybird-cli-docker \
+            sh -c 'tb auth --host http://tinybird:80 && tb push')
+
+        STATUS=$?
+
+        whisper "$OUTPUT"
+
+        if [[ "$OUTPUT" == *"502 error responses"* ]] || [[ "$OUTPUT" == *"Max retries exceeded"* ]] || [ $STATUS -ne 0 ]; then
+            yell "Tinybird push failed (Attempt: $attempt/$MAX_RETRIES). Retrying in $RETRY_DELAY seconds..."
+            sleep $RETRY_DELAY
+        else
+            say "Tinybird project deployed successfully!"
+            return 0
+        fi
+
+        attempt=$((attempt + 1))
+    done
+
+    set -eo pipefail
+
+    error "Tinybird push failed after $MAX_RETRIES attempts. Exiting."
+    return 1
+}
+
+function migrate_postgres_local() {
     say "Building crowd flyway migration image..."
     docker build $DOCKER_PLATFORM_FLAGS -t crowd_flyway -f $CLI_HOME/../backend/src/database/Dockerfile.flyway $CLI_HOME/../backend/src/database --load
     say "Applying PostgreSQL migrations!"
@@ -547,15 +627,19 @@ function migrate_local() {
         -e PGPASSWORD=example \
         -e PGDATABASE=crowd-web \
         crowd_flyway
+}
 
+function migrate_questdb_local() {
     say "Applying QuestDB migrations!"
     docker run --rm --network "${PROJECT_NAME}-bridge" \
         -v $CLI_HOME/../services/migrations/questdb/:/tmp/migrations \
         postgres psql postgresql://admin:quest@questdb:8812/qdb \
         -f /tmp/migrations/V1716382997__init-questdb.sql
+}
 
+function migrate_productdb_local() {
     say "Building product flyway migration image..."
-    docker build $DOCKER_PLATFORM_FLAGS -t product_flyway -f $CLI_HOME/../backend/src/product/Dockerfile.flyway $CLI_HOME/../backend/src/product --load
+    docker build $DOCKER_PLATFORM_FLAGS -t product_flyway -f $CLI_HOME/../backend/src/product/Dockerfile.flyway $CLI_HOME/../backend/src/product
     say "Applying product database migrations!"
     docker run --rm --network "${PROJECT_NAME}-bridge" \
         -e PGHOST=product \
@@ -566,6 +650,13 @@ function migrate_local() {
         product_flyway
 }
 
+function migrate_local() {
+    migrate_postgres_local
+    migrate_questdb_local
+    migrate_productdb_local
+    migrate_tinybird_local
+}
+
 function up_test_scaffold() {
     scaffold_set_up_network "${PROJECT_NAME}-bridge-test" $DOCKET_TEST_NETWORK_SUBNET $DOCKER_TEST_NETWORK_GATEWAY
     $_DC -p "$PROJECT_NAME-test" -f $CLI_HOME/../backend/docker-compose.test.yaml down
@@ -597,6 +688,14 @@ function install_libs() {
     (cd $CLI_HOME/.. && pnpm i --frozen-lockfile)
 }
 
+function download_kafka_connect_http() {
+    say "Downloading Kafka Connect HTTP plugin..."
+    wget -nv https://github.com/lensesio/stream-reactor/releases/download/8.1.28/kafka-connect-http-8.1.28.zip -O $CLI_HOME/kafka-connect-http.zip
+    unzip -q $CLI_HOME/kafka-connect-http.zip -d $CLI_HOME/scaffold/kafka-connect/kafka-connect-http
+    rm $CLI_HOME/kafka-connect-http.zip
+    say "Kafka Connect HTTP plugin downloaded."
+}
+
 function up_scaffold() {
     scaffold_set_up_network "$PROJECT_NAME-bridge" $DOCKER_NETWORK_SUBNET $DOCKER_NETWORK_GATEWAY
 
@@ -605,6 +704,13 @@ function up_scaffold() {
         profile='--profile nginx'
     fi
 
+    if [[ ! -d "$CLI_HOME/scaffold/kafka-connect/kafka-connect-http" || 
+          -z "$(ls -A "$CLI_HOME/scaffold/kafka-connect/kafka-connect-http")"  || 
+          ! -d "$CLI_HOME/scaffold/kafka-connect/kafka-connect-http/kafka-connect-http-8.1.28" ||
+          -z "$(ls -A "$CLI_HOME/scaffold/kafka-connect/kafka-connect-http/kafka-connect-http-8.1.28")" ]]; then
+        download_kafka_connect_http
+    fi
+
     $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/scaffold.yaml ${profile} up -d --build
 }
 
@@ -897,6 +1003,10 @@ while test $# -gt 0; do
         (cd $CLI_HOME/../services/scripts && ./lint_apps.sh && ./lint_libs.sh)
         exit
         ;;
+    tb)
+        run_tinybird_cli_command "$*"
+        exit
+        ;;
     *)
         error "Invalid command '$1'" && say "$SCRIPT_USAGE"
         exit 1

scripts/scaffold.yaml

@@ -1,17 +1,16 @@
-version: '3.1'
-
 services:
   db:
     image: postgres:14-alpine
     restart: unless-stopped
-    command: -c 'max_connections=300'
+    command: -c 'max_connections=300' -c 'wal_level=logical' -c 'max_replication_slots=11' -c 'max_replication_slots=15'
     environment:
       - POSTGRES_PASSWORD=example
       - POSTGRES_DB=crowd-web
     ports:
       - 5432:5432
     volumes:
       - pgdata-dev:/var/lib/postgresql/data
+      - ./scaffold/sequin/postgres-docker-entrypoint-initdb.d/create-sequin-user-and-database.sql:/docker-entrypoint-initdb.d/create-sequin-user-and-database.sql
     shm_size: 1gb
     networks:
       - crowd-bridge
@@ -154,6 +153,7 @@ services:
       - KAFKA_CFG_LOG_DIRS=/opt/bitnami/kafka/data
       - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=1
       - KAFKA_KRAFT_CLUSTER_ID=OTMwNzFhYTY1ODNiNGE5OT
+      - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
     ports:
       - '9092:9092'
       - '9093:9093'
@@ -165,16 +165,15 @@ services:
     build:
       context: scaffold/kafka-connect
     restart: unless-stopped
-    entrypoint:
-      - connect-standalone
-      - /etc/kafka-connect/worker-local.properties
-      - /etc/kafka-connect/console-local-sink.properties
-      - /etc/kafka-connect/questdb-local-sink.properties
+    entrypoint: ["/bin/sh", "-c", "/wait-for-tinybird.sh"]
     volumes:
       - kafka-connect-dev:/storage
+      - ./scaffold/kafka-connect/wait-for-tinybird.sh:/wait-for-tinybird.sh
       - ./scaffold/kafka-connect/worker-local.properties:/etc/kafka-connect/worker-local.properties
       - ./scaffold/kafka-connect/console-local-sink.properties:/etc/kafka-connect/console-local-sink.properties
       - ./scaffold/kafka-connect/questdb-local-sink.properties:/etc/kafka-connect/questdb-local-sink.properties
+      - ./scaffold/kafka-connect/tinybird-local-sink.properties:/etc/kafka-connect/tinybird-local-sink.properties
+
     networks:
       - crowd-bridge
 
@@ -188,6 +187,44 @@ services:
     networks:
       - crowd-bridge
 
+  sequin:
+    image: sequin/sequin:latest
+    restart: unless-stopped
+    ports:
+      - "7376:7376"
+    environment:
+      - PG_HOSTNAME=db
+      - PG_DATABASE=sequin
+      - PG_PORT=5432
+      - PG_USERNAME=postgres
+      - PG_PASSWORD=example
+      - PG_POOL_SIZE=10
+      - SECRET_KEY_BASE=${CROWD_SEQUIN_SECRET_KEY_BASE:-WyPLiGs0pvD6qJhKJICO4dauYPXfO/Yl782Zjtpew5qRBDp7CZvbWtQmY0eB13If}
+      - VAULT_KEY=${CROWD_SEQUIN_VAULT_KEY:-2Sig69bIpuSm2kv0VQfDekET2qy8qUZGI8v3/h3ASiY=}
+      - REDIS_URL=redis://default:crowdtest@redis:6379
+      - SERVER_HOST=${CROWD_SEQUIN_SERVER_HOST:-localhost}
+      - SERVER_PORT=${CROWD_SEQUIN_SERVER_PORT:-7376}
+    volumes:
+      - ./scaffold/sequin.yml:/config/sequin.yml
+    depends_on:
+      - db
+      - redis
+      - kafka
+    networks:
+      - crowd-bridge
+
+  tinybird:
+    image: tinybirdco/tinybird-local:latest
+    ports:
+      - "80:80"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:80/tokens"]
+      interval: 10s
+      retries: 5
+      start_period: 10s
+    networks:
+      - crowd-bridge
+
 networks:
   crowd-bridge:
     external: true
@@ -200,3 +237,5 @@ volumes:
   s3-dev:
   redis-dev:
   kafka-connect-dev:
+  sequin-dev:
+  tinybird-data:

scripts/scaffold/kafka-connect/Dockerfile

@@ -6,8 +6,10 @@ RUN yum install -y jq findutils unzip
 
 RUN confluent-hub install snowflakeinc/snowflake-kafka-connector:2.5.0 --no-prompt
 RUN confluent-hub install questdb/kafka-questdb-connector:0.12 --no-prompt
+COPY kafka-connect-http/ /usr/share/confluent-hub-components/kafka-connect-http/
+
 
 VOLUME /storage
 
-USER appuser
+USER root
 

scripts/scaffold/kafka-connect/README.md

@@ -0,0 +1,5 @@
+The Kafka HTTP Connector we're using comes from here:
+https://github.com/lensesio/stream-reactor/releases
+
+Its documentation is available here:
+https://docs.lenses.io/latest/connectors/kafka-connectors/sinks/http

scripts/scaffold/kafka-connect/tinybird-local-sink.properties

@@ -0,0 +1,13 @@
+name=tinybird-sink
+connector.class=io.lenses.streamreactor.connect.http.sink.HttpSinkConnector
+tasks.max=1
+topics=activities,activityRelations,members,memberIdentities,organizations,collections,insightsProjects,collectionsInsightsProjects
+connect.http.method=POST
+connect.http.endpoint=http://tinybird:80/v0/events?name={{topic}}
+connect.http.request.headers=Content-Type: application/json,Authorization: Bearer ${CROWD_TINYBIRD_WORKSPACE_ADMIN_TOKEN}
+connect.http.batch.count=10
+connect.http.time.interval=5
+key.converter=org.apache.kafka.connect.storage.StringConverter
+value.converter=org.apache.kafka.connect.storage.StringConverter
+value.converter.schemas.enable=false
+connect.http.request.content={{value}}