update

src/containers/docker/dagster/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.8'
+version: "3.8"
 x-dagster-common: &dagster_common
   build:
     context: .

src/containers/docker/minio/docker-compose.yml

@@ -10,12 +10,12 @@ services:
       - 9000:9000
       - 9001:9001
     environment:
-      MINIO_ROOT_USER: 'minio_user'
-      MINIO_ROOT_PASSWORD: 'minio_pwd'
-      MINIO_ADDRESS: ':9000'
-      MINIO_PORT: '9000'
+      MINIO_ROOT_USER: "minio_user"
+      MINIO_ROOT_PASSWORD: "minio_pwd"
+      MINIO_ADDRESS: ":9000"
+      MINIO_PORT: "9000"
       MINIO_STORAGE_USE_HTTPS: False
-      MINIO_CONSOLE_ADDRESS: ':9001'
+      MINIO_CONSOLE_ADDRESS: ":9001"
     command: server /data
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]

src/tf/aws-dagster/.env.sample

@@ -0,0 +1,4 @@
+POSTGRES_USER=
+POSTGRES_PASSWORD=
+POSTGRES_DB=
+POSTGRES_PORT=

src/tf/aws-dagster/Dockerfile.dagster

@@ -0,0 +1,17 @@
+FROM python:3.10-slim
+
+RUN pip install \
+    dagster \
+    dagster-graphql \
+    dagster-webserver \
+    dagster-postgres \
+    dagster-docker
+
+# Set $DAGSTER_HOME and copy dagster instance and workspace YAML there
+ENV DAGSTER_HOME=/opt/dagster/dagster_home/
+
+RUN mkdir -p $DAGSTER_HOME
+
+COPY dagster.yaml workspace.yaml $DAGSTER_HOME
+
+WORKDIR $DAGSTER_HOME

src/tf/aws-dagster/Dockerfile.pgvector

@@ -0,0 +1,17 @@
+FROM postgres:14.4-alpine AS pgvector-builder
+RUN apk add git
+RUN apk add build-base
+RUN apk add clang
+RUN apk add llvm13-dev
+WORKDIR /home
+RUN git clone --branch v0.4.4 https://github.com/pgvector/pgvector.git
+WORKDIR /home/pgvector
+RUN make
+RUN make install
+
+FROM postgres:14.4-alpine
+# COPY ./src/sql/init.sql /docker-entrypoint-initdb.d/
+# COPY ./src/docker/postgres/postgresql.conf /var/lib/postgresql/data/
+COPY --from=pgvector-builder /usr/local/lib/postgresql/bitcode/vector.index.bc /usr/local/lib/postgresql/bitcode/vector.index.bc
+COPY --from=pgvector-builder /usr/local/lib/postgresql/vector.so /usr/local/lib/postgresql/vector.so
+COPY --from=pgvector-builder /usr/local/share/postgresql/extension /usr/local/share/postgresql/extension

src/tf/aws-dagster/codelocations/Dockerfile.user_code

@@ -0,0 +1,35 @@
+ARG PYTHON_VERSION=3.11.3
+ARG MODULE_SRC_DIR=codelocations
+ARG MODULE_NAME=etl1
+ARG PORT=4000
+
+FROM python:${PYTHON_VERSION}-slim
+ARG MODULE_SRC_DIR
+ARG MODULE_NAME
+ARG PORT
+ENV MODULE_SRC_DIR=${MODULE_SRC_DIR}
+ENV MODULE NAME=${MODULE_NAME}
+ENV PORT=${PORT}
+
+# Checkout and install dagster libraries needed to run the gRPC server
+# exposing your repository to dagster-webserver and dagster-daemon, and to load the DagsterInstance
+
+RUN pip install \
+    dagster \
+    dagster-postgres \
+    dagster-docker \
+    pandas
+
+# Add repository code
+
+WORKDIR /opt/dagster/app
+
+COPY "$MODULE_SRC_DIR/$MODULE_NAME" /opt/dagster/app/
+RUN ls /opt/dagster/app
+
+# Run dagster gRPC server on port 4000
+EXPOSE ${PORT}
+
+# CMD allows this to be overridden from run launchers or executors that want
+# to run other commands against your repository
+CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", ${PORT}, "-f", "/opt/dagster/app/etl1/etl1/__init__.py"]

src/tf/aws-dagster/codelocations/etl1/etl1/__init__.py

@@ -0,0 +1,7 @@
+from dagster import asset
+from dagster_docker import docker_executor
+
+
+@asset
+def asset1():
+    return [1,2]

src/tf/aws-dagster/dagster.yaml

@@ -0,0 +1,64 @@
+scheduler:
+  module: dagster.core.scheduler
+  class: DagsterDaemonScheduler
+
+run_coordinator:
+  module: dagster.core.run_coordinator
+  class: QueuedRunCoordinator
+
+run_launcher:
+  module: dagster_docker
+  class: DockerRunLauncher
+  config:
+    env_vars:
+      - DAGSTER_POSTGRES_USER
+      - DAGSTER_POSTGRES_PASSWORD
+      - DAGSTER_POSTGRES_DB
+    network: net
+    container_kwargs:
+      volumes: # Make docker client accessible to any launched containers as well
+        - /var/run/docker.sock:/var/run/docker.sock
+        # - /Users/spare/Documents/data/io_manager_storage:/tmp/io_manager_storage
+        # - /Users/spare/Documents/proj/systemDeploy/src/containers/docker/dagster/repo.py:/opt/dagster/app/repo.py
+
+run_storage:
+  module: dagster_postgres.run_storage
+  class: PostgresRunStorage
+  config:
+    postgres_db:
+      hostname: dagster-db
+      username:
+        env: DAGSTER_POSTGRES_USER
+      password:
+        env: DAGSTER_POSTGRES_PASSWORD
+      db_name:
+        env: DAGSTER_POSTGRES_DB
+      port: 5432
+
+schedule_storage:
+  module: dagster_postgres.schedule_storage
+  class: PostgresScheduleStorage
+  config:
+    postgres_db:
+      hostname: dagster-db
+      username:
+        env: DAGSTER_POSTGRES_USER
+      password:
+        env: DAGSTER_POSTGRES_PASSWORD
+      db_name:
+        env: DAGSTER_POSTGRES_DB
+      port: 5432
+
+event_log_storage:
+  module: dagster_postgres.event_log
+  class: PostgresEventLogStorage
+  config:
+    postgres_db:
+      hostname: dagster-db
+      username:
+        env: DAGSTER_POSTGRES_USER
+      password:
+        env: DAGSTER_POSTGRES_PASSWORD
+      db_name:
+        env: DAGSTER_POSTGRES_DB
+      port: 5432

src/tf/aws-dagster/docker-compose.yml

@@ -0,0 +1,95 @@
+version: "3.11"
+x-dagster-common: &dagster_common
+  build:
+    context: .
+    dockerfile: Dockerfile.dagster
+  environment:
+    DAGSTER_POSTGRES_USER: ${POSTGRES_USER}
+    DAGSTER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    DAGSTER_POSTGRES_DB: ${POSTGRES_DB}
+    DAGSTER_POSTGRES_PORT: ${POSTGRES_PORT}
+  # volumes:
+  #   - /var/run/docker.sock:/var/run/docker.sock
+  # - /Users/spare/Documents/data/io_manager_storage:/tmp/io_manager_storage
+  networks:
+    - net
+  depends_on:
+    - dagster-db
+    - dagster-user-code-etl1
+networks:
+  net:
+    driver: bridge
+services:
+  dagster-db:
+    build:
+      context: .
+      dockerfile: Dockerfile.pgvector
+    restart: always
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DATABASE=postgres
+      # - PGDATA=/var/lib/postgresql/data/
+    # volumes:
+    #   - "./src/sql/init.sql:/docker-entrypoint-initdb.d/"
+    ports:
+      - "5455:5432"
+    networks:
+      - net
+  dagster-s3:
+    restart: always
+    image: minio/minio
+    container_name: minio
+    volumes:
+      - ./minio_data:/data
+    ports:
+      - 9000:9000
+      - 9001:9001
+    environment:
+      MINIO_ROOT_USER: "minio_user"
+      MINIO_ROOT_PASSWORD: "minio_pwd"
+      MINIO_ADDRESS: ":9000"
+      MINIO_PORT: "9000"
+      MINIO_STORAGE_USE_HTTPS: False
+      MINIO_CONSOLE_ADDRESS: ":9001"
+    command: server /data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    networks:
+      - net
+  dagster-user-code-etl1:
+    build:
+      context: .
+      dockerfile: codelocations/Dockerfile.user_code
+    container_name: dagster_user_code_etl1
+    restart: always
+    networks:
+      - net
+    depends_on:
+      - dagster-db
+  # webserver:
+  #   <<: *dagster_common
+  #   entrypoint:
+  #     - dagster-webserver
+  #     - -h
+  #     - "0.0.0.0"
+  #     - -p
+  #     - "3000"
+  #     - -w
+  #     - workspace.yaml
+  #   container_name: dagster_webserver
+  #   expose:
+  #     - 3000
+  #   ports:
+  #     - 3000:3000
+
+  # daemon:
+  #   <<: *dagster_common
+  #   entrypoint:
+  #     - dagster-daemon
+  #     - run
+  #   container_name: dagster_daemon
+  #   restart: on-failure