docker-compose.test.yml

services:
  db:
    image: postgres
    restart: unless-stopped
    env_file: .env.docker
    ports:
      - 5432:5432
    volumes:
      - postgres_test_data:/var/lib/postgresql/data
    healthcheck:
      test: pg_isready
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3

  rabbitmq:
    image: rabbitmq
    restart: unless-stopped
    ports:
      - 5672:5672
    volumes:
      - rabbitmq_test_data:/var/lib/rabbitmq
    healthcheck:
      test: rabbitmq-diagnostics -q ping
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3

  # Using worker image here to skip building server image build in CI.
  # Worker test image already has all neccessary dependencies
  migrations:
    image: mtsrus/syncmaster-worker:${WORKER_IMAGE_TAG:-test}
    restart: no
    build:
      dockerfile: docker/Dockerfile.worker
      context: .
      target: test
    volumes:
      - ./syncmaster:/app/syncmaster
      - ./tests:/app/tests
    entrypoint: [python, -m, syncmaster.db.migrations, upgrade, head]
    env_file: .env.docker
    depends_on:
      db:
        condition: service_healthy

  server:
    image: mtsrus/syncmaster-server:${SERVER_IMAGE_TAG:-test}
    restart: unless-stopped
    build:
      dockerfile: docker/Dockerfile.server
      context: .
      target: test
    env_file: .env.docker
    ports:
      - 8000:8000
    volumes:
      - ./syncmaster:/app/syncmaster
      - ./docs/_static:/app/docs/_static
      - ./reports:/app/reports
      - ./tests:/app/tests
      - ./pyproject.toml:/app/pyproject.toml
    depends_on:
      db:
        condition: service_healthy
      migrations:
        condition: service_completed_successfully
      rabbitmq:
        condition: service_healthy
    healthcheck:
      test: [CMD-SHELL, curl -f http://localhost:8000/monitoring/ping]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 5s
    profiles: [server, all]

  scheduler:
    image: mtsrus/syncmaster-scheduler:${SCHEDULER_IMAGE_TAG:-test}
    restart: unless-stopped
    build:
      dockerfile: docker/Dockerfile.scheduler
      context: .
      target: test
    env_file: .env.docker
    volumes:
      - ./syncmaster:/app/syncmaster
      - ./tests:/app/tests
      - ./reports:/app/reports
      - ./pyproject.toml:/app/pyproject.toml
    depends_on:
      db:
        condition: service_healthy
      migrations:
        condition: service_completed_successfully
      rabbitmq:
        condition: service_healthy
    profiles: [scheduler, all]

  worker:
    image: mtsrus/syncmaster-worker:${WORKER_IMAGE_TAG:-test}
    restart: unless-stopped
    build:
      dockerfile: docker/Dockerfile.worker
      context: .
      target: test
    command: --loglevel=info -Q test_queue
    entrypoint: [coverage, run, -m, celery, -A, tests.test_integration.celery_test, worker, --max-tasks-per-child=1]
    env_file: .env.docker
    environment:
      # CI runs tests in the worker container, so we need to turn off interaction with static files for it
      - SYNCMASTER__SERVER__STATIC_FILES__ENABLED=false
    volumes:
      - ./syncmaster:/app/syncmaster
      - ./cached_jars:/root/.ivy2
      - ./reports:/app/reports
      - ./tests:/app/tests
      - ./pyproject.toml:/app/pyproject.toml
    depends_on:
      db:
        condition: service_healthy
      migrations:
        condition: service_completed_successfully
      rabbitmq:
        condition: service_healthy
    profiles: [worker, scheduler, s3, oracle, hdfs, hive, clickhouse, mysql, mssql, sftp, ftp, ftps, samba, all]

  test-postgres:
    image: postgres
    restart: unless-stopped
    ports:
      - 5433:5432
    env_file: .env.docker
    healthcheck:
      test: pg_isready
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3
    profiles: [s3, oracle, clickhouse, mysql, mssql, hdfs, hive, sftp, ftp, ftps, samba, all]

  test-s3:
    image: bitnami/minio:latest
    container_name: test-s3
    restart: unless-stopped
    env_file: .env.docker
    ports:
      - 9010:9000
      - 9011:9001
    healthcheck:
      test: curl -f http://localhost:9000/minio/health/live
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3
    profiles: [s3, all]

  test-oracle:
    image: gvenzl/oracle-xe:slim-faststart
    restart: unless-stopped
    ports:
      - 1522:1521
    environment:
      TZ: UTC
      ORACLE_PASSWORD: changeme
      ORACLE_DATABASE: syncmaster
      APP_USER: syncmaster
      APP_USER_PASSWORD: changeme
    profiles: [oracle, all]

  test-clickhouse:
    image: clickhouse/clickhouse-server
    restart: unless-stopped
    ports:
      - 8123:8123
      - 9001:9000
    profiles: [clickhouse, all]

  test-mssql:
    image: mcr.microsoft.com/mssql/server
    restart: unless-stopped
    environment:
      ACCEPT_EULA: Y
      MSSQL_PID: Developer
      MSSQL_SA_PASSWORD: 7ellowEl7akey
      MSSQL_DATABASE: syncmaster
      MSSQL_USER: syncmaster
      MSSQL_PASSWORD: 7ellowEl7akey
    ports:
      - 1433:1433
    volumes:
      - ./docker/mssql/:/usr/config/
    entrypoint: [/usr/config/entrypoint.sh]
    platform: linux/amd64
    profiles: [mssql, all]

  test-mysql:
    image: mysql
    restart: unless-stopped
    environment:
      MYSQL_ROOT_PASSWORD: ohbuz9Eochaj9saibooK3thooGa5aesh
      MYSQL_DATABASE: syncmaster
      MYSQL_USER: syncmaster
      MYSQL_PASSWORD: ohbuz9Eochaj9saibooK3thooGa5aesh
    ports:
      - 3306:3306
    platform: linux/amd64
    profiles: [mysql, all]


  metastore-hive:
    image: postgres
    restart: unless-stopped
    environment:
      POSTGRES_DB: metastore
      POSTGRES_USER: test_hive
      POSTGRES_PASSWORD: test_hive
    ports:
      - 5440:5432
    healthcheck:
      test: pg_isready
      start_period: 5s
      interval: 30s
      timeout: 5s
      retries: 3
    profiles: [hive, hdfs, s3, sftp, ftp, ftps, samba, all]

  keycloak:
    image: quay.io/keycloak/keycloak:latest
    command: start-dev
    restart: unless-stopped
    environment:
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: admin
    ports:
      - 8080:8080
    volumes:
      - keycloak_data:/opt/keycloak/data
    profiles: [keycloak, all]

  test-hive:
    image: mtsrus/hadoop:hadoop2.7.3-hive2.3.9
    restart: unless-stopped
    hostname: test-hive
    depends_on:
      metastore-hive:
        condition: service_healthy
    ports:
      - 9820:9820  # HDFS IPC
      - 9870:9870  # HDFS WebHDFS
      - 8088:8088  # Yarn UI
      - 8042:8042  # NodeManager UI
      - 10000:10000  # Hive server
      - 10002:10002  # Hive server Admin UI
      - 19888:19888  # MapReduce JobServer History UI
      - 9083:9083  # Hive Metastore server
      - 9864:9864 # Datanode UI
    environment:
      WITH_HIVE_SERVER: 'false'  # We leave only the metastore server, we don’t need Hive itself, we don’t waste resources on it
      HIVE_METASTORE_DB_URL: jdbc:postgresql://metastore-hive:5432/metastore
      HIVE_METASTORE_DB_DRIVER: org.postgresql.Driver
      HIVE_METASTORE_DB_USER: test_hive
      HIVE_METASTORE_DB_PASSWORD: test_hive
    # writing spark dataframe to s3, sftp, ftp, ftps xml file fails without running hive metastore server
    profiles: [hive, hdfs, s3, sftp, ftp, ftps, samba, all]

  test-sftp:
    image: ${SFTP_IMAGE:-linuxserver/openssh-server}
    restart: unless-stopped
    ports:
      - 2222:2222
    environment:
      PUID: 1000
      PGID: 1000
      USER_NAME: syncmaster
      PASSWORD_ACCESS: true
      SUDO_ACCESS: true
      USER_PASSWORD: test_only
    profiles: [sftp, all]

  test-ftp:
    image: ${FTP_IMAGE:-chonjay21/ftps:latest}
    restart: unless-stopped
    environment:
      - USE_SSL=false
      - PASSV_MIN_PORT=30000
      - PASSV_MAX_PORT=30010
      - APP_USER_NAME=syncmaster
      - APP_USER_PASSWD=test_only
      - APP_UID=1000
      - APP_GID=1000
    ports:
      - 2121:21
      - 30000-30010:30000-30010
    volumes:
      - ./docker/ftp/on_post_init.sh:/sources/ftp/eventscripts/on_post_init.sh
    profiles: [ftp, all]

  test-ftps:
    image: ${FTPS_IMAGE:-chonjay21/ftps:latest}
    restart: unless-stopped
    environment:
      - USE_SSL=true
      - PASSV_MIN_PORT=30020
      - PASSV_MAX_PORT=30030
      - APP_USER_NAME=syncmaster
      - APP_USER_PASSWD=test_only
      - APP_UID=1000
      - APP_GID=1000
    ports:
      - 2122:21
      - 30020-30030:30020-30030
    volumes:
      - ./docker/ftp/on_post_init.sh:/sources/ftps/eventscripts/on_post_init.sh
    profiles: [ftps, all]

  test-samba:
    image: ${SAMBA_IMAGE:-elswork/samba}
    restart: unless-stopped
    ports:
      - 139:139
      - 1445:445
    volumes:
      - ./docker/samba/custom_entrypoint.sh:/custom_entrypoint.sh
    entrypoint: [/custom_entrypoint.sh]
    profiles: [samba, all]

volumes:
  postgres_test_data:
  rabbitmq_test_data:
  keycloak_data: