test_destinations.yml

name: test postgres and duckdb

on:
  pull_request:
    branches:
      - master
      - devel
  workflow_dispatch:

env:
  ALL_DESTINATIONS: '["postgres", "duckdb"]'
  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
  # DESTINATIONS_SECRETS: ${{ secrets.DESTINATIONS_SECRETS }}
  # SOURCES_SECRETS: ${{ secrets.SOURCES_SECRETS }}
  RUNTIME__LOG_LEVEL: ERROR

jobs:
  get_changed_pipelines:
    uses: ./.github/workflows/get_changed_pipelines.yml

  run_loader:
    name: test destinations redshift, postgres and duckdb
    needs: get_changed_pipelines
    if: needs.get_changed_pipelines.outputs.pipelines_list != ''
    strategy:
      max-parallel: 2
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}

    steps:
      - name: Check out
        uses: actions/checkout@master

      - name: Setup Python
        uses: actions/setup-python@v1
        with:
          python-version: "3.9.x"

      - name: Install Poetry
        uses: snok/install-poetry@v1.3.2
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v2
        with:
          # path: ${{ steps.pip-cache.outputs.dir }}
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-destinations

      - name: Install dependencies
        # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --all-extras

      - name: create secrets.toml
        run: pwd && echo "$DLT_SECRETS_TOML" > pipelines/.dlt/secrets.toml
      #   run: pwd && echo "$DESTINATIONS_SECRETS" > pipelines/.dlt/secrets.toml && echo "$SOURCES_SECRETS" >> pipelines/.dlt/secrets.toml
      # - name: Setup upterm session
      #   uses: lhotari/action-upterm@v1
      - run: |
          pipelines_list="${{ needs.get_changed_pipelines.outputs.pipelines_list }}"
          test_paths=$(echo "$pipelines_list" | awk '{for(i=1;i<=NF;i++) printf "tests/%s ", $i}')
          poetry run pytest $test_paths
        if: runner.os != 'Windows'
        name: Run tests on Linux/macOS
      - run: |
          FOR %%i IN (${{ needs.get_changed_pipelines.outputs.pipelines_list }}) DO (
            echo Running tests for: %%i
            poetry run pytest tests/%%i
          )
        if: runner.os == 'Windows'
        name: Run tests on Windows
        shell: cmd