Create postgresql test for amt

.github/workflows/ci.yml

@@ -73,7 +73,24 @@ jobs:
           trivy-config: trivy.yaml
           scan-type: fs
           scan-ref: '.'
-  test:
+
+  test-compose:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: mv compose.test.yml compose.override.yml
+      - run: docker compose build
+      - run: docker compose down -v --remove-orphans
+      - run: docker compose up -d
+      - name: test app
+        run: docker compose run amt-test poetry run pytest -m 'not slow' --db postgresql
+      - name: db downgrade test
+        run: docker compose exec -T amt alembic downgrade -1
+      - name: db upgrade test
+        run: docker compose exec -T amt alembic upgrade head
+      - run: docker compose down -v --remove-orphans
+
+  test-local:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -144,7 +161,7 @@ jobs:
 
 
   build:
-    needs: test
+    needs: [test-local, test-compose]
     runs-on: ubuntu-latest
     permissions:
       packages: write
@@ -296,7 +313,7 @@ jobs:
 
   notifyMattermost:
     runs-on: ubuntu-latest
-    needs: [lint, security, test, build ]
+    needs: [lint, security, test-local, test-compose, build ]
     if: ${{ always() && contains(needs.*.result, 'failure') }}
     steps:
       - uses: mattermost/action-mattermost-notify@master

Dockerfile

@@ -45,8 +45,9 @@ RUN ruff format --check
 RUN pyright
 
 FROM development AS test
-RUN coverage run -m pytest
-RUN coverage report
+
+COPY ./example/ ./example/
+# RUN poetry run playwright install --with-deps
 
 FROM project-base AS production
 

amt/migrations/versions/9ce2341f2922_remove_the_status_table.py

@@ -9,6 +9,7 @@
 from collections.abc import Sequence
 
 import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
 from alembic import op
 
 # revision identifiers, used by Alembic.
@@ -35,15 +36,24 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table("task", schema=None) as batch_op:
-        batch_op.create_foreign_key("task_status_id_fkey", "status", ["status_id"], ["id"])
+
+    naming_convention = {
+        "fk":
+            "%(table_name)s_%(column_0_name)s_fkey"
+    }
 
     op.create_table(
         "status",
-        sa.Column("id", sa.INTEGER(), nullable=False),
-        sa.Column("name", sa.VARCHAR(), nullable=False),
-        sa.Column("sort_order", sa.FLOAT(), nullable=False),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+        sa.Column("sort_order", sa.Float(), nullable=False),
         sa.PrimaryKeyConstraint("id"),
     )
+
+    op.execute("UPDATE task SET status_id = NULL")
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("task", naming_convention=naming_convention) as batch_op:
+        batch_op.create_foreign_key("task_status_id_fkey", "status", ["status_id"], ["id"])
+
     # ### end Alembic commands ###

compose.test.yml

@@ -0,0 +1,10 @@
+services:
+  amt-test:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: test
+    image: ghcr.io/minbzk/amt-test:latest
+    env_file:
+      - path: prod.env
+        required: true

compose.yml

@@ -5,6 +5,8 @@ services:
       dockerfile: Dockerfile
     image: ghcr.io/minbzk/amt:latest
     restart: unless-stopped
+    volumes:
+      - ./amt:/app/amt/:cached
     depends_on:
       db:
         condition: service_healthy

database/init-user-db.sh

@@ -3,6 +3,6 @@ set -e
 
 # todo(berry): make user and database variables
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-	CREATE USER amt WITH PASSWORD 'changethis';
+	CREATE USER amt WITH PASSWORD 'changethis' CREATEDB;
 	CREATE DATABASE amt OWNER amt;
 EOSQL

script/test-docker

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+cp -f compose.test.yml compose.override.yml
+
+if ! docker compose build ; then
+    echo "Failed building container"
+    rm -f compose.override.yml
+    exit 1
+fi
+
+docker compose up -d
+docker compose run amt-test poetry run pytest -m 'not slow' --db postgresql
+rm -f compose.override.yml

tests/api/routes/test_projects.py

@@ -64,7 +64,9 @@ def test_post_new_projects_bad_request(client: TestClient, mock_csrf: Generator[
     assert b"name: Field required" in response.content
 
 
-def test_post_new_projects(client: TestClient, mock_csrf: Generator[None, None, None]) -> None:
+def test_post_new_projects(
+    client: TestClient, mock_csrf: Generator[None, None, None], init_instruments: Generator[None, None, None]
+) -> None:
     client.cookies["fastapi-csrf-token"] = "1"
     new_project = ProjectNew(name="default project")
 
@@ -77,7 +79,9 @@ def test_post_new_projects(client: TestClient, mock_csrf: Generator[None, None,
     assert response.headers["HX-Redirect"] == "/project/1"
 
 
-def test_post_new_projects_write_system_card(client: TestClient, mock_csrf: Generator[None, None, None]) -> None:
+def test_post_new_projects_write_system_card(
+    client: TestClient, mock_csrf: Generator[None, None, None], init_instruments: Generator[None, None, None]
+) -> None:
     # Given
     client.cookies["fastapi-csrf-token"] = "1"
     origin = FileSystemStorageService.write

tests/api/routes/test_status.py

@@ -3,12 +3,13 @@
 from amt.schema.task import MovedTask
 from fastapi.testclient import TestClient
 
-from tests.constants import default_task
+from tests.constants import default_task, default_user
 from tests.database_test_utils import DatabaseTestUtils
 
 
 def test_post_move_task(client: TestClient, db: DatabaseTestUtils, mock_csrf: Generator[None, None, None]) -> None:
     db.given([default_task(), default_task(), default_task()])
+    db.given([default_user()])
     client.cookies["fastapi-csrf-token"] = "1"
 
     move_task: MovedTask = MovedTask(taskId=2, statusId=2, previousSiblingId=1, nextSiblingId=3)

tests/conftest.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import re
 from collections.abc import Callable, Generator
 from multiprocessing import Process
 from pathlib import Path
@@ -14,6 +15,7 @@
 from fastapi.testclient import TestClient
 from fastapi_csrf_protect import CsrfProtect  # type: ignore
 from playwright.sync_api import Browser
+from sqlalchemy import text
 from sqlmodel import Session, SQLModel, create_engine
 
 from tests.database_e2e_setup import setup_database_e2e
@@ -31,10 +33,16 @@ def run_server_uvicorn(database_file: Path, host: str = "127.0.0.1", port: int =
 
 
 @pytest.fixture(scope="session")
-def setup_db_and_server(tmp_path_factory: pytest.TempPathFactory) -> Generator[Any, None, None]:
+def setup_db_and_server(
+    tmp_path_factory: pytest.TempPathFactory, request: pytest.FixtureRequest
+) -> Generator[Any, None, None]:
     test_dir = tmp_path_factory.mktemp("e2e_database")
     database_file = test_dir / "test.sqlite3"
-    engine = create_engine(f"sqlite:///{database_file}", connect_args={"check_same_thread": False})
+
+    if request.config.getoption("--db") == "postgresql":
+        engine = create_engine(get_db_uri())
+    else:
+        engine = create_engine(f"sqlite:///{database_file}", connect_args={"check_same_thread": False})
     SQLModel.metadata.create_all(engine)
 
     with Session(engine, expire_on_commit=False) as session:
@@ -44,12 +52,18 @@ def setup_db_and_server(tmp_path_factory: pytest.TempPathFactory) -> Generator[A
     process.start()
     yield "http://127.0.0.1:3462"
     process.terminate()
+    SQLModel.metadata.drop_all(engine)
 
 
 def pytest_configure(config: pytest.Config) -> None:
-    os.environ.clear()  # lets always start with a clean environment to make tests consistent
-    os.environ["ENVIRONMENT"] = "local"
-    os.environ["APP_DATABASE_SCHEME"] = "sqlite"
+    if "APP_DATABASE_SCHEME" not in os.environ:
+        os.environ["APP_DATABASE_SCHEME"] = str(config.getoption("--db"))
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--db", action="store", default="sqlite", help="db: sqlite or postgresql", choices=("sqlite", "postgresql")
+    )
 
 
 def pytest_collection_modifyitems(session: pytest.Session, config: pytest.Config, items: list[pytest.Item]):
@@ -106,10 +120,63 @@ def browser(
     browser.close()
 
 
+def get_db_uri() -> str:
+    user = os.getenv("APP_DATABASE_USER", "amt")
+    password = os.getenv("APP_DATABASE_PASSWORD", "changethis")
+    server = os.getenv("APP_DATABASE_SERVER", "db")
+    port = os.getenv("APP_DATABASE_PORT", "5432")
+    db = os.getenv("APP_DATABASE_DB", "amt")
+    return f"postgresql://{user}:{password}@{server}:{port}/{db}"
+
+
+def create_db(new_db: str) -> str:
+    url = get_db_uri()
+    engine = create_engine(url, isolation_level="AUTOCOMMIT")
+
+    if new_db == os.getenv("APP_DATABASE_USER", "amt"):
+        return url
+
+    logger.info(f"Creating database {new_db}")
+    user = os.getenv("APP_DATABASE_USER", "amt")
+    with Session(engine) as session:
+        session.execute(text(f"DROP DATABASE IF EXISTS {new_db};"))  # type: ignore
+        session.execute(text(f"CREATE DATABASE {new_db} OWNER {user};"))  # type: ignore
+        session.commit()
+
+    path = Path(url)
+
+    return str(path.parent / new_db).replace("postgresql:/", "postgresql://")
+
+
+def generate_db_name(request: pytest.FixtureRequest) -> str:
+    pattern = re.compile(r"[^a-zA-Z0-9_]")
+
+    inverted_modulename = ".".join(request.module.__name__.split(".")[::-1])  # type: ignore
+    testname = request.node.name  # type: ignore
+
+    db_name = testname + "_" + inverted_modulename  # type: ignore
+    sanitized_name = pattern.sub("_", db_name)  # type: ignore
+
+    # postgres has a limit of 63 bytes for database names
+    if len(sanitized_name) > 63:
+        sanitized_name = sanitized_name[:63]
+
+    return sanitized_name
+
+
 @pytest.fixture()
-def db(tmp_path: Path) -> Generator[DatabaseTestUtils, None, None]:
+def db(
+    tmp_path: Path, request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch
+) -> Generator[DatabaseTestUtils, None, None]:
     database_file = tmp_path / "test.sqlite3"
-    engine = create_engine(f"sqlite:///{database_file}", connect_args={"check_same_thread": False})
+
+    if request.config.getoption("--db") == "postgresql":
+        db_name: str = generate_db_name(request)
+        url = create_db(db_name)
+        monkeypatch.setenv("APP_DATABASE_DB", db_name)
+        engine = create_engine(url)
+    else:
+        engine = create_engine(f"sqlite:///{database_file}", connect_args={"check_same_thread": False})
     SQLModel.metadata.create_all(engine)
 
     with Session(engine, expire_on_commit=False) as session:

tests/core/test_config.py

@@ -3,25 +3,15 @@
 from amt.core.exceptions import SettingsError
 
 
-def test_default_settings():
-    settings = Settings(_env_file=None)  # pyright: ignore [reportCallIssue]
-
-    assert settings.ENVIRONMENT == "local"
-    assert settings.LOGGING_LEVEL == "INFO"
-    assert settings.APP_DATABASE_SCHEME == "sqlite"
-    assert settings.APP_DATABASE_SERVER == "db"
-    assert settings.APP_DATABASE_PORT == 5432
-    assert settings.APP_DATABASE_USER == "amt"
-    assert settings.APP_DATABASE_DB == "amt"
-
-
 def test_environment_settings(monkeypatch: pytest.MonkeyPatch):
     monkeypatch.setenv("ENVIRONMENT", "production")
     monkeypatch.setenv("SECRET_KEY", "mysecret")
     monkeypatch.setenv("APP_DATABASE_SCHEME", "postgresql")
     monkeypatch.setenv("APP_DATABASE_USER", "amt2")
     monkeypatch.setenv("APP_DATABASE_DB", "amt2")
     monkeypatch.setenv("APP_DATABASE_PASSWORD", "mypassword")
+    monkeypatch.setenv("APP_DATABASE_SERVER", "db")
+
     settings = Settings(_env_file=None)  # pyright: ignore [reportCallIssue]
 
     assert settings.SECRET_KEY == "mysecret"  # noqa: S105

tests/core/test_exception_handlers.py

@@ -41,7 +41,7 @@ def test_request_validation_exception_handler_htmx(client: TestClient):
     assert response.headers["content-type"] == "text/html; charset=utf-8"
 
 
-def test_request_csrf_protect_exception_handler_invalid_token_in_header_htmx(client: TestClient):
+def test_request_csrf_protect_exception_handler_invalid_token(client: TestClient):
     data = client.get("/projects/new")
     new_project = ProjectNew(name="default project")
     response = client.post(

tests/database_test_utils.py

@@ -11,11 +11,9 @@ class DatabaseTestUtils:
     def __init__(self, session: Session, database_file: Path | None = None) -> None:
         self.session: Session = session
         self.database_file: Path | None = database_file
-        self.models: list[BaseModel] = []
 
     def given(self, models: list[BaseModel]) -> None:
         session = self.get_session()
-        self.models.extend(models)
         session.add_all(models)
 
         session.commit()

tests/repositories/test_tasks.py

@@ -3,7 +3,7 @@
 from amt.enums.status import Status
 from amt.models import Task
 from amt.repositories.tasks import TasksRepository
-from tests.constants import default_task
+from tests.constants import default_project, default_task
 from tests.database_test_utils import DatabaseTestUtils
 
 
@@ -70,7 +70,7 @@ def test_save_all_failed(db: DatabaseTestUtils):
     tasks_repository.delete(task)  # cleanup
 
 
-def test_delete(db: DatabaseTestUtils):
+def test_delete_task(db: DatabaseTestUtils):
     tasks_repository: TasksRepository = TasksRepository(db.get_session())
     task: Task = Task(id=1, title="Test title", description="Test description", sort_order=10)
 
@@ -127,6 +127,7 @@ def test_find_by_status_id(db: DatabaseTestUtils):
 
 
 def test_find_by_project_id_and_status_id(db: DatabaseTestUtils):
+    db.given([default_project()])
     task = default_task(status_id=Status.TODO, project_id=1)
     db.given([task, default_task()])