Split packages and add publish action

.github/workflows/publish_release.yml

@@ -0,0 +1,81 @@
+name: Publish Release
+on:
+  push:
+    branches:
+      - release_bfabric
+      - release_bfabric_scripts
+      - release_app_runner
+  workflow_dispatch:
+    inputs:
+      package:
+        description: 'Package to release'
+        type: choice
+        options:
+          - bfabric
+          - bfabric_scripts
+          - app_runner
+        default: bfabric
+      environment:
+        description: 'Target PyPI'
+        type: choice
+        options:
+          - test
+          - production
+        default: test
+        required: true
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      # Step: Determine the package that is being built
+      - name: Set package variable
+        id: set-package
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "PACKAGE=${{ github.event.inputs.package }}" >> $GITHUB_ENV
+          else
+            # Extract package name from branch name by removing 'release_' prefix
+            BRANCH_NAME="${{ github.ref_name }}"
+            PACKAGE=${BRANCH_NAME#release_}
+            echo "PACKAGE=$PACKAGE" >> $GITHUB_ENV
+          fi
+      # Step: Set the PyPI repository URL based on environment
+      - name: Set PyPI repository URL
+        id: set-repo-url
+        run: |
+          if [ "${{ github.event.inputs.environment }}" == "production" ] || [ "${{ github.event_name }}" == "push" ]; then
+            echo "PYPI_REPOSITORY_URL=https://pypi.org/legacy/" >> $GITHUB_ENV
+          else
+            echo "PYPI_REPOSITORY_URL=https://test.pypi.org/legacy/" >> $GITHUB_ENV
+          fi
+      - name: Debug variables
+        run: |
+          echo "Selected package: ${{ env.PACKAGE }}"
+          echo "Target PyPI repository: ${{ env.PYPI_REPOSITORY_URL }}"
+      # Step: Build the package
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Install hatch
+        run: pip install hatch
+      - name: Build package
+        run: |
+          cd ${{ env.PACKAGE }}
+          hatch build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: ${{ env.PYPI_REPOSITORY_URL }}
+          packages-dir: ${{ env.PACKAGE }}/dist
+      - name: Debug package info
+        run: |
+          echo "Built and published package: ${{ env.PACKAGE }}"
+          if [ "${{ env.PYPI_REPOSITORY_URL }}" == "https://pypi.org/legacy/" ]; then
+            echo "Check it at: https://pypi.org/project/${{ env.PACKAGE }}/"
+          else
+            echo "Check it at: https://test.pypi.org/project/${{ env.PACKAGE }}/"
+          fi

.github/workflows/run_unit_tests.yml

@@ -16,18 +16,20 @@ jobs:
           python-version: 3.9
       - name: Install nox
         run: pip install nox uv
-      - name: Run unit tests
+      - name: Run checks
         run: nox
   code_style:
     name: Code Style
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-bfabricpy
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.9
+      - name: Install nox
+        run: pip install nox uv
       - name: Check code with ruff
-        run: ruff bfabric || true
+        run: nox -s code_style
   list_todos:
     name: List TODOs
     runs-on: ubuntu-latest
@@ -37,13 +39,3 @@ jobs:
         name: Install ripgrep
       - run: rg -n TODO .
         name: List TODOs
-  license_check:
-    name: License Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-bfabricpy
-        with:
-          python-version: 3.9
-      - name: Check licenses
-        run: licensecheck

.pre-commit-config.yaml

@@ -14,7 +14,6 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: "v0.8.6"
     hooks:
-      - id: ruff-format
       - id: ruff
         args: ["--fix", "--show-fixes"]
   - repo: https://github.com/pre-commit/pre-commit-hooks

___pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+logot_capturer = logot.loguru.LoguruCapturer

app_runner/pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
-requires = ["setuptools >= 61.0"]
-build-backend = "setuptools.build_meta"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
 [project]
 name = "app_runner"
@@ -12,7 +12,7 @@ authors = [
 ]
 requires-python = ">=3.12"
 dependencies = [
-    "bfabric @ git+https://github.com/fgcz/bfabricPy.git@main",
+    "bfabric",
     "pydantic",
     "glom",
     "mako",

app_runner/src/app_runner/output_registration/register.py

@@ -15,15 +15,17 @@
 from app_runner.util.scp import scp
 from bfabric.entities import Resource
 from bfabric.entities import Storage, Workunit
-from bfabric_scripts.bfabric_save_csv2dataset import bfabric_save_csv2dataset
+from bfabric.experimental.upload_dataset import bfabric_save_csv2dataset
 
 if TYPE_CHECKING:
     from pathlib import Path
     from bfabric import Bfabric
     from bfabric.experimental.workunit_definition import WorkunitDefinition
 
 
-def _get_output_folder(spec: CopyResourceSpec, workunit_definition: WorkunitDefinition) -> Path:
+def _get_output_folder(
+    spec: CopyResourceSpec, workunit_definition: WorkunitDefinition
+) -> Path:
     if not spec.store_folder_path:
         return workunit_definition.registration.storage_output_folder
     else:
@@ -38,8 +40,14 @@ def register_file_in_workunit(
 ) -> None:
     """Registers a file in the workunit."""
     existing_id = _identify_existing_resource_id(client, spec, workunit_definition)
-    if resource_id is not None and existing_id is not None and resource_id != existing_id:
-        raise ValueError(f"Resource id {resource_id} does not match existing resource id {existing_id}")
+    if (
+        resource_id is not None
+        and existing_id is not None
+        and resource_id != existing_id
+    ):
+        raise ValueError(
+            f"Resource id {resource_id} does not match existing resource id {existing_id}"
+        )
 
     checksum = md5sum(spec.local_path)
     output_folder = _get_output_folder(spec, workunit_definition=workunit_definition)
@@ -68,18 +76,26 @@ def _identify_existing_resource_id(
         # TODO maybe it would be more accurate to use relativepath here, however historically it would often start
         #      with `/` which can be confusing.
         resources = Resource.find_by(
-            {"name": spec.store_entry_path.name, "workunitid": workunit_definition.registration.workunit_id},
+            {
+                "name": spec.store_entry_path.name,
+                "workunitid": workunit_definition.registration.workunit_id,
+            },
             client=client,
         ).values()
         if resources:
             return list(resources)[0].id
         elif spec.update_existing == UpdateExisting.REQUIRED:
-            raise ValueError(f"Resource {spec.store_entry_path.name} not found in workunit {workunit_definition.id}")
+            raise ValueError(
+                f"Resource {spec.store_entry_path.name} not found in workunit {workunit_definition.id}"
+            )
     return None
 
 
 def copy_file_to_storage(
-    spec: CopyResourceSpec, workunit_definition: WorkunitDefinition, storage: Storage, ssh_user: str | None
+    spec: CopyResourceSpec,
+    workunit_definition: WorkunitDefinition,
+    storage: Storage,
+    ssh_user: str | None,
 ) -> None:
     """Copies a file to the storage, according to the spec."""
     # TODO here some direct uses of storage could still be optimized away
@@ -88,7 +104,9 @@ def copy_file_to_storage(
     scp(spec.local_path, output_uri, user=ssh_user)
 
 
-def _save_dataset(spec: SaveDatasetSpec, client: Bfabric, workunit_definition: WorkunitDefinition) -> None:
+def _save_dataset(
+    spec: SaveDatasetSpec, client: Bfabric, workunit_definition: WorkunitDefinition
+) -> None:
     """Saves a dataset to the bfabric."""
     # TODO should not print to stdout in the future
     # TODO also it should not be imported from bfabric_scripts, but rather the generic functionality should be available
@@ -105,11 +123,17 @@ def _save_dataset(spec: SaveDatasetSpec, client: Bfabric, workunit_definition: W
     )
 
 
-def find_default_resource_id(workunit_definition: WorkunitDefinition, client: Bfabric) -> int | None:
+def find_default_resource_id(
+    workunit_definition: WorkunitDefinition, client: Bfabric
+) -> int | None:
     """Finds the default resource's id for the workunit. Maybe in the future, this will be always `None`."""
-    workunit = Workunit.find(id=workunit_definition.registration.workunit_id, client=client)
+    workunit = Workunit.find(
+        id=workunit_definition.registration.workunit_id, client=client
+    )
     candidate_resources = [
-        resource for resource in workunit.resources if resource["name"] not in ["slurm_stdout", "slurm_stderr"]
+        resource
+        for resource in workunit.resources
+        if resource["name"] not in ["slurm_stdout", "slurm_stderr"]
     ]
     # We also check that the resource is pending, as else we might re-use a resource that was created by the app...
     if len(candidate_resources) == 1 and candidate_resources[0]["status"] == "pending":
@@ -129,15 +153,27 @@ def register_all(
     for spec in specs_list:
         logger.debug(f"Registering {spec}")
         if isinstance(spec, CopyResourceSpec):
-            storage = Storage.find(workunit_definition.registration.storage_id, client=client)
-            copy_file_to_storage(spec, workunit_definition=workunit_definition, storage=storage, ssh_user=ssh_user)
+            storage = Storage.find(
+                workunit_definition.registration.storage_id, client=client
+            )
+            copy_file_to_storage(
+                spec,
+                workunit_definition=workunit_definition,
+                storage=storage,
+                ssh_user=ssh_user,
+            )
             if not default_resource_was_reused:
-                resource_id = find_default_resource_id(workunit_definition=workunit_definition, client=client)
+                resource_id = find_default_resource_id(
+                    workunit_definition=workunit_definition, client=client
+                )
                 default_resource_was_reused = True
             else:
                 resource_id = None
             register_file_in_workunit(
-                spec, client=client, workunit_definition=workunit_definition, resource_id=resource_id
+                spec,
+                client=client,
+                workunit_definition=workunit_definition,
+                resource_id=resource_id,
             )
         elif isinstance(spec, SaveDatasetSpec):
             _save_dataset(spec, client, workunit_definition=workunit_definition)

bfabric/pyproject.toml

@@ -0,0 +1,87 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "bfabric"
+description = "Python client for the B-Fabric API"
+version = "1.13.18"
+license = { text = "GPL-3.0" }
+authors = [
+    { name = "Christian Panse", email = "cp@fgcz.ethz.ch" },
+    { name = "Leonardo Schwarz", email = "leonardo.schwarz@fgcz.ethz.ch" },
+    { name = "Aleksejs Fomins" },
+    { name = "Marco Schmidt" },
+    { name = "Maria d'Errico" },
+    { name = "Witold Eryk Wolski" },
+]
+requires-python = ">=3.9"
+dependencies = [
+    "suds >= 1.1.2",
+    "PyYAML >= 6.0",
+    "Flask >= 3.0.3",
+    "rich >= 13.7.1",
+    "zeep >= 4.2.1",
+    "polars-lts-cpu >= 0.20.25; platform_machine == 'x86_64' and platform_system == 'Darwin'",
+    "polars >= 0.20.25; platform_machine != 'x86_64' or platform_system != 'Darwin'",
+    "loguru>=0.7",
+    "pydantic>=2.9.2",
+    "eval_type_backport; python_version < '3.10'",
+    "python-dateutil >= 2.9.0",
+    "cyclopts >= 2.9.9",
+    #"platformdirs >= 4.3",
+]
+
+[project.optional-dependencies]
+dev = [
+    "bfabric[doc,test]",
+    "black",
+    "isort",
+    "ruff",
+    "licensecheck",
+    "nox",
+    "uv",
+    "ipython",
+]
+doc = ["mkdocs", "mkdocs-material", "mkdocstrings[python]"]
+test = ["pytest", "pytest-mock", "logot[pytest,loguru]"]
+typing = ["mypy", "types-requests", "lxml-stubs", "pandas-stubs", "types-python-dateutil"]
+
+[project.urls]
+Homepage = "https://github.com/fgcz/bfabricPy"
+Repository = "https://github.com/fgcz/bfabricPy"
+
+
+[tool.uv]
+reinstall-package = ["bfabric", "bfabric_scripts", "app_runner"]
+
+[tool.black]
+line-length = 120
+target-version = ["py39"]
+
+[tool.ruff]
+line-length = 120
+indent-width = 4
+target-version = "py39"
+
+[tool.ruff.lint]
+select = ["ANN", "BLE", "D103", "E", "EXE", "F", "N", "PLW", "PTH", "SIM", "TCH", "UP", "W191"]
+ignore = ["ANN401"]
+
+[tool.ruff.lint.per-file-ignores]
+"**/bfabric_scripts/**" = ["ALL"]
+"**/wrapper_creator/**" = ["ALL"]
+"**/examples/**" = ["ALL"]
+"**/tests/**" = ["ALL"]
+"noxfile.py" = ["ALL"]
+
+[tool.licensecheck]
+using = "PEP631"
+
+#[tool.pytest.ini_options]
+#logot_capturer = "logot.loguru.LoguruCapturer"
+
+#[tool.check-tests-structure]
+#sources_path = "src/bfabric"
+#tests_path = "tests/unit"
+#allow_missing_tests = true