Create a basic task

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 name = "hipercow"
 description = ''
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.10"
 license = "MIT"
 keywords = []
 authors = [
@@ -53,8 +53,8 @@ cov-ci = [
   "cov-report-xml",
 ]
 
-[[tool.hatch.envs.test.matrix]]
-python = ["37", "38", "39", "310", "311"]
+[[tool.hatch.envs.all.matrix]]
+python = ["3.9", "3.10", "3.11", "3.12", "3.13"]
 
 [tool.hatch.envs.lint]
 extra-dependencies = [
@@ -65,12 +65,12 @@ extra-dependencies = [
 [tool.hatch.envs.lint.scripts]
 typing = "mypy --install-types --non-interactive {args:src tests}"
 style = [
-  "ruff {args:.}",
-  "black --check --diff {args:.}",
+  "ruff check {args:.}",
+   "black --check --diff {args:.}",
 ]
 fmt = [
   "black {args:.}",
-  "ruff --fix {args:.}",
+  "ruff check --fix {args:.}",
   "style",
 ]
 all = [
@@ -97,13 +97,13 @@ exclude_lines = [
 ]
 
 [tool.black]
-target-version = ["py37"]
 line-length = 80
 skip-string-normalization = true
 
 [tool.ruff]
-target-version = "py37"
 line-length = 80
+
+[tool.ruff.lint]
 select = [
   "A",
   "ARG",
@@ -145,6 +145,8 @@ ignore = [
   "D100", "D101", "D102", "D103", "D104", "D105",
   # Ignore shadowing
   "A001", "A002", "A003",
+  # Allow pickle
+  "S301",
   # Allow print until we find the alternative to R's cli
   "T201"
 ]
@@ -154,9 +156,9 @@ unfixable = [
   "F401",
 ]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "numpy"

src/hipercow/root.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+from hipercow.util import find_file_descend
+
+
+def init(path: str | Path) -> None:
+    path = Path(path)
+    dest = path / "hipercow"
+    if dest.exists():
+        if dest.is_dir():
+            print(f"hipercow already initialised at {path}")
+        else:
+            msg = (
+                "Unexpected file 'hipercow' (rather than directory)"
+                f"found at {path}"
+            )
+            raise Exception(msg)
+    else:
+        dest.mkdir(parents=True)
+        print(f"Initialised hipercow at {path}")
+
+
+class Root:
+    def __init__(self, path: str | Path) -> None:
+        path = Path(path)
+        if not (path / "hipercow").is_dir():
+            msg = f"Failed to open 'hipercow' root at {path}"
+            raise Exception(msg)
+        self.path = path
+
+
+def open_root(path: None | str | Path = None) -> Root:
+    root = find_file_descend("hipercow", path or Path.cwd())
+    if not root:
+        msg = f"Failed to find 'hipercow' from {path}"
+        raise Exception(msg)
+    return Root(root)

src/hipercow/task.py

@@ -0,0 +1,32 @@
+import pickle
+from dataclasses import dataclass
+
+from hipercow.root import Root
+
+
+@dataclass
+class TaskData:
+    task_id: str
+    method: str  # shell etc
+    data: dict
+    path: str
+    envvars: dict[str, str]
+
+    def write(self, root: Root):
+        task_data_write(root, self)
+
+    @staticmethod
+    def read(root: Root, task_id: str):
+        return task_data_read(root, task_id)
+
+
+def task_data_write(root: Root, data: TaskData) -> None:
+    path_task_dir = root.path / "tasks"
+    path_task_dir.mkdir(parents=True, exist_ok=True)
+    with open(path_task_dir / data.task_id, "wb") as f:
+        pickle.dump(data, f)
+
+
+def task_data_read(root: Root, task_id: str) -> TaskData:
+    with open(root.path / "tasks" / task_id, "rb") as f:
+        return pickle.load(f)

src/hipercow/task_create.py

@@ -0,0 +1,28 @@
+import secrets
+
+from hipercow.root import Root, open_root
+from hipercow.task import TaskData
+from hipercow.util import relative_workdir
+
+
+def task_create_shell(
+    cmd: list[str], envvars: dict[str, str] | None = None
+) -> str:
+    root = open_root()
+    data = {"cmd": cmd}
+    id = task_create(root, "shell", data, envvars or {})
+    # task_submit_maybe(id, driver, root)
+    return id
+
+
+def task_create(
+    root: Root, method: str, data: dict, envvars: dict[str, str]
+) -> str:
+    path = relative_workdir(root.path)
+    task_id = new_task_id()
+    TaskData(task_id, method, data, str(path), envvars).write(root)
+    return task_id
+
+
+def new_task_id() -> str:
+    return secrets.token_hex(16)

src/hipercow/util.py

@@ -1,3 +1,5 @@
+import os
+from contextlib import contextmanager
 from pathlib import Path
 
 
@@ -12,3 +14,19 @@ def find_file_descend(filename, path):
         path = path.parent
 
     return None
+
+
+def relative_workdir(path: str | Path, base: None | str | Path = None) -> Path:
+    return Path(path).relative_to(Path(base) if base else Path.cwd())
+
+
+@contextmanager
+def transient_working_directory(path):
+    origin = os.getcwd()
+    try:
+        if path is not None:
+            os.chdir(path)
+        yield
+    finally:
+        if path is not None:
+            os.chdir(origin)

tests/test_create.py

@@ -0,0 +1,21 @@
+import re
+
+from hipercow import root
+from hipercow import task_create as tc
+from hipercow.task import TaskData
+from hipercow.util import transient_working_directory
+
+
+def test_create_simple_task(tmp_path):
+    root.init(tmp_path)
+    with transient_working_directory(tmp_path):
+        tid = tc.task_create_shell(["echo", "hello world"])
+    assert re.match("^[0-9a-f]{32}$", tid)
+    assert (tmp_path / "tasks" / tid).exists()
+    d = TaskData.read(root.open_root(tmp_path), tid)
+    assert isinstance(d, TaskData)
+    assert d.task_id == tid
+    assert d.method == "shell"
+    assert d.data == {"cmd": ["echo", "hello world"]}
+    assert d.path == "."
+    assert d.envvars == {}

tests/test_root.py

@@ -0,0 +1,47 @@
+import pytest
+
+from hipercow import root
+
+
+def test_create_root(tmp_path):
+    path = tmp_path / "ex"
+    root.init(path)
+    assert path.exists()
+    assert path.is_dir()
+    r = root.open_root(path)
+    assert isinstance(r, root.Root)
+    assert r.path == path
+
+
+def test_notify_if_root_exists(tmp_path, capsys):
+    path = tmp_path
+    root.init(path)
+    capsys.readouterr()
+    root.init(path)
+    captured = capsys.readouterr()
+    assert captured.out.startswith("hipercow already initialised at")
+
+
+def test_error_if_root_invalid(tmp_path):
+    with open(tmp_path / "hipercow", "w") as _:
+        pass
+    with pytest.raises(Exception, match="Unexpected file 'hipercow'"):
+        root.init(tmp_path)
+
+
+def test_error_if_root_does_not_exist(tmp_path):
+    with pytest.raises(Exception, match="Failed to open 'hipercow' root"):
+        root.Root(tmp_path)
+
+
+def test_find_root_by_descending(tmp_path):
+    path = tmp_path / "a" / "b"
+    root.init(tmp_path)
+    r = root.open_root(path)
+    assert r.path == tmp_path
+
+
+def test_error_if_no_root_found_by_descending(tmp_path):
+    path = tmp_path / "a" / "b"
+    with pytest.raises(Exception, match="Failed to find 'hipercow' from"):
+        root.open_root(path)

tests/test_util.py

@@ -1,4 +1,6 @@
-from hipercow.util import find_file_descend
+from pathlib import Path
+
+from hipercow.util import find_file_descend, transient_working_directory
 
 
 def test_find_descend(tmp_path):
@@ -9,3 +11,11 @@ def test_find_descend(tmp_path):
         == (tmp_path / "a" / "b").resolve()
     )
     assert find_file_descend(".foo", tmp_path / "a") is None
+
+
+def test_transient_working_directory(tmp_path):
+    here = Path.cwd()
+    with transient_working_directory(None):
+        assert Path.cwd() == here
+    with transient_working_directory(tmp_path):
+        assert Path.cwd() == tmp_path