Add support for the file:// protocol for base-url. (#23)

python/CHANGES.md

@@ -1,5 +1,9 @@
 # insta-science
 
+## 0.5.0
+
+Add support for file:// base URLs to allow using local `science` binary mirrors.
+
 ## 0.4.5
 
 De-dup `insta-science-util download --platform` and fix its CLI help.

python/README.md

@@ -40,7 +40,8 @@ There is partial support for offline or firewalled `science` use with `insta-sci
 a repository of science binaries by using the `insta-science-util download` command to download
 `science` binaries for one or more versions and one or more target platforms. The directory you
 download these binaries to will have the appropriate structure for `insta-science` to use if you
-serve up that directory using your method of choice at the configured base url.
+serve up that directory using your method of choice at the configured base url. Note that file://
+base URLs are supported.
 
 Full offline use requires similar support in `science` for downloading offline copies of the
 [`scie-jump` binaries](https://github.com/a-scie/jump/releases), [`ptex` binaries](

python/insta_science/_internal/fetcher.py

@@ -4,16 +4,21 @@
 from __future__ import annotations
 
 import hashlib
+import io
 import logging
 import os
 import sys
+import urllib.parse
+from contextlib import contextmanager
+from dataclasses import dataclass
 from datetime import timedelta
 from netrc import NetrcParseError
-from pathlib import PurePath
-from typing import Mapping
+from pathlib import Path, PurePath
+from types import TracebackType
+from typing import BinaryIO, Iterator, Mapping, Protocol
 
 import httpx
-from httpx import HTTPStatusError, TimeoutException
+from httpx import HTTPStatusError, Request, Response, SyncByteStream, TimeoutException, codes
 from tenacity import (
     before_sleep_log,
     retry,
@@ -29,6 +34,7 @@
 from .errors import InputError
 from .hashing import Digest, ExpectedDigest, Fingerprint
 from .model import Url
+from .platform import Platform
 from .version import __version__
 
 logger = logging.getLogger(__name__)
@@ -94,7 +100,103 @@ def require_password(auth_type: str) -> str:
     return None
 
 
-def _configured_client(url: Url, headers: Mapping[str, str] | None = None) -> httpx.Client:
+class Client(Protocol):
+    def __enter__(self) -> Client: ...
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None = None,
+        exc_value: BaseException | None = None,
+        traceback: TracebackType | None = None,
+    ) -> None: ...
+
+    def get(self, url: Url) -> Response: ...
+
+    @contextmanager
+    def stream(self, method: str, url: Url) -> Iterator[Response]: ...
+
+
+class FileClient:
+    def __enter__(self) -> Client:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None = None,
+        exc_value: BaseException | None = None,
+        traceback: TracebackType | None = None,
+    ) -> None:
+        return None
+
+    @staticmethod
+    def _vet_request(url: Url, method: str = "GET") -> tuple[Request, Path] | Response:
+        request = Request(method=method, url=url)
+        if request.method not in ("GET", "HEAD"):
+            return Response(status_code=codes.METHOD_NOT_ALLOWED, request=request)
+
+        raw_path = urllib.parse.unquote_plus(url.info.path)
+        if Platform.current().is_windows:
+            # Handle `file:///C:/a/path` -> `C:/a/path`.
+            parts = raw_path.split("/")
+            if ":" == parts[1][-1]:
+                parts.pop(0)
+            path = Path("/".join(parts))
+        else:
+            path = Path(raw_path)
+
+        if not path.exists():
+            return Response(status_code=codes.NOT_FOUND, request=request)
+        if not path.is_file():
+            return Response(status_code=codes.BAD_REQUEST, request=request)
+        if not os.access(path, os.R_OK):
+            return Response(status_code=codes.FORBIDDEN, request=request)
+
+        return request, path
+
+    def get(self, url: Url) -> Response:
+        result = self._vet_request(url)
+        if isinstance(result, Response):
+            return result
+
+        request, path = result
+        content = path.read_bytes()
+        return httpx.Response(
+            status_code=codes.OK,
+            headers={"Content-Length": str(len(content))},
+            content=content,
+            request=request,
+        )
+
+    @dataclass(frozen=True)
+    class FileByteStream(SyncByteStream):
+        stream: BinaryIO
+
+        def __iter__(self) -> Iterator[bytes]:
+            return iter(lambda: self.stream.read(io.DEFAULT_BUFFER_SIZE), b"")
+
+        def close(self) -> None:
+            self.stream.close()
+
+    @contextmanager
+    def stream(self, method: str, url: Url) -> Iterator[httpx.Response]:
+        result = self._vet_request(url, method=method)
+        if isinstance(result, Response):
+            yield result
+            return
+
+        request, path = result
+        with path.open("rb") as fp:
+            yield httpx.Response(
+                status_code=codes.OK,
+                headers={"Content-Length": str(path.stat().st_size)},
+                stream=self.FileByteStream(fp),
+                request=request,
+            )
+
+
+def _configured_client(url: Url, headers: Mapping[str, str] | None = None) -> Client:
+    if "file" == url.info.scheme:
+        return FileClient()
     headers = dict(headers) if headers else {}
     headers.setdefault("User-Agent", f"insta-science/{__version__}")
     auth = _configure_auth(url) if "Authorization" not in headers else None
@@ -134,7 +236,9 @@ def _expected_digest(
 
     with _configured_client(url, headers) as client:
         return ExpectedDigest(
-            fingerprint=Fingerprint(client.get(f"{url}.{algorithm}").text.split(" ", 1)[0].strip()),
+            fingerprint=Fingerprint(
+                client.get(Url(f"{url}.{algorithm}")).text.split(" ", 1)[0].strip()
+            ),
             algorithm=algorithm,
         )
 

python/insta_science/_internal/platform.py

@@ -50,8 +50,7 @@ def current(cls) -> Platform:
                 return cls.Windows_x86_64
 
         raise InputError(
-            f"The current operating system / machine pair is not supported!: "
-            f"{system} / {machine}"
+            f"The current operating system / machine pair is not supported!: {system} / {machine}"
         )
 
     @property

python/insta_science/_internal/version.py

@@ -1,4 +1,4 @@
 # Copyright 2024 Science project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "0.4.5"
+__version__ = "0.5.0"

python/scripts/release.py

@@ -155,7 +155,7 @@ def check_version_and_changelog() -> Release | ReleaseError:
                 For example:
                 ------------
     
-                {'#' * RELEASE_HEADING_LEVEL} {__version__}
+                {"#" * RELEASE_HEADING_LEVEL} {__version__}
     
                 These are the {__version__} release notes...
                 """

python/tests/test_util.py

@@ -13,6 +13,8 @@
 import pytest
 from _pytest.monkeypatch import MonkeyPatch
 
+from insta_science import Platform
+
 
 @pytest.fixture(autouse=True)
 def cache_dir(monkeypatch: MonkeyPatch, tmp_path: Path) -> Path:
@@ -64,7 +66,7 @@ def read_data():
         process.kill()
 
 
-def test_download(pyproject_toml: Path, server: Server) -> None:
+def test_download_http_mirror(pyproject_toml: Path, server: Server) -> None:
     pyproject_toml.write_text(
         dedent(
             f"""\
@@ -77,3 +79,20 @@ def test_download(pyproject_toml: Path, server: Server) -> None:
 
     subprocess.run(args=["insta-science-util", "download", server.root], check=True)
     subprocess.run(args=["insta-science", "-V"], check=True)
+
+
+def test_download_file_mirror(pyproject_toml: Path, tmp_path: Path) -> None:
+    mirror_dir = tmp_path / "mirror"
+    mirror_url = f"file:{'///' if Platform.current().is_windows else '//'}{mirror_dir.as_posix()}"
+    pyproject_toml.write_text(
+        dedent(
+            f"""\
+            [tool.insta-science.science]
+            version = "0.9.0"
+            base-url = "{mirror_url}"
+            """
+        )
+    )
+
+    subprocess.run(args=["insta-science-util", "download", mirror_dir], check=True)
+    subprocess.run(args=["insta-science", "-V"], check=True)