ENH: containers-add: Try to link layers in OCI directory

TODO: Finalize approach in Datalad for Docker Registry URLs.

datalad_container/adapters/oci.py

@@ -196,6 +196,67 @@ def save(image, path):
     _store_annotation(path, _IMAGE_SOURCE_KEY, image)
 
 
+_ENDPOINTS = {"docker.io": "https://registry-1.docker.io/v2/"}
+
+
+def link(ds, path, reference):
+    """Add Docker registry URLs to annexed layer images.
+
+    Parameters
+    ----------
+    ds : Dataset
+    path : pathlib.Path
+        Absolute path to the image directory.
+    reference : str
+        Docker reference (e.g., "busybox:1.32"). This should not include the
+        transport (i.e. "docker://").
+    """
+    from datalad.downloaders.providers import Providers
+    from datalad.support.exceptions import CommandError
+    from datalad_container.utils import ensure_datalad_remote
+
+    res = sp.run(["skopeo", "inspect", "oci:" + str(path)],
+                 capture_output=True, universal_newlines=True, check=True)
+    info = json.loads(res.stdout)
+
+    ref = parse_docker_reference(reference, normalize=True)
+    registry, name = ref.name.split("/", maxsplit=1)
+    endpoint = _ENDPOINTS.get(registry)
+    if not endpoint:
+        lgr.debug("No known endpoint for %s. Skipping linking.", registry)
+        return
+    provider = Providers.from_config_files().get_provider(
+        endpoint + name, only_nondefault=True)
+    if not provider:
+        lgr.debug("Required Datalad provider configuration "
+                  "for Docker registry links not detected. Skipping linking.")
+        return
+
+    layers = {}  # path => digest
+    for layer in info["Layers"]:
+        algo, digest = layer.split(":")
+        layer_path = path / "blobs" / algo / digest
+        layers[layer_path] = layer
+
+    ds_repo = ds.repo
+    checked_dl_remote = False
+    for st in ds.status(layers.keys(), annex="basic", result_renderer=None):
+        if "keyname" in st:
+            if not checked_dl_remote:
+                ensure_datalad_remote(ds_repo)
+                checked_dl_remote = True
+            path = Path(st["path"])
+            url = "{}{}/blobs/{}".format(endpoint, name, layers[path])
+            try:
+                ds_repo.add_url_to_file(
+                    path, url, batch=True, options=['--relaxed'])
+            except CommandError as exc:
+                lgr.warning("Registering %s with %s failed: %s",
+                            path, url, exc)
+        else:
+            lgr.warning("Skipping non-annexed layer: %s", st["path"])
+
+
 def get_image_id(path):
     """Return a directory's image ID.
 

datalad_container/adapters/tests/test_oci_more.py

@@ -17,11 +17,16 @@
     StdOutErrCapture,
     WitlessRunner,
 )
+from datalad.consts import (
+    DATALAD_SPECIAL_REMOTE,
+    DATALAD_SPECIAL_REMOTES_UUIDS,
+)
 from datalad_container.adapters import oci
 from datalad_container.adapters.utils import get_docker_image_ids
 from datalad.tests.utils import (
     assert_in,
     integration,
+    ok_,
     skip_if_no_network,
     SkipTest,
     slow,
@@ -54,3 +59,21 @@ def test_oci_add_and_run(path):
         if not existed:
             WitlessRunner().run(["docker", "rmi", image_id])
     assert_in("BusyBox v1.30", out["stdout"])
+
+    from datalad.downloaders.providers import Providers
+    if not Providers.from_config_files().get_provider(
+            "https://registry-1.docker.io/v2/library",
+            only_nondefault=True):
+        # The rest of the test is about Docker Hub registry links, which
+        # require provider configuration for authentication.
+        return
+
+    layers = [r["path"]
+              for r in ds.status(image_path / "blobs", annex="basic",
+                                 result_renderer=None)
+              if "key" in r]
+    ok_(layers)
+
+    dl_uuid = DATALAD_SPECIAL_REMOTES_UUIDS[DATALAD_SPECIAL_REMOTE]
+    for where_res in ds.repo.whereis(list(map(str, layers))):
+        assert_in(dl_uuid, where_res)

datalad_container/containers_add.py

@@ -319,3 +319,7 @@ def __call__(name, url=None, dataset=None, call_fmt=None, image=None,
             yield r
         result["status"] = "ok"
         yield result
+
+        # We need to do this after the image is saved.
+        if url and url.startswith("oci:docker://"):
+            oci.link(ds, Path(image), url[len("oci:docker://"):])