nydus: support host-sharing

.github/workflows/tests.yml

@@ -171,6 +171,54 @@ jobs:
           sleep 5
           echo "Knative test succesful!"
 
+      - name: "Run nydus host-share test"
+        run: |
+          # Change the snapshotter mode
+          ./bin/inv_wrapper.sh nydus-snapshotter.set-mode host-share
+
+          export SC2_RUNTIME_CLASS=qemu-${{ matrix.tee }}-sc2
+          export POD_LABEL="apps.sc2.io/name=helloworld-py"
+
+          # ----- Python Test ----
+
+          echo "Running python test..."
+          envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl apply -f -
+
+          # Wait for pod to be ready
+          until [ "$(./bin/kubectl get pods -l ${POD_LABEL} -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; do echo "Waiting for pod to be ready..."; sleep 2; done
+          sleep 1
+
+          # Get the pod's IP
+          service_ip=$(./bin/kubectl get services -o jsonpath='{.items[?(@.metadata.name=="coco-helloworld-py-node-port")].spec.clusterIP}')
+          [ "$(curl --retry 3 -X GET ${service_ip}:8080)" = "Hello World!" ]
+          envsubst < ./demo-apps/helloworld-py-nydus/deployment.yaml | ./bin/kubectl delete -f -
+
+          # Wait for pod to be deleted
+          ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=30s
+
+          # Extra cautionary sleep
+          sleep 5
+          echo "Python test succesful!"
+
+          # ----- Knative Test ----
+          envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl apply -f -
+          sleep 1
+
+          # Get the service URL
+          service_url=$(./bin/kubectl get ksvc helloworld-knative  --output=custom-columns=URL:.status.url --no-headers)
+          [ "$(curl --retry 3 ${service_url})" = "Hello World!" ]
+
+          # Wait for pod to be deleted
+          envsubst < ./demo-apps/helloworld-knative-nydus/service.yaml | ./bin/kubectl delete -f -
+          ./bin/kubectl wait --for=delete -l ${POD_LABEL} pod --timeout=60s
+
+          # Extra cautionary sleep
+          sleep 5
+          echo "Knative test succesful!"
+
+          # Change the snapshotter mode back again
+          ./bin/inv_wrapper.sh nydus-snapshotter.set-mode guest-pull
+
       - name: "Enable default-memory annotation"
         run: |
           for runtime_class in ${{ matrix.runtime_classes }}; do
@@ -182,8 +230,7 @@ jobs:
 
           # Aftre changing the annotation of the qemu-snp-sc2 runtime class we
           # need to restart the VM cache
-          sudo -E ./vm-cache/target/release/vm-cache stop
-          sudo -E ./vm-cache/target/release/vm-cache background
+          sudo -E ./vm-cache/target/release/vm-cache restart
 
       - name: "Run knative chaining demo"
         run: |

README.md

@@ -81,6 +81,7 @@ For further documentation, you may want to check these other documents:
 * [CoCo Upgrade](./docs/upgrade_coco.md) - upgrade the current CoCo version.
 * [Guest Components](./docs/guest_components.md) - instructions to patch components inside SC2 guests.
 * [Host Kernel](./docs/host_kernel.md) - bump the kernel version in the host.
+* [Image Pull](./docs/image_pull.md) - details on the image-pulling mechanisms supported in SC2.
 * [K8s](./docs/k8s.md) - documentation about configuring a single-node Kubernetes cluster.
 * [Kata](./docs/kata.md) - instructions to build our custom Kata fork and `initrd` images.
 * [Key Broker Service](./docs/kbs.md) - docs on using and patching the KBS.

docs/image_pull.md

@@ -0,0 +1,54 @@
+## Image Pull
+
+This document describes the different mechanisms to get a container image
+inside a cVM in SC2. We _always_ assume that the integrity of container images
+must be validated. We also consider the situation in which their confidentiality
+must also be preserved.
+
+### Guest Pull
+
+The guest pull mechanism always pulls the container image inside the guest cVM.
+This is the default mechanism in CoCo as it allows the most secure, and simplest
+deployment: users sign (and encrypt) container images locally, they upload
+them to a container registry, pull them inside the cVM, and decrypt them inside
+the cVM.
+
+Albeit secure, this mechanism has high performance overheads as the image must
+be pulled every single time, precluding any caching benefits.
+
+To mitigate the performance overheads, we can convert the OCI image to a
+Nydus image, that supports lazy loading of container data.
+
+### Host Share
+
+The host share mechanism mounts a container image from the host to the guest.
+Given that the host is untrusted, this mechanism only works for images that
+do not have confidentiality requirements. To maintain integrity, we mount
+the image with `dm-verity`, and validate the `dm-verity` device as part of
+attestation.
+
+We choose to mount individual layers separately (rather than whole images),
+but we should measure that the former is actually better than the latter.
+
+We could mount encrypted images from the host to the guest, but we would be
+losing on the de-duplication opportunities in the host.
+
+### Usage
+
+Each image pull mechanism is implemented as a different remote snapshotter
+in containerd, all of them based on the [nydus-snapshotter](
+https://github.com/containerd/nydus-snapshotter/) plus our modifications.
+
+To switch between different image-pulling mechanisms, you only need to change
+the snapshotter mode:
+
+```bash
+inv nydus-snapshotter.set-mode [guest-pull,host-share]
+```
+
+If you see any snapshotter related issues (either in the `containerd` or the
+`nydus-snapshotter` journal logs), you can purge the snapshotters:
+
+```bash
+inv nydus-snapshotter.purge
+```

docs/troubleshooting.md

@@ -151,14 +151,27 @@ ctr -n k8s.io content fetch ${IMAGE_NAME}
 the image name is the image tag appearing right before the error message in
 the pod logs.
 
-### Nydus snapshot corruption
+### Rootfs Mount Issue
 
-Sometimes, after hot-replacing the nydus-snapshotter, snapshots become corrupted,
-and we can see the error below.
+Sometimes, if we are mixing and matching different snapshotters, we may run
+into the following error:
 
 ```
 Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create containerd task: failed to create shim task: failed to mount /run/kata-containers/shared/containers/0a583f0691d78e2036425f99bdac8e03302158320c1c55a5c6482cae7e729009/rootfs to /run/kata-containers/0a583f0691d78e2036425f99bdac8e03302158320c1c55a5c6482cae7e729009/rootfs, with error: ENOENT: No such file or directory
 ```
 
-The only solution I found was to bump to a more up-to-date version of nydus.
-This seemed to fix the issue.
+this is because the pause image bundle has not been unpacked correctly. Note
+that the pause image bundle is unpacked into the `/run/kata-containers/shared`
+directory, and then mounted into the `/run/kata-containers/<cid>` one.
+
+This usually happens when containerd believes that we already have the pause
+image, so we do not need to pull it. This prevents the snapshotter from
+generating the respective Kata virtual volumes.
+
+As a rule of thumb, a good fix is to remove all images involved in the app
+from the content store, and purge snapshotter caches:
+
+```bash
+sudo crictl rmi <hash>
+inv nydus-snapshotter.purge
+```

tasks/containerd.py

@@ -31,23 +31,24 @@
 CONTAINERD_HOST_BINPATH = "/usr/bin"
 
 
-def do_build(debug=False):
-    docker_cmd = "docker build -t {} -f {} .".format(
+def do_build(nocache=False):
+    docker_cmd = "docker build{} -t {} -f {} .".format(
+        " --no-cache" if nocache else "",
         CONTAINERD_IMAGE_TAG,
         join(PROJ_ROOT, "docker", "containerd.dockerfile"),
     )
-    result = run(docker_cmd, shell=True, capture_output=True, cwd=PROJ_ROOT)
-    assert result.returncode == 0, print(result.stderr.decode("utf-8").strip())
-    if debug:
-        print(result.stdout.decode("utf-8").strip())
+    run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT)
 
 
 @task
-def build(ctx):
+def build(ctx, nocache=False, push=False):
     """
     Build the containerd fork for CoCo
     """
-    do_build(debug=True)
+    do_build(nocache=nocache)
+
+    if push:
+        run(f"docker push {CONTAINERD_IMAGE_TAG}", shell=True, check=True)
 
 
 @task
@@ -73,19 +74,23 @@ def cli(ctx, mount_path=join(PROJ_ROOT, "..", "containerd")):
 
 
 @task
-def set_log_level(ctx, log_level):
+def stop(ctx):
     """
-    Set containerd's log level, must be one in: info, debug
+    Stop the containerd work-on container
     """
-    allowed_log_levels = ["info", "debug"]
-    if log_level not in allowed_log_levels:
-        print(
-            "Unsupported log level '{}'. Must be one in: {}".format(
-                log_level, allowed_log_levels
-            )
-        )
-        return
+    result = run(
+        "docker rm -f {}".format(CONTAINERD_CTR_NAME),
+        shell=True,
+        check=True,
+        capture_output=True,
+    )
+    assert result.returncode == 0
+
 
+def set_log_level(log_level):
+    """
+    Set containerd's log level, must be one in: info, debug
+    """
     updated_toml_str = """
     [debug]
     level = "{log_level}"

tasks/kata.py

@@ -67,20 +67,10 @@ def stop(ctx):
     stop_kata_workon_ctr()
 
 
-@task
-def set_log_level(ctx, log_level):
+def set_log_level(log_level):
     """
     Set kata's log level, must be one in: info, debug
     """
-    allowed_log_levels = ["info", "debug"]
-    if log_level not in allowed_log_levels:
-        print(
-            "Unsupported log level '{}'. Must be one in: {}".format(
-                log_level, allowed_log_levels
-            )
-        )
-        return
-
     enable_debug = str(log_level == "debug").lower()
 
     for runtime in KATA_RUNTIMES + SC2_RUNTIMES:
@@ -146,6 +136,7 @@ def hot_replace_shim(ctx, runtime="qemu-snp-sc2"):
             ),
         ),
         sc2=runtime in SC2_RUNTIMES,
+        hot_replace=True,
     )
 
     restart_containerd()

tasks/kernel.py

@@ -54,8 +54,10 @@ def build_guest(debug=False, hot_replace=False):
             ctr_path, host_path, sudo=False, debug=debug, hot_replace=hot_replace
         )
 
+    # The -V option enables dm-verity support in the guest (technically only
+    # needed for SC2)
     build_kernel_base_cmd = [
-        f"./build-kernel.sh -x -f -v {GUEST_KERNEL_VERSION}",
+        f"./build-kernel.sh -x -V -f -v {GUEST_KERNEL_VERSION}",
         "-u 'https://cdn.kernel.org/pub/linux/kernel/v{}.x/'".format(
             GUEST_KERNEL_VERSION.split(".")[0]
         ),
@@ -117,4 +119,7 @@ def build_guest(debug=False, hot_replace=False):
 
 @task
 def hot_replace_guest(ctx, debug=False):
+    """
+    Hot-replace guest kernel
+    """
     build_guest(debug=debug, hot_replace=True)

tasks/nydus.py

@@ -2,7 +2,7 @@
 from os.path import join
 from subprocess import run
 from tasks.util.docker import copy_from_ctr_image
-from tasks.util.env import GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line
+from tasks.util.env import COCO_ROOT, GHCR_URL, GITHUB_ORG, PROJ_ROOT, print_dotted_line
 from tasks.util.nydus import NYDUSIFY_PATH
 from tasks.util.versions import NYDUS_VERSION
 
@@ -27,12 +27,20 @@ def build(ctx, nocache=False, push=False):
 
 
 def do_install():
-    print_dotted_line(f"Installing nydusify (v{NYDUS_VERSION})")
+    print_dotted_line(f"Installing nydus image services (v{NYDUS_VERSION})")
 
+    # Non root-owned binaries
     ctr_bin = ["/go/src/github.com/sc2-sys/nydus/contrib/nydusify/cmd/nydusify"]
     host_bin = [NYDUSIFY_PATH]
     copy_from_ctr_image(NYDUS_IMAGE_TAG, ctr_bin, host_bin, requires_sudo=False)
 
+    # Root-owned binaries
+    # The host-pull functionality requires nydus-image >= 2.3.0, but the one
+    # installed with the daemon is 2.2.4
+    ctr_bin = ["/go/src/github.com/sc2-sys/nydus/target/release/nydus-image"]
+    host_bin = [join(COCO_ROOT, "bin", "nydus-image")]
+    copy_from_ctr_image(NYDUS_IMAGE_TAG, ctr_bin, host_bin, requires_sudo=True)
+
     print("Success!")