From bc17aa181f55fae0620abe3696c43bfadb2e2a1e Mon Sep 17 00:00:00 2001 From: Tuomo Tanskanen Date: Wed, 10 Apr 2024 10:53:54 +0300 Subject: [PATCH] add container signing POC Proof-of-concept focusing on enabling container signing with any signing backend with external signer plugin. 1. Add Notation External Signer plugin and sign images with that 2. Validate the Signature runtime with Kyverno admission control Signed-off-by: Tuomo Tanskanen --- security/container-signing-poc/Makefile | 24 ++ security/container-signing-poc/README.md | 64 +++++ .../container-signing-poc/kyverno/Makefile | 105 ++++++++ .../container-signing-poc/kyverno/README.md | 226 ++++++++++++++++++ .../kyverno/policy/kyverno-policy.yaml | 55 +++++ .../kyverno/scripts/kind-cluster.sh | 65 +++++ .../container-signing-poc/notation/.gitignore | 27 +++ .../container-signing-poc/notation/LICENSE | 201 ++++++++++++++++ .../container-signing-poc/notation/Makefile | 83 +++++++ .../container-signing-poc/notation/README.md | 211 ++++++++++++++++ .../cmd/notation-external-signer/.gitignore | 1 + .../cmd/notation-external-signer/key.go | 46 ++++ .../cmd/notation-external-signer/main.go | 78 ++++++ .../cmd/notation-external-signer/metadata.go | 28 +++ .../cmd/notation-external-signer/sign.go | 144 +++++++++++ .../notation/examples/.gitignore | 4 + .../examples/read-signature-from-file.sh | 30 +++ .../notation/examples/rsassa-pss-sha512.sh | 16 ++ .../notation/examples/trustpolicy.json | 20 ++ .../container-signing-poc/notation/go.mod | 8 + .../container-signing-poc/notation/go.sum | 5 + .../container-signing-poc/oras/.gitignore | 2 + security/container-signing-poc/oras/Makefile | 62 +++++ security/container-signing-poc/oras/README.md | 108 +++++++++ 24 files changed, 1613 insertions(+) create mode 100644 security/container-signing-poc/Makefile create mode 100644 security/container-signing-poc/README.md create mode 100644 security/container-signing-poc/kyverno/Makefile create mode 100644 security/container-signing-poc/kyverno/README.md create mode 100644 security/container-signing-poc/kyverno/policy/kyverno-policy.yaml create mode 100755 security/container-signing-poc/kyverno/scripts/kind-cluster.sh create mode 100644 security/container-signing-poc/notation/.gitignore create mode 100644 security/container-signing-poc/notation/LICENSE create mode 100644 security/container-signing-poc/notation/Makefile create mode 100644 security/container-signing-poc/notation/README.md create mode 100644 security/container-signing-poc/notation/cmd/notation-external-signer/.gitignore create mode 100644 security/container-signing-poc/notation/cmd/notation-external-signer/key.go create mode 100644 security/container-signing-poc/notation/cmd/notation-external-signer/main.go create mode 100644 security/container-signing-poc/notation/cmd/notation-external-signer/metadata.go create mode 100644 security/container-signing-poc/notation/cmd/notation-external-signer/sign.go create mode 100644 security/container-signing-poc/notation/examples/.gitignore create mode 100755 security/container-signing-poc/notation/examples/read-signature-from-file.sh create mode 100755 security/container-signing-poc/notation/examples/rsassa-pss-sha512.sh create mode 100644 security/container-signing-poc/notation/examples/trustpolicy.json create mode 100644 security/container-signing-poc/notation/go.mod create mode 100644 security/container-signing-poc/notation/go.sum create mode 100644 security/container-signing-poc/oras/.gitignore create mode 100644 security/container-signing-poc/oras/Makefile create mode 100644 security/container-signing-poc/oras/README.md diff --git a/security/container-signing-poc/Makefile b/security/container-signing-poc/Makefile new file mode 100644 index 00000000..fbdcbb0f --- /dev/null +++ b/security/container-signing-poc/Makefile @@ -0,0 +1,24 @@ +# top level makefile to run e2e test +# notation -> oras copy -> kyverno + +NOTATION_DIR := notation +ORAS_DIR := oras +KYVERNO_DIR := kyverno + +SHELL := /bin/bash + +.PHONY: all e2e clean + +all: + @echo "targets: e2e clean" + +e2e: + make -C $(NOTATION_DIR) e2e + make -C $(ORAS_DIR) e2e + make -C $(KYVERNO_DIR) e2e + @echo "e2e test done!" + +clean: + make -C $(NOTATION_DIR) clean-e2e + make -C $(ORAS_DIR) clean-e2e + make -C $(KYVERNO_DIR) clean-e2e diff --git a/security/container-signing-poc/README.md b/security/container-signing-poc/README.md new file mode 100644 index 00000000..e20fc8af --- /dev/null +++ b/security/container-signing-poc/README.md @@ -0,0 +1,64 @@ +# Container Signing POC + +This experiment is two-fold: + +1. [Signing images with Notation, especially with custom signers](notation/README.md) +1. [Moving signatures from one registry to another](oras/README.md) +1. [Validating the signing with Kyverno, in a Kind cluster](kyverno/README.md) + +## TL;DR of the POC + +Basically, the POC is finding the following: + +1. Notation is extendable with their plugin system +1. Plugin that can call any external script or binary to produce a signature has + been implemented. +1. This allows any in-house, custom integration to private signer, regardless + of the interface, even manual/email works (despite being brittle), without + writing a full-fledged plugin with Go. +1. Kyverno can easily be configured to verify Notation signatures runtime, via + their admission controller and pluggable policies. +1. Oras can be used to move containers and signatures from CI to production + +## e2e test + +End-to-end test for this POC can be run with `make e2e` from this directory. +This does the following: + +1. Build and install Notation plugin +1. Sign busybox image on local registry at port `5002` +1. Copy container and signature to another registry at port `5001` +1. Launch Kind cluster, install Kyverno and add ClusterPolicy for signature + verification, and then run workloads that pass the verification and workloads + that fail the verification to check, if the signing works e2e + +End-to-end test will use the same certificates and same signature through the +whole chain for verify end-to-end functionality. + +## Notes + +### Notes on Notation + +Notary V1 was part of TUF (The Update Framework) organization, yet separate +project in CNCF. Notary V2 has been rebranded as Notation to make the branding +clearer that it is based on different principles than Notary V1. Notary V1/TUF +has more strict "security principles", which also made it hard to use, and it +did not gain traction. + +It should be noted that Notary V1 and Notary V2/Notation are completely different +projects, with claims of [hostile takeover](https://github.com/cncf/toc/issues/981) +by the new maintainers (Microsoft/Docker). CNCF TOC did not see it that way, +but funded a +[security audit](https://www.cncf.io/blog/2023/07/11/announcing-results-of-notation-security-audit-2023/) +for Notation, with no critical findings, but the argument from the Notary V1 +folks is that the foundation of Notation security model is not good enough. This +design aspect was not part of the audit based on the findings. + +### Notes on Kyverno + +[Kyverno Image Signature verification](https://kyverno.io/docs/writing-policies/verify-images/) + is in beta. + +Kyverno can also verify Sigstore Cosign signatures. + +Kyverno is generic policy engine, capable of replacing OPA Gatekeeper etc. diff --git a/security/container-signing-poc/kyverno/Makefile b/security/container-signing-poc/kyverno/Makefile new file mode 100644 index 00000000..00225804 --- /dev/null +++ b/security/container-signing-poc/kyverno/Makefile @@ -0,0 +1,105 @@ +# build and install notation-external-signer + +SCRIPTS_DIR := scripts +POLICY_DIR := policy + +NOTATION_DIR := ../notation +EXAMPLES_DIR := $(NOTATION_DIR)/examples +NOTATION_SIGNER := $(EXAMPLES_DIR)/rsassa-pss-sha512.sh + +TEST_REGISTRY := 127.0.0.1:5001 +TEST_REGISTRY_NAME := kind-registry +TEST_REGISTRY_IMAGE := registry:2 +TEST_IMAGE_UNSIGNED := busybox:1.36.0-glibc +TEST_IMAGE_SIGNED := busybox:1.36.1-glibc +TEST_DIGEST := sha256:db16cd196b8a37ba5f08414e6f6e71003d76665a5eac160cb75ad3759d8b3e29 + +CLUSTER_REGISTRY := 172.19.0.3:5000 + +SHELL := /bin/bash + +.PHONY: all test setup check-tools sign clean clean-tests + +all: check-tools + @echo "targets: test clean" + +test: check-tools setup sign tests +e2e: check-tools setup-e2e tests + +.PHONY: tests test-pod test-deployment + +setup: setup-registry create-cluster install-kyverno install-notation-plugin certificates +setup-e2e: create-cluster install-kyverno + +.PHONY: setup-registry create-cluster install-kyverno install-notation-plugin install-policy certificates + +check-tools: + @type -a helm &>/dev/null || echo "error: Install helm: https://helm.sh/docs/intro/install/" + @type -a docker &>/dev/null || echo "error: Install docker: https://docs.docker.com/engine/install/" + @type -a kind &>/dev/null || echo "error: Install kind: https://kind.sigs.k8s.io/docs/user/quick-start/" + @type -a notation &>/dev/null || echo "error: Install notation: https://notaryproject.dev/docs/user-guides/installation/cli/" + +setup-registry: + docker run -d -p $(TEST_REGISTRY):5000 --name $(TEST_REGISTRY_NAME) $(TEST_REGISTRY_IMAGE) + for image in $(TEST_IMAGE_UNSIGNED) $(TEST_IMAGE_SIGNED); do \ + docker pull $${image}; \ + docker tag $${image} $(TEST_REGISTRY)/$${image}; \ + docker push $(TEST_REGISTRY)/$${image}; \ + done + +create-cluster: + ./scripts/kind-cluster.sh + +install-kyverno: + helm repo add kyverno https://kyverno.github.io/kyverno/ + helm repo update + helm install kyverno kyverno/kyverno -n kyverno --create-namespace + sleep 60 + # NOTE: we need to edit Kyverno config to allow insecure registries + kubectl -n kyverno get deployment kyverno-admission-controller -o yaml | \ + sed -e 's/allowInsecureRegistry=false/allowInsecureRegistry=true/' | \ + kubectl apply -f - + sleep 30 + +install-notation-plugin: + make -C $(NOTATION_DIR) install + +certificates: + make -C $(NOTATION_DIR) certificates + +install-policy: + # replace example cert with the generated certs + cat $(POLICY_DIR)/kyverno-policy.yaml | \ + sed -re '/-----BEGIN/,/END CERTIFICATE-----/d' | \ + { cat -; cat $(EXAMPLES_DIR)/ca.crt | sed -e 's/^/ /g'; } | \ + kubectl apply -f - + sleep 30 + +sign: + make -C $(NOTATION_DIR) sign TEST_IMAGE_SIGN=$(TEST_REGISTRY)/$(TEST_IMAGE_SIGNED) + +tests: install-policy test-pod test-deployment + sleep 5 + kubectl get pods -A + @echo "Success (if only pods with success are visible - ignore the ImagePull issues)" + +test-pod: + kubectl run --image $(CLUSTER_REGISTRY)/$(TEST_IMAGE_UNSIGNED) pod-fail || true + kubectl run --image $(CLUSTER_REGISTRY)/$(TEST_IMAGE_SIGNED) pod-success + +test-deployment: + kubectl create deployment --image $(CLUSTER_REGISTRY)/$(TEST_IMAGE_UNSIGNED) deployment-fail || true + kubectl create deployment --image $(CLUSTER_REGISTRY)/$(TEST_IMAGE_SIGNED) deployment-success + +clean-tests: + -kubectl delete pod pod-fail + -kubectl delete deployment deployment-fail + -kubectl delete pod pod-success + -kubectl delete deployment deployment-success + +clean: clean-e2e + make -C $(NOTATION_DIR) clean + +clean-e2e: + -docker rm -f $(TEST_REGISTRY_NAME) + -kind delete cluster diff --git a/security/container-signing-poc/kyverno/README.md b/security/container-signing-poc/kyverno/README.md new file mode 100644 index 00000000..7957d1a9 --- /dev/null +++ b/security/container-signing-poc/kyverno/README.md @@ -0,0 +1,226 @@ + + +# Image signature verification + +Verifying image signatures runtime using Notation and Kyverno. + +```mermaid +flowchart TB + user(User) + cert(CA public root cert) + registry(Docker registry) + kyverno(Kyverno admission controller) + policy(Kyverno ClusterPolicy) + cluster(k8s cluster) + workload(Workload) + + user-->|load image and signature|registry + user-->|install|kyverno + user-->|apply workload|workload + user-->|configures|policy + policy-->|verifies signature with|kyverno + cert-->|configured in|policy + registry-->|create from|workload + workload-->|runs in|cluster + kyverno-->|admits|workload + kyverno-->|reads signature from|registry +``` + +## Non-runtime verification + +Verifying signatures with Notation "offline", please see +[Notation External Signer documentation](../notation/README.md). + +## Runtime verification + +For runtime signature verification, we use +[Kyverno with Notation](https://kyverno.io/docs/writing-policies/verify-images/notary/). + +`make test` tests the full setup with Kind cluster, local registry, Notation +external signer plugin etc. `make clean` to remove everything. + +Run `make` to get all meaningful make targets, and check the Makefile itself +for all targets. + +### Steps explained + +1. Run Kind cluster with [local registry](https://kind.sigs.k8s.io/docs/user/local-registry/) + + ```bash + ./scripts/kind-cluster.sh + ``` + +1. Load images for testing + + ```bash + # load busybox 1.36.1-glibc image for testing, signed + docker pull busybox:1.36.1-glibc + docker tag busybox:1.36.1-glibc 127.0.0.1:5001/busybox:1.36.1-glibc + docker push 127.0.0.1:5001/busybox:1.36.1-glibc + + # load busybox 1.36.0-glibc image for testing, not signed + docker pull busybox:1.36.0-glibc + docker tag busybox:1.36.0-glibc 127.0.0.1:5001/busybox:1.36.0-glibc + docker push 127.0.0.1:5001/busybox:1.36.0-glibc + ``` + +1. Sign image with Notation + + Sign image with Notation and Custom plugin. [See here](../notation/README.md). + + ```bash + $ make certificates + $ export EXTERNAL_CERT_CHAIN=$(pwd)/examples/certificate_chain.pem + $ export EXTERNAL_PRIVATE_KEY=$(pwd)/examples/leaf.key + $ export EXTERNAL_SIGNER=../notation/examples/rsassa-pss-sha512.sh + $ notation sign --insecure-registry --id "anything" --plugin "external-signer" 127.0.0.1:5001/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7 + Successfully signed 127.0.0.1:5001/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7 + ``` + + When finished, you should be able to inspect the signatures. + + ```bash + notation inspect 127.0.0.1:5001/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7 + ``` + +1. Run Kyverno + + If you don't have Helm installed, see + [here](https://helm.sh/docs/intro/install/). + + ```bash + helm repo add kyverno https://kyverno.github.io/kyverno/ + helm repo update + # check available versions and configurations + helm search repo kyverno -l + # install suitable one + helm install kyverno kyverno/kyverno -n kyverno --create-namespace + # NOTE: we need to edit Kyverno config to allow insecure registries + kubectl -n kyverno edit deployment kyverno-admission-controller + # change --allowInsecureRegistry=false to true + ``` + +1. Add Kyverno policy for image signatures + + ```bash + # NOTE: this has CA cert you need to replace with your signing CA cert + # NOTE: in Kind, the local registry is running as 172.19.0.3:5000 + kubectl apply -n kyverno -f examples/kyverno-policy.yaml + ``` + +1. Test the policy with Pod + + ```bash + # signed busybox will pass + $ kubectl run test --image=172.19.0.3:5000/busybox:1.36.1-glibc --dry-run=server + pod/test created (server dry run) + ``` + + with following logging in Kyverno admission controller: + + ```console + I0206 13:39:12.390636 1 event_broadcaster.go:338] "Event occurred" object="check-image-notary" kind="ClusterPolicy" apiVersion="kyverno.io/v1" type="Warning" reason="PolicyViolation" action="Resource Blocked" note="Pod default/test: [verify-signature-notary] fail (blocked); failed to verify image 172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: .attestors[0].entries[0]: failed to parse image reference: 172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: HEAD http://172.19.0.3:5000/v2/busybox/manifests/sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: unexpected status code 404 Not Found (HEAD responses have no body, use GET for details)" + I0206 13:41:31.383870 1 imageverifier.go:261] "cache entry found" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Pod" new.namespace="default" new.name="test" rule.name="verify-signature-notary" namespace="" policy="check-image-notary" ruleName="verify-signature-notary" imageRef="172.19.0.3:5000/busybox:1.36.1-glibc" + I0206 13:41:31.389427 1 event_broadcaster.go:338] "Event occurred" object="check-image-notary" kind="ClusterPolicy" apiVersion="kyverno.io/v1" type="Normal" reason="PolicyApplied" action="Resource Passed" note="Pod default/test: pass" + I0206 13:41:31.396586 1 validation.go:108] "validation passed" logger="webhooks.resource.validate" gvk="/v1, Kind=Pod" gvr={"group":"","version":"v1","resource":"pods"} namespace="default" name="test" operation="CREATE" uid="dd4f5ed5-dd0b-46d8-8b5c-642f4ae97857" user={"username":"kubernetes-admin","groups":["system:masters","system:authenticated"]} roles=null clusterroles=["cluster-admin","system:basic-user","system:discovery","system:public-info-viewer"] resource.gvk="/v1, Kind=Pod" kind="Pod" action="validate" resource="default/Pod/test" operation="CREATE" gvk="/v1, Kind=Pod" policy="check-image-notary" + I0206 13:41:31.396703 1 event_broadcaster.go:338] "Event occurred" object="check-image-notary" kind="ClusterPolicy" apiVersion="kyverno.io/v1" type="Normal" reason="PolicyApplied" action="Resource Passed" note="Pod default/test: pass" + ``` + + ```bash + # non-signed busybox will not pass + $ kubectl run test --image=172.19.0.3:5000/busybox:1.36.0-glibc --dry-run=server + Error from server: admission webhook "mutate.kyverno.svc-fail" denied the request: + + resource Pod/default/test was blocked due to the following policies + + check-image-notary: + verify-signature-notary: 'failed to verify image 172.19.0.3:5000/busybox:1.36.0-glibc: + .attestors[0].entries[0]: failed to verify 172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: + no signature is associated with "172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0", + make sure the artifact was signed successfully' + + ``` + + with following logging in Kyverno admission controller: + + ```console + I0206 13:41:41.401723 1 imageverifier.go:265] "cache entry not found" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Pod" new.namespace="default" new.name="test" rule.name="verify-signature-notary" namespace="" policy="check-image-notary" ruleName="verify-signature-notary" imageRef="172.19.0.3:5000/busybox:1.36.0-glibc" + I0206 13:41:41.401737 1 imageverifier.go:321] "verifying image signatures" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Pod" new.namespace="default" new.name="test" rule.name="verify-signature-notary" image="172.19.0.3:5000/busybox:1.36.0-glibc" attestors=1 attestations=0 + I0206 13:41:41.401911 1 notary.go:44] "verifying image" logger="Notary" reference="172.19.0.3:5000/busybox:1.36.0-glibc" + I0206 13:41:41.407184 1 imageverifier.go:498] "image attestors verification failed" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Pod" new.namespace="default" new.name="test" rule.name="verify-signature-notary" verifiedCount=0 requiredCount=1 errors=".attestors[0].entries[0]: failed to verify 172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: no signature is associated with \"172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0\", make sure the artifact was signed successfully" + E0206 13:41:41.407214 1 imageverifier.go:360] "failed to verify image" err=".attestors[0].entries[0]: failed to verify 172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: no signature is associated with \"172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0\", make sure the artifact was signed successfully" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Pod" new.namespace="default" new.name="test" rule.name="verify-signature-notary" + I0206 13:41:41.413149 1 block.go:29] "blocking admission request" logger="webhooks.resource.mutate" gvk="/v1, Kind=Pod" gvr={"group":"","version":"v1","resource":"pods"} namespace="default" name="test" operation="CREATE" uid="428db8d9-9268-4fef-916f-6d0d653f2843" user={"username":"kubernetes-admin","groups":["system:masters","system:authenticated"]} roles=null clusterroles=["cluster-admin","system:basic-user","system:discovery","system:public-info-viewer"] resource.gvk="/v1, Kind=Pod" kind="Pod" policy="check-image-notary" + E0206 13:41:41.413203 1 handlers.go:173] "image verification failed" err=< + ``` + +1. Test the policy with Deployment + +Same as with Pod, the failure is immediate, and not delayed until the Pod would +created by the Deployment. + +Unsigned image: + +```shell +$ kubectl create deployment --image 172.19.0.3:5000/busybox:1.36.0-glibc busybox +error: failed to create deployment: admission webhook "mutate.kyverno.svc-fail" denied the request: + +resource Deployment/default/busybox was blocked due to the following policies + +check-image-notary: + autogen-verify-signature-notary: 'failed to verify image 172.19.0.3:5000/busybox:1.36.0-glibc: + .attestors[0].entries[0]: failed to verify 172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: + no signature is associated with "172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0", + make sure the artifact was signed successfully' +``` + +Kyverno logs with failure: + +```console +I0213 10:03:30.428913 1 event_broadcaster.go:338] "Event occurred" object="check-image-notary" kind="ClusterPolicy" apiVersion="kyverno.io/v1" type="Warning" reason="PolicyViolation" action="Resource Blocked" note="Deployment default/busybox: [autogen-verify-signature-notary] fail (blocked); failed to verify image 172.19.0.3:5000/busybox:1.36.1-glibc: .attestors[0].entries[0]: failed to parse image reference: 172.19.0.3:5000/busybox:1.36.1-glibc: HEAD http://172.19.0.3:5000/v2/busybox/manifests/1.36.1-glibc: unexpected status code 404 Not Found (HEAD responses have no body, use GET for details)" +I0213 10:03:30.428937 1 event_broadcaster.go:338] "Event occurred" object="check-image-notary" kind="ClusterPolicy" apiVersion="kyverno.io/v1" type="Warning" reason="PolicyViolation" action="Resource Blocked" note="Deployment default/busybox: [autogen-verify-signature-notary] error (blocked); failed to update digest: failed to fetch image reference: 172.19.0.3:5000/busybox:1.36.1-glibc, error: GET http://172.19.0.3:5000/v2/busybox/manifests/1.36.1-glibc: MANIFEST_UNKNOWN: manifest unknown; map[Tag:1.36.1-glibc]" +I0213 10:06:16.332751 1 imageverifier.go:265] "cache entry not found" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Deployment" new.namespace="default" new.name="busybox" rule.name="autogen-verify-signature-notary" namespace="" policy="check-image-notary" ruleName="autogen-verify-signature-notary" imageRef="172.19.0.3:5000/busybox:1.36.0-glibc" +I0213 10:06:16.332771 1 imageverifier.go:321] "verifying image signatures" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Deployment" new.namespace="default" new.name="busybox" rule.name="autogen-verify-signature-notary" image="172.19.0.3:5000/busybox:1.36.0-glibc" attestors=1 attestations=0 +I0213 10:06:16.332989 1 notary.go:44] "verifying image" logger="Notary" reference="172.19.0.3:5000/busybox:1.36.0-glibc" +I0213 10:06:16.339005 1 imageverifier.go:498] "image attestors verification failed" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Deployment" new.namespace="default" new.name="busybox" rule.name="autogen-verify-signature-notary" verifiedCount=0 requiredCount=1 errors=".attestors[0].entries[0]: failed to verify 172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: no signature is associated with \"172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0\", make sure the artifact was signed successfully" +E0213 10:06:16.339030 1 imageverifier.go:360] "failed to verify image" err=".attestors[0].entries[0]: failed to verify 172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: no signature is associated with \"172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0\", make sure the artifact was signed successfully" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Deployment" new.namespace="default" new.name="busybox" rule.name="autogen-verify-signature-notary" +I0213 10:06:16.343496 1 block.go:29] "blocking admission request" logger="webhooks.resource.mutate" gvk="apps/v1, Kind=Deployment" gvr={"group":"apps","version":"v1","resource":"deployments"} namespace="default" name="busybox" operation="CREATE" uid="217365f6-cf1d-4249-8eff-1a1743f17866" user={"username":"kubernetes-admin","groups":["system:masters","system:authenticated"]} roles=null clusterroles=["cluster-admin","system:basic-user","system:discovery","system:public-info-viewer"] resource.gvk="apps/v1, Kind=Deployment" kind="Deployment" policy="check-image-notary" +E0213 10:06:16.343577 1 handlers.go:173] "image verification failed" err=< + + + resource Deployment/default/busybox was blocked due to the following policies + + check-image-notary: + autogen-verify-signature-notary: 'failed to verify image 172.19.0.3:5000/busybox:1.36.0-glibc: + .attestors[0].entries[0]: failed to verify 172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0: + no signature is associated with "172.19.0.3:5000/busybox@sha256:086417a48026173aaadca4ce43a1e4b385e8e62cc738ba79fc6637049674cac0", + make sure the artifact was signed successfully' + > logger="webhooks.resource.mutate" gvk="apps/v1, Kind=Deployment" gvr={"group":"apps","version":"v1","resource":"deployments"} namespace="default" name="busybox" operation="CREATE" uid="217365f6-cf1d-4249-8eff-1a1743f17866" user={"username":"kubernetes-admin","groups":["system:masters","system:authenticated"]} roles=null clusterroles=["cluster-admin","system:basic-user","system:discovery","system:public-info-viewer"] resource.gvk="apps/v1, Kind=Deployment" kind="Deployment" +``` + +Signed image: + +```shell +$ kubectl create deployment --image 172.19.0.3:5000/busybox:1.36.1-glibc busybox +deployment.apps/busybox created +``` + +Kyverno logs with success: + +```console +I0213 10:06:22.364263 1 event_broadcaster.go:338] "Event occurred" object="check-image-notary" kind="ClusterPolicy" apiVersion="kyverno.io/v1" type="Normal" reason="PolicyApplied" action="Resource Passed" note="Deployment default/busybox: pass" +I0213 10:06:22.364075 1 validation.go:108] "validation passed" logger="webhooks.resource.validate" gvk="apps/v1, Kind=Deployment" gvr={"group":"apps","version":"v1","resource":"deployments"} namespace="default" name="busybox" operation="CREATE" uid="39d07ca8-928c-4fec-95e5-8b50e583a741" user={"username":"kubernetes-admin","groups":["system:masters","system:authenticated"]} roles=null clusterroles=["cluster-admin","system:basic-user","system:discovery","system:public-info-viewer"] resource.gvk="apps/v1, Kind=Deployment" kind="Deployment" action="validate" resource="default/Deployment/busybox" operation="CREATE" gvk="apps/v1, Kind=Deployment" policy="check-image-notary" +I0213 10:06:22.405312 1 imageverifier.go:265] "cache entry not found" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Pod" new.namespace="default" new.name="" rule.name="verify-signature-notary" namespace="" policy="check-image-notary" ruleName="verify-signature-notary" imageRef="172.19.0.3:5000/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7" +I0213 10:06:22.405329 1 imageverifier.go:321] "verifying image signatures" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Pod" new.namespace="default" new.name="" rule.name="verify-signature-notary" image="172.19.0.3:5000/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7" attestors=1 attestations=0 +I0213 10:06:22.405639 1 notary.go:44] "verifying image" logger="Notary" reference="172.19.0.3:5000/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7" +I0213 10:06:22.416321 1 notary.go:130] "content" logger="Notary" type="application/vnd.cncf.notary.payload.v1+json" data="{\"targetArtifact\":{\"digest\":\"sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7\",\"mediaType\":\"application/vnd.docker.distribution.manifest.v2+json\",\"size\":527}}" +I0213 10:06:22.416337 1 notary.go:81] "verified image" logger="Notary" type="application/vnd.docker.distribution.manifest.v2+json" digest="sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7" size=527 +I0213 10:06:22.416346 1 imageverifier.go:489] "image attestors verification succeeded" logger="engine.verify" policy.name="check-image-notary" policy.namespace="" policy.apply="All" new.kind="Pod" new.namespace="default" new.name="" rule.name="verify-signature-notary" verifiedCount=1 requiredCount=1 +I0213 10:06:22.421954 1 validation.go:108] "validation passed" logger="webhooks.resource.validate" gvk="/v1, Kind=Pod" gvr={"group":"","version":"v1","resource":"pods"} namespace="default" name="busybox-86f6b9cb-8qsds" operation="CREATE" uid="5a492250-5abb-4866-8c1b-f771e69d5d35" user={"username":"system:serviceaccount:kube-system:replicaset-controller","uid":"0a4d1ed8-be02-4815-872a-3d8e11ca3d4b","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]} roles=null clusterroles=["system:basic-user","system:controller:replicaset-controller","system:discovery","system:public-info-viewer","system:service-account-issuer-discovery"] resource.gvk="/v1, Kind=Pod" kind="Pod" action="validate" resource="default/Pod/busybox-86f6b9cb-8qsds" operation="CREATE" gvk="/v1, Kind=Pod" policy="check-image-notary" +I0213 10:06:22.422093 1 event_broadcaster.go:338] "Event occurred" object="check-image-notary" kind="ClusterPolicy" apiVersion="kyverno.io/v1" type="Normal" reason="PolicyApplied" action="Resource Passed" note="Pod default/busybox-86f6b9cb-8qsds: pass" +``` + +NOTE: the `pod-success` and `deployment-success` pods will show up as +`ImagePullBackoff` as configuring non-localhost Docker registry within +Kind is wonky. This does not mean anything bad in this POC context. Kind +integration with registries is just not set up properly for pulling, even though +the verification of signatures work as expected. diff --git a/security/container-signing-poc/kyverno/policy/kyverno-policy.yaml b/security/container-signing-poc/kyverno/policy/kyverno-policy.yaml new file mode 100644 index 00000000..a1efccb5 --- /dev/null +++ b/security/container-signing-poc/kyverno/policy/kyverno-policy.yaml @@ -0,0 +1,55 @@ +apiVersion: kyverno.io/v2beta1 +kind: ClusterPolicy +metadata: + name: check-image-notary +spec: + validationFailureAction: Enforce + webhookTimeoutSeconds: 30 + failurePolicy: Fail + rules: + - name: verify-signature-notary + match: + any: + - resources: + kinds: + - Pod + verifyImages: + - type: Notary + imageReferences: + - "172.19.0.3:*/*" + attestors: + - count: 1 + entries: + - certificates: + cert: |- + -----BEGIN CERTIFICATE----- + MIIFTTCCAzWgAwIBAgIUaoK1qGzu3IEiYXSRWWDKC4TIBBUwDQYJKoZIhvcNAQEL + BQAwLjERMA8GA1UECgwITm90YXRpb24xGTAXBgNVBAMMEE5vdGF0aW9uIFJvb3Qg + Q0EwHhcNMjQwMjEzMDgwNTQwWhcNMjUwMjEyMDgwNTQwWjAuMREwDwYDVQQKDAhO + b3RhdGlvbjEZMBcGA1UEAwwQTm90YXRpb24gUm9vdCBDQTCCAiIwDQYJKoZIhvcN + AQEBBQADggIPADCCAgoCggIBAJ0mw0O7FvTFDpuX1FJ4xK9BEN+ku6zgRVWX4iNk + rp994sfaVe+Di+zBtY7A/JpmzxJVOc42dS4xYmhlldRV/TVMmz7kEVFmDkDyCfbV + Qgdl8A3WtkioZgLoDm7KjwEGl4IOHEuNZbKUF0XQ2aB6ZmdcxRY4rd7Eop2VpmU5 + YJPBYc0+oyXEDhVHRibHRO0gdJ0ZqFyZtUrXSAV53nh4YheWRbTacX2B/8tjyUAs + RSGdM8zY+bnQ7F5heAqFJMXje+mOlQ2YL34JlsRU2ltW3/g4xJdmbphBqj6QF+UW + VxR5vqwA4Eeke4TgMxJwXoSuQqghGnamAoaDqZoBhJJE/+P8IhhA0Tux5XgREE8S + izE5PME5VJZfzvBPkDtTmj/o40iL7SJOSI69NWjroNxaO67s/wrrXwOjPe/d5DbK + HV8GvS3LVjExq+t3QgWbllw65IMEiz3oNn/kPBbxY0aPmVHcIyrRhx67/Le7nlcb + AMERtzBhXa7cfV0ym78yAdt/b3bLFxkXhYkNHcRaGk5/E+JZ0zhSiOp/DaMdCwcn + hJJqV9XSwMLbrNhvXmrD+1MjRBjjy13tnxWLN4f2PEyzpfcchAA2ql6yNs/xGryF + 0pox5W6EIkOvDz3ZsqGqYLaFdAHfLdE/bgcEv7Pvz3tXiczOD1HZTqajy1rihhbv + oquzAgMBAAGjYzBhMB0GA1UdDgQWBBTIUfHlUg7oj9aJn6C3sXM7x1EYvjAfBgNV + HSMEGDAWgBTIUfHlUg7oj9aJn6C3sXM7x1EYvjAPBgNVHRMBAf8EBTADAQH/MA4G + A1UdDwEB/wQEAwICBDANBgkqhkiG9w0BAQsFAAOCAgEAJt/b+e62tJ4sXD4AY6KI + RE37zSxLrYChBK6uH2RgCBHT/+Q4BSTdOZ1wpOXXsGzi23SbqLODudMJsmMw+84G + ariOxvvsFxPeHEufb+xVFI8vXreRfEJ0InpYSr1VCed1PBeGB/IXcZV/v6nVmakv + 0V5biwg2Zdzca/KdxuJWCPux/vIljeL2lsw1C5G6A9DYOjMFbYnZyG0kEr5R5v0n + XK1d7Ot0YYx9G28Uuv5zGoa40S6Kyuq027m+hnguvKiLNe8HLFDAF0s8VKEh4Ar1 + 7zeIFdHRFQn2ttxKuSgTBkYprmFXn686SpbYQ2+ajCW6h3j4tbVD/IlyGJZYne7m + C1NRvlWN55oxmoLAwAzUjjy7zlUD2//tVskSOUmYzISzUsG6myHXb/z33IfSlxwF + wBfNGyBCDhSUKhtj2Jiy7SMq1b0ALffJ3e7JDZNSSrIdEZ2S4lWUqvQZGt4V8n/J + AOrLPVIRuAGhnV3/9j5EL3dIbxSO8Te6edS7qxuOtfBKRomSOkpILq6Y3bmI6k/O + +/TS9FX9qVA0Vw5xPeSugdX3rOJxv9mUAn0DgW7pxnuAff8s/XcnFx0QTvF36d6l + hehvf8aHNaB6GgtP+YaFaH4z1dJwUWurZI3QAodbGWjDBunXnEY1qS/Nlm7thXVL + YOSTqfr9ajXGL54v9rWfQjw= + -----END CERTIFICATE----- diff --git a/security/container-signing-poc/kyverno/scripts/kind-cluster.sh b/security/container-signing-poc/kyverno/scripts/kind-cluster.sh new file mode 100755 index 00000000..643542b2 --- /dev/null +++ b/security/container-signing-poc/kyverno/scripts/kind-cluster.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -eux + +# 1. Create registry container unless it already exists +REG_NAME="kind-registry" +REG_PORT="5001" + +if [[ "$(docker inspect -f '{{.State.Running}}' "${REG_NAME}" 2>/dev/null || true)" != 'true' ]]; then + docker run -d --restart=always -p "127.0.0.1:${REG_PORT}:5000" \ + --network bridge --name "${REG_NAME}" registry:2 +fi + +# 2. Create kind cluster with containerd registry config dir enabled +# TODO: kind will eventually enable this by default and this patch will +# be unnecessary. +# +# See: +# https://github.com/kubernetes-sigs/kind/issues/2875 +# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration +# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md +cat </dev/null || echo "error: Install docker: https://docs.docker.com/engine/install/" + @type -a notation &>/dev/null || echo "error: Install notation: https://notaryproject.dev/docs/user-guides/installation/cli/" + +build: + rm -f "$(SRC_DIR)/$(SRC_BIN)" + cd "$(SRC_DIR)" && go build . + +install: build + mkdir -p "$(TGT_DIR)" + mv "$(SRC_DIR)/$(SRC_BIN)" "$(TGT_DIR)/$(TGT_BIN)" + +clean: + rm -f "$(SRC_DIR)/$(SRC_BIN)" + rm -rf "$(TGT_DIR)" + rm -f $(EXAMPLES_DIR)/*.{cer,crt,csr,key,pem} + docker rm -f $(TEST_REGISTRY_NAME) + -notation cert delete -s external -t ca --all -y + +clean-e2e: clean + +registry: + docker run -d -p $(TEST_REGISTRY):5000 --name $(TEST_REGISTRY_NAME) $(TEST_REGISTRY_IMAGE) + docker pull $(TEST_IMAGE) + docker tag $(TEST_IMAGE) $(TEST_IMAGE_LOCAL) + docker push $(TEST_IMAGE_LOCAL) + +certificates: + openssl genrsa -out $(EXAMPLES_DIR)/ca.key 4096 + openssl req -new -x509 -days 365 -key $(EXAMPLES_DIR)/ca.key \ + -subj "/O=Notation/CN=Notation Root CA" \ + -out $(EXAMPLES_DIR)/ca.crt -addext "keyUsage=critical,keyCertSign" + + openssl genrsa -out $(EXAMPLES_DIR)/leaf.key 4096 + openssl req -newkey rsa:4096 -nodes -keyout $(EXAMPLES_DIR)/leaf.key \ + -subj "/CN=Notation.leaf" -out $(EXAMPLES_DIR)/leaf.csr + + openssl x509 -req \ + -extfile <(printf "basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature") \ + -days 365 -in $(EXAMPLES_DIR)/leaf.csr -CA $(EXAMPLES_DIR)/ca.crt -CAkey $(EXAMPLES_DIR)/ca.key \ + -CAcreateserial -out $(EXAMPLES_DIR)/leaf.crt + + cat $(EXAMPLES_DIR)/leaf.crt $(EXAMPLES_DIR)/ca.crt > $(EXAMPLES_DIR)/certificate_chain.pem + +test: check-tools install registry certificates sign verify +e2e: test + +sign: + EXTERNAL_CERT_CHAIN=$(EXAMPLES_DIR)/certificate_chain.pem \ + EXTERNAL_PRIVATE_KEY=$(EXAMPLES_DIR)/leaf.key \ + EXTERNAL_SIGNER=$(EXAMPLES_DIR)/rsassa-pss-sha512.sh \ + notation sign --debug --insecure-registry --id "anything" --plugin "external-signer" $(TEST_IMAGE_SIGN) + +inspect: + notation inspect --insecure-registry $(TEST_IMAGE_SIGN) + +verify: inspect + notation cert add -t ca -s external "$(EXAMPLES_DIR)/ca.crt" + notation policy import --force $(EXAMPLES_DIR)/trustpolicy.json + notation verify --insecure-registry -v $(TEST_IMAGE_SIGN) diff --git a/security/container-signing-poc/notation/README.md b/security/container-signing-poc/notation/README.md new file mode 100644 index 00000000..e3e7510c --- /dev/null +++ b/security/container-signing-poc/notation/README.md @@ -0,0 +1,211 @@ + + +# External Signer Plugin for Notation + +This repository implements a plugin that will call any external signer as in +any external program or script, that will be provided with the payload via +`stdin` and should return raw signature via `stdout`. Additionally the +certificate chain needs to be provided. + +This allows implementing custom integration to whatever signing backend, whether +it is a Python script, Bash script, Go executable, or anything you have +available without having to implement the whole thing as Notation Plugin in Go. + +```mermaid +flowchart TB + user(User) + certs(CA cert chain) + notation(Notation) + signer(External Signer plugin) + script(BYO script) + signingservice(Signing service) + registry(Docker registry) + + user-->|notation sign ...|notation + certs-->|passed|signer + notation-->|calls internally|signer + signer-->|writes payload to stdin|script + script-->|calls with payload|signingservice + signingservice-->|returns signature|script + script-->|writes signature to stdout|signer + signer-->|passes signature|notation + notation-->|wraps to envelope, push|registry +``` + +## Plugin + +This plugin is based on the +[Notation plugin spec](https://github.com/notaryproject/specifications/blob/main/specs/plugin-extensibility.md). + +### Building and Installing + +Building the plugin is as simple as `make build`. + +Install the notation-external-signer to the Notation path specified by Notation +plugin spec. On Unix, the path is +`${XDG_CONFIG_HOME}/notation/plugins/external-signer/notation-external-signer`, ie. +`${HOME}/.config/notation/plugins/external-signer/notation-external-signer` + +`make install` will build and install the script into above directory, where it +will be auto-located by Notation. + +Run `make` to get all meaningful make targets, and check the Makefile itself +for all targets. + +### Usage + +Two example scripts are provided in `examples` to test out the plugin. + +Expectation for external signer script is that it reads the Notation payload +from the `stdin` and outputs the signature to `stdout`. + +In the simplest form of external signer, the signer could can sign the payload +with OpenSSL, emulating importing key to Notation and then signing the artifact. + +In more complex form, the script could be talking to any external signer using +their API, which would then sign the payload and return signature to the plugin. + +#### Local signing example + +This example is doing the same thing as importing the key directly to Notation +and use Notation to do the signing. For signing locally, you need the leaf +key (signing directly with the CA key is not recommended) and the supporting +certificate chain. + +You need certificates and certificate chain for the signature: + +```bash +# root ca +openssl genrsa -out examples/ca.key 4096 +openssl req -new -x509 -days 365 -key examples/ca.key \ + -subj "/O=Notation/CN=Notation Root CA" \ + -out examples/ca.crt -addext "keyUsage=critical,keyCertSign" + +# private key and leaf cert +openssl genrsa -out examples/leaf.key 4096 +openssl req -newkey rsa:4096 -nodes -keyout examples/leaf.key \ + -subj "/CN=Notation.leaf" -out examples/leaf.csr + +openssl x509 -req \ + -extfile <(printf "basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature") \ + -days 365 -in examples/leaf.csr -CA examples/ca.crt -CAkey examples/ca.key \ + -CAcreateserial -out examples/leaf.crt + +# certificate chain +cat examples/leaf.crt examples/ca.crt > examples/certificate_chain.pem +``` + +Assuming you have [local Docker registry](https://hub.docker.com/_/registry) +with an image stored there you want signed, +say [busybox:1.36.1-glibc](https://hub.docker.com/_/busybox) with a digest of +`sha256:e046063223f7eaafbfbc026aa3954d9a31b9f1053ba5db04a4f1fdc97abd8963`: + +Example: [rsassa-pss-sha512.sh](examples/rsassa-pss-sha512.sh) + +```bash +export EXTERNAL_PRIVATE_KEY="$(pwd)/tmp/leaf.key" +export EXTERNAL_CERT_CHAIN="$(pwd)/tmp/certificate_chain.pem" +export EXTERNAL_SIGNER="$(pwd)/examples/rsassa-pss-sha512.sh" +notation sign --insecure-registry --id="anything" --plugin "external-signer" \ + 127.0.0.1:5000/busybox@sha256:e046063223f7eaafbfbc026aa3954d9a31b9f1053ba5db04a4f1fdc97abd8963 +``` + +#### Payload and Signature from file example + +We need to pass correct +([Notation specific](https://github.com/notaryproject/specifications/blob/main/specs/signature-specification.md#payload)) +payload to external services for signing. When signing with +`read-signature-from-file.sh`, the script will save a `payload.txt` that need +to be signed by the service or any manual process you like. Notation plugin +needs to wait during the operation to keep metadata intact. + +Information on the payload, which is in base64+base64url double-encoded +[JWS format](https://www.redhat.com/en/blog/jose-json-object-signing-and-encryption). +The payload is delivered as text, and will contain two base64url encoded JSON +objects, concatenated by `.` + +For signing the payload, equivalent of command below needs to be executed: + +```bash +# payload will be read as-is from stdin, raw signature writted to stdout +openssl dgst -sha512 -sign "${EXTERNAL_PRIVATE_KEY}" \ + -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:32 +``` + +Example: [read-signature-from-file.sh](examples/read-signature-from-file.sh) + +```bash +# expectation is to have "payload.sig" with the raw signature present +export EXTERNAL_CERT_CHAIN="$(pwd)/tmp/certificate_chain.pem" +export EXTERNAL_SIGNER="$(pwd)/examples/read-signature-from-file.sh" +notation sign --insecure-registry --id="foo" --plugin "script" \ + 127.0.0.1:5000/busybox@sha256:e046063223f7eaafbfbc026aa3954d9a31b9f1053ba5db04a4f1fdc97abd8963 +``` + +#### Combine the examples + +It is possible to combine the two examples. + +```bash +export EXTERNAL_SIGNER=$(pwd)/examples/read-signature-from-file.sh +export EXTERNAL_CERT_CHAIN=$(pwd)/examples/certificate_chain.pem +notation sign --insecure-registry --plugin script --id "anything" \ + 127.0.0.1:5000/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7 +# at this point, script has writted payload.txt to disk, and is waiting +# payload.sig to appear... after 60s, execution will continue: +Successfully signed 127.0.0.1:5000/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7 +``` + +In another shell: + +```bash +cat payload.txt | EXTERNAL_PRIVATE_KEY=$(pwd)/examples/leaf.key \ + ./examples/rsassa-pss-sha512.sh > payload.sig +``` + +After 60s, the first script will continue and read the signature from disk +and successfully complete the signing. + +### Verifying + +1. Put the CA into trust store: + `notation cert add -t ca -s external "examples/ca.crt"` + +1. Configure trust policy: + + ```bash + $ cat < ./trustpolicy.json + { + "version": "1.0", + "trustPolicies": [ + { + "name": "external-signer-policy", + "registryScopes": [ "*" ], + "signatureVerification": { + "level" : "strict" + }, + "trustStores": [ "ca:external" ], + "trustedIdentities": [ + "*" + ] + } + ] + } + EOF + $ notation policy import ./trustpolicy.json + ``` + +1. Verify the artifact: + + ```bash + # notation verify /@ -v + $ notation verify --insecure-registry \ + 127.0.0.1:5000/busybox:@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7 + Successfully verified signature for 127.0.0.1:5000/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7 + ``` + +1. Inspect the signature: + + ```bash + notation inspect --insecure-registry 127.0.0.1:5000/busybox@sha256:d319b0e3e1745e504544e931cde012fc5470eba649acc8a7b3607402942e5db7 + ``` diff --git a/security/container-signing-poc/notation/cmd/notation-external-signer/.gitignore b/security/container-signing-poc/notation/cmd/notation-external-signer/.gitignore new file mode 100644 index 00000000..6cd8be7d --- /dev/null +++ b/security/container-signing-poc/notation/cmd/notation-external-signer/.gitignore @@ -0,0 +1 @@ +notation-plugin-script diff --git a/security/container-signing-poc/notation/cmd/notation-external-signer/key.go b/security/container-signing-poc/notation/cmd/notation-external-signer/key.go new file mode 100644 index 00000000..23bb5fd5 --- /dev/null +++ b/security/container-signing-poc/notation/cmd/notation-external-signer/key.go @@ -0,0 +1,46 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + + "github.com/notaryproject/notation-core-go/signature" + "github.com/notaryproject/notation-go/plugin/proto" +) + +func runDescribeKey(ctx context.Context, input io.Reader) (*proto.DescribeKeyResponse, error) { + // parse input request + var req proto.DescribeKeyRequest + if err := json.NewDecoder(input).Decode(&req); err != nil { + return nil, &proto.RequestError{ + Code: proto.ErrorCodeValidation, + Err: fmt.Errorf("failed to unmarshal request input: %w", err), + } + } + + // get key spec for notation + keySpec, err := notationKeySpec(ctx, req.KeyID) + if err != nil { + return nil, err + } + return &proto.DescribeKeyResponse{ + KeyID: req.KeyID, + KeySpec: keySpec, + }, nil +} + +func notationKeySpec(ctx context.Context, keyID string) (proto.KeySpec, error) { + certs, err := readCertificateChain(ctx) + if err != nil { + return "", err + } + leafCert := certs[0] + // extract key spec from certificate + keySpec, err := signature.ExtractKeySpec(leafCert) + if err != nil { + return "", err + } + return proto.EncodeKeySpec(keySpec) +} diff --git a/security/container-signing-poc/notation/cmd/notation-external-signer/main.go b/security/container-signing-poc/notation/cmd/notation-external-signer/main.go new file mode 100644 index 00000000..93924baa --- /dev/null +++ b/security/container-signing-poc/notation/cmd/notation-external-signer/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + + "github.com/notaryproject/notation-go/plugin/proto" +) + +func main() { + if len(os.Args) < 2 { + help() + return + } + ctx := context.Background() + var err error + var resp any + switch proto.Command(os.Args[1]) { + case proto.CommandGetMetadata: + resp = runGetMetadata() + case proto.CommandDescribeKey: + resp, err = runDescribeKey(ctx, os.Stdin) + case proto.CommandGenerateSignature: + resp, err = runSign(ctx, os.Stdin) + default: + err = fmt.Errorf("invalid command: %s", os.Args[1]) + } + + // output the response + if err == nil { + // ignore the error because the response only contains valid JSON field. + jsonResp, err2 := json.Marshal(resp) + if err2 != nil { + data, _ := json.Marshal(wrapError(err2)) + os.Stderr.Write(data) + os.Exit(1) + } + _, err = os.Stdout.Write(jsonResp) + } + + // output the error + if err != nil { + data, _ := json.Marshal(wrapError(err)) + os.Stderr.Write(data) + os.Exit(1) + } +} + +func wrapError(err error) *proto.RequestError { + // already wrapped + var nerr *proto.RequestError + if errors.As(err, &nerr) { + return nerr + } + + // default error code + code := proto.ErrorCodeGeneric + return &proto.RequestError{ + Code: code, + Err: err, + } +} + +func help() { + fmt.Printf(`notation-external-signer - Notation External Signer plugin +Usage: + notation-external-signer +Version: + %s +Commands: + describe-key Key description + generate-signature Sign artifacts with an external signer script + get-plugin-metadata Get plugin metadata +`, getVersion()) +} diff --git a/security/container-signing-poc/notation/cmd/notation-external-signer/metadata.go b/security/container-signing-poc/notation/cmd/notation-external-signer/metadata.go new file mode 100644 index 00000000..0e330f55 --- /dev/null +++ b/security/container-signing-poc/notation/cmd/notation-external-signer/metadata.go @@ -0,0 +1,28 @@ +package main + +import ( + "github.com/notaryproject/notation-go/plugin/proto" +) + +var ( + Version = "v0.1.0" + BuildMetadata = "unreleased" +) + +func getVersion() string { + if BuildMetadata == "" { + return Version + } + return Version + "+" + BuildMetadata +} + +func runGetMetadata() *proto.GetMetadataResponse { + return &proto.GetMetadataResponse{ + Name: "external-signer", + Description: "Sign artifacts with any external signer", + Version: getVersion(), + URL: "https://github.com/Nordix/notation-external-signer", + SupportedContractVersions: []string{proto.ContractVersion}, + Capabilities: []proto.Capability{proto.CapabilitySignatureGenerator}, + } +} diff --git a/security/container-signing-poc/notation/cmd/notation-external-signer/sign.go b/security/container-signing-poc/notation/cmd/notation-external-signer/sign.go new file mode 100644 index 00000000..c792e7aa --- /dev/null +++ b/security/container-signing-poc/notation/cmd/notation-external-signer/sign.go @@ -0,0 +1,144 @@ +package main + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/notaryproject/notation-go/plugin/proto" +) + +func runSign(ctx context.Context, input io.Reader) (*proto.GenerateSignatureResponse, error) { + var req proto.GenerateSignatureRequest + if err := json.NewDecoder(input).Decode(&req); err != nil { + return nil, &proto.RequestError{ + Code: proto.ErrorCodeValidation, + Err: fmt.Errorf("failed to unmarshal request input: %w", err), + } + } + + return sign(ctx, &req) +} + +func sign(ctx context.Context, req *proto.GenerateSignatureRequest) (*proto.GenerateSignatureResponse, error) { + // validate request + if req == nil || req.KeyID == "" || req.KeySpec == "" || req.Hash == "" { + return nil, &proto.RequestError{ + Code: proto.ErrorCodeValidation, + Err: errors.New("invalid request input"), + } + } + + // get keySpec + keySpec, err := proto.DecodeKeySpec(req.KeySpec) + if err != nil { + return nil, &proto.RequestError{ + Code: proto.ErrorCodeValidation, + Err: fmt.Errorf("failed to get keySpec, %v", err), + } + } + + external_cmd := os.Getenv("EXTERNAL_SIGNER") + sigBytes, err := runCommand(external_cmd, string(req.Payload)) + if err != nil { + return nil, &proto.RequestError{ + Code: proto.ErrorCodeGeneric, + Err: fmt.Errorf("signing with EXTERNAL_SIGNER=%q failed, %v", external_cmd, err), + } + } + + // read certificate chain from a file + rawCertChain, err := getCertificateChain(ctx) + if err != nil { + return nil, &proto.RequestError{ + Code: proto.ErrorCodeGeneric, + Err: fmt.Errorf("failed to get certificate chain, %v", err), + } + } + + signatureAlgorithmString, err := proto.EncodeSigningAlgorithm(keySpec.SignatureAlgorithm()) + if err != nil { + return nil, &proto.RequestError{ + Code: proto.ErrorCodeGeneric, + Err: fmt.Errorf("failed to encode signing algorithm, %v", err), + } + } + + return &proto.GenerateSignatureResponse{ + KeyID: req.KeyID, + Signature: sigBytes, + SigningAlgorithm: string(signatureAlgorithmString), + CertificateChain: rawCertChain, + }, nil +} + +func runCommand(external_cmd string, payload string) ([]byte, error) { + cmd := exec.Command(external_cmd) + cmd.Stdin = strings.NewReader(payload) + + var out strings.Builder + cmd.Stdout = &out + + err := cmd.Run() + if err != nil { + return nil, err + } + return []byte(out.String()), nil +} + +func getCertificateChain(ctx context.Context) ([][]byte, error) { + certs, err := readCertificateChain(ctx) + if err != nil { + return nil, err + } + // build raw cert chain + rawCertChain := make([][]byte, 0, len(certs)) + for _, cert := range certs { + rawCertChain = append(rawCertChain, cert.Raw) + } + fmt.Fprintf(os.Stderr, "rawchain: %s\n", rawCertChain[0]) + return rawCertChain, nil +} + +func readCertificateChain(ctx context.Context) ([]*x509.Certificate, error) { + // read a certChain from a file + certChainFile := os.Getenv("EXTERNAL_CERT_CHAIN") + certBytes, err := os.ReadFile(certChainFile) + if err != nil { + return nil, errors.New("failed to read certificate chain from EXTERNAL_CERT_CHAIN=" + certChainFile) + } + return parseCertificates(certBytes) +} + +// parseCertificates parses certificates from either PEM or DER data +// returns an empty list if no certificates are found +func parseCertificates(data []byte) ([]*x509.Certificate, error) { + var certs []*x509.Certificate + block, rest := pem.Decode(data) + if block == nil { + // data may be in DER format + derCerts, err := x509.ParseCertificates(data) + if err != nil { + return nil, err + } + certs = append(certs, derCerts...) + } else { + // data is in PEM format + for block != nil { + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + certs = append(certs, cert) + block, rest = pem.Decode(rest) + } + } + return certs, nil +} diff --git a/security/container-signing-poc/notation/examples/.gitignore b/security/container-signing-poc/notation/examples/.gitignore new file mode 100644 index 00000000..3598e43d --- /dev/null +++ b/security/container-signing-poc/notation/examples/.gitignore @@ -0,0 +1,4 @@ +*.pem +*.crt +*.key +*.csr diff --git a/security/container-signing-poc/notation/examples/read-signature-from-file.sh b/security/container-signing-poc/notation/examples/read-signature-from-file.sh new file mode 100755 index 00000000..8661bfbf --- /dev/null +++ b/security/container-signing-poc/notation/examples/read-signature-from-file.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# This is example of custom script reading signature from a file. +# +# INPUT is coming to stdin, and is the payload in correct format. +# OUTPUT needs to be raw signature. +# +# Any errors/logging go to stderr, and will be ignored by the calling plugin, +# so the payload.txt will be written into a file in case "payload.sig" does +# not exist. This way the payload is available for any manual handling/signing. +# +# NOTE: the signer must wait in the same request for the payload.sig to appear +# If signer is rerun, the metadata will change, and the signature will not be +# valid. Increase sleep timeout, make it intelligent, or whatever suits you. + +set -eu + +# we actually can ignore the input unless we want to decode the payload +# and read different signature per input +INPUT=$(cat -) +PAYLOAD="payload.txt" +OUTPUT="payload.sig" + +# if signature is not found, write payload to file, wait 60s for payload.sig +# to appear, and then read it +if ! cat "${OUTPUT}"; then + echo -n "${INPUT}" > "${PAYLOAD}" + sleep 60 + cat "${OUTPUT}" +fi diff --git a/security/container-signing-poc/notation/examples/rsassa-pss-sha512.sh b/security/container-signing-poc/notation/examples/rsassa-pss-sha512.sh new file mode 100755 index 00000000..d997de59 --- /dev/null +++ b/security/container-signing-poc/notation/examples/rsassa-pss-sha512.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# This is example of custom script doing rsassa-pss-sha512 signing. +# This would be pretty much the same as importing the key into Notation +# and let Notation do the signing for you. +# +# INPUT is coming to stdin, and is the payload in correct format. +# OpenSSL reads stdin by default. +# OUTPUT needs to be raw signature. +# OpenSSL writes to stdout by default. +# EXTERNAL_PRIVATE_KEY needs to point to file where the signing key is. + +set -eu + +openssl dgst -sha512 -sign "${EXTERNAL_PRIVATE_KEY}" \ + -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:32 diff --git a/security/container-signing-poc/notation/examples/trustpolicy.json b/security/container-signing-poc/notation/examples/trustpolicy.json new file mode 100644 index 00000000..85e4fe37 --- /dev/null +++ b/security/container-signing-poc/notation/examples/trustpolicy.json @@ -0,0 +1,20 @@ +{ + "version": "1.0", + "trustPolicies": [ + { + "name": "custom-plugin-policy", + "registryScopes": [ + "*" + ], + "signatureVerification": { + "level": "strict" + }, + "trustStores": [ + "ca:external" + ], + "trustedIdentities": [ + "*" + ] + } + ] +} \ No newline at end of file diff --git a/security/container-signing-poc/notation/go.mod b/security/container-signing-poc/notation/go.mod new file mode 100644 index 00000000..e1b01f99 --- /dev/null +++ b/security/container-signing-poc/notation/go.mod @@ -0,0 +1,8 @@ +module github.com/Nordix/notation-external-signer + +go 1.20 + +require ( + github.com/notaryproject/notation-core-go v1.0.2 + github.com/notaryproject/notation-go v1.1.0 +) diff --git a/security/container-signing-poc/notation/go.sum b/security/container-signing-poc/notation/go.sum new file mode 100644 index 00000000..a6e00e27 --- /dev/null +++ b/security/container-signing-poc/notation/go.sum @@ -0,0 +1,5 @@ +github.com/notaryproject/notation-core-go v1.0.2 h1:VEt+mbsgdANd9b4jqgmx2C7U0DmwynOuD2Nhxh3bANw= +github.com/notaryproject/notation-core-go v1.0.2/go.mod h1:2HkQzUwg08B3x9oVIztHsEh7Vil2Rj+tYgxH+JObLX4= +github.com/notaryproject/notation-go v1.1.0 h1:7WBeH8FGoA+GkeUwmBIBnlJc/PpdYaUKfiXu6ZZeEeg= +github.com/notaryproject/notation-go v1.1.0/go.mod h1:ZSk34URQar5fnWflaFByzpDvuefgZKm/mp8Q2tQpBaw= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= diff --git a/security/container-signing-poc/oras/.gitignore b/security/container-signing-poc/oras/.gitignore new file mode 100644 index 00000000..5db5aa66 --- /dev/null +++ b/security/container-signing-poc/oras/.gitignore @@ -0,0 +1,2 @@ +busybox.tar +/busybox diff --git a/security/container-signing-poc/oras/Makefile b/security/container-signing-poc/oras/Makefile new file mode 100644 index 00000000..816b1b48 --- /dev/null +++ b/security/container-signing-poc/oras/Makefile @@ -0,0 +1,62 @@ +# use oras to move busybox image with signatures to another registry + +NOTATION_DIR := ../notation + +# registry1 is same as in ../notation, containing container with signature +NOTATION_REGISTRY := 127.0.0.1:5002 +# registry2 is clean registry where we import continer and signature +TEST_REGISTRY := 127.0.0.1:5001 +# name needs to match kyverno registry for e2e +TEST_REGISTRY_NAME := kind-registry +TEST_REGISTRY_IMAGE := registry:2 + +TEST_TAG := 1.36.1-glibc +TEST_IMAGE := busybox:$(TEST_TAG) +TEST_DIGEST := sha256:db16cd196b8a37ba5f08414e6f6e71003d76665a5eac160cb75ad3759d8b3e29 +TEST_IMAGE1_LOCAL := $(NOTATION_REGISTRY)/$(TEST_IMAGE) +TEST_IMAGE1_SIGN := $(NOTATION_REGISTRY)/busybox@$(TEST_DIGEST) +TEST_IMAGE2_LOCAL := $(TEST_REGISTRY)/$(TEST_IMAGE) +TEST_IMAGE2_SIGN := $(TEST_REGISTRY)/busybox@$(TEST_DIGEST) +TEST_IMAGE_TARBALL := busybox.tar + +SHELL := /bin/bash + +.PHONY: all test e2e verify-image sign-image registry export-image import-image clean check-tools + +all: check-tools + @echo "targets: test clean" + +check-tools: + @type -a oras &>/dev/null || echo "error: Install oras: https://oras.land/docs/installation" + @type -a notation &>/dev/null || echo "error: Install notation: https://notaryproject.dev/docs/user-guides/installation/cli/" + +registry: + docker run -d -p $(TEST_REGISTRY):5000 --name $(TEST_REGISTRY_NAME) $(TEST_REGISTRY_IMAGE) + +sign-image: + # we run full notation test to get signatures + make -C $(NOTATION_DIR) test + +export-image: + oras cp --recursive --from-plain-http $(TEST_IMAGE1_SIGN) --to-oci-layout $(TEST_IMAGE) + tar cf busybox.tar busybox + sudo rm -rf busybox + +import-image: + tar xf busybox.tar + oras cp --recursive --from-oci-layout $(TEST_IMAGE) --to-plain-http $(TEST_IMAGE2_LOCAL) + +verify-image: + notation verify --insecure-registry $(TEST_IMAGE2_SIGN) + +test: check-tools sign-image export-image registry import-image verify-image + @echo Success! + +e2e: check-tools export-image registry import-image verify-image + +clean: clean-e2e + make -C $(NOTATION_DIR) clean + +clean-e2e: clean + sudo rm -rf busybox.tar busybox + -docker rm -f $(TEST_REGISTRY_NAME) diff --git a/security/container-signing-poc/oras/README.md b/security/container-signing-poc/oras/README.md new file mode 100644 index 00000000..15647c92 --- /dev/null +++ b/security/container-signing-poc/oras/README.md @@ -0,0 +1,108 @@ +# Moving signatures from CI to Prod + +Key part of the problem is moving signatures along with images from CI registry +to Production registry. Notation as of right now only signs images in a +registry and if product is shipped to customer as a delivery (airgap +installations for example), some extra steps are needed. + +## Make targets + +`make test` to run full test, and `make clean` to clean it up. Makefile targets +utilize Notation's Makefile for signing containers. + +Run `make` to get all meaningful make targets, and check the Makefile itself +for all targets. + +## Exporting signatures as blobs with Oras + +[Oras](https://oras.book/) is capable of handling Notation signatures, as well +as any OCI artifacts in a registry. + +```mermaid +flowchart TB + user(User) + registry(Docker registry) + oras(Oras CLI) + orasprod(Oras CLI production) + registry(CI registry) + registryprod(Production registry) + notation(Notation signing workflow) + tarball(Exported artifacts) + tarballprod(Artifacts) + kyverno(Kyverno admission controller) + + subgraph ci["CI"] + user-->|notation sign ...|notation + notation-->|push signature|registry + registry-->|exported by|oras-->|packaged|tarball + end + + subgraph prod["Production"] + tarballprod-->|loaded by|orasprod + orasprod-->|imports to|registryprod + registryprod-->|used by|kyverno + end + + ci-->|delivered|prod +``` + +Oras has +[documentation](https://oras.land/docs/how_to_guides/distributing_oci_layouts) +specifically for this. + +```shell +# sign with notation first, then export to oci layout +# note: use a "tag", not sha in oci layout and include --recursive to copy signatures too +$ oras cp --recursive --from-plain-http 127.0.0.1:5002/busybox@sha256:db16cd196b8a37ba5f08414e6f6e71003d76665a5eac160cb75ad3759d8b3e29 --to-oci-layout busybox:1.36.1-glibc +... + +# package the oci-layout into tarball for delivery +$ tar -cf busybox.tar busybox + +# move tarball to prod, and unpack tarball +$ tar -xf busybox.tar + +# push from oci-layout to registry +$ oras cp --recursive --from-oci-layout busybox:1.36.1-glibc --to-plain-http 127.0.0.1:5000/busybox:1.36.1-glibc +Copied [oci-layout] busybox:1.36.1-glibc => [registry] 127.0.0.1:5000/busybox:1.36.1-glibc +Digest: sha256:db16cd196b8a37ba5f08414e6f6e71003d76665a5eac160cb75ad3759d8b3e29 + +# verify the copied images signatures with notation +$ notation verify --insecure-registry 127.0.0.1:5000/busybox@sha256:db16cd196b8a37ba5f08414e6f6e71003d76665a5eac160cb75ad3759d8b3e29 +Successfully verified signature for 127.0.0.1:5000/busybox@sha256:db16cd196b8a37ba5f08414e6f6e71003d76665a5eac160cb75ad3759d8b3e29 + +# inspect signature +$ notation inspext --insecure-registry 127.0.0.1:5000/busybox +Inspecting all signatures for signed artifact +127.0.0.1:5000/busybox@sha256:db16cd196b8a37ba5f08414e6f6e71003d76665a5eac160cb75ad3759d8b3e29 +└── application/vnd.cncf.notary.signature + └── sha256:724e90a163d792655803ebd579e61523dc40512099842139f922dc65c3431a4e + ├── media type: application/jose+json + ├── signature algorithm: RSASSA-PSS-SHA-512 + ├── signed attributes + │ ├── signingScheme: notary.x509 + │ └── signingTime: Thu Mar 21 08:54:23 2024 + ├── user defined attributes + │ └── (empty) + ├── unsigned attributes + │ └── signingAgent: Notation/1.0.0 external-signer/v0.1.0+unreleased + ├── certificates + │ ├── SHA256 fingerprint: df3cc01f956e06610b52d2d8922949e9fb605d7c8b3b9b74cc4d136d732fc562 + │ │ ├── issued to: CN=Notation.leaf + │ │ ├── issued by: CN=Notation Root CA,O=Notation + │ │ └── expiry: Fri Mar 21 06:54:23 2025 + │ └── SHA256 fingerprint: 680ad58c555df9437b9485c5f4a7e023c7685dd38e5be3e56d6e5bd5fa0110e6 + │ ├── issued to: CN=Notation Root CA,O=Notation + │ ├── issued by: CN=Notation Root CA,O=Notation + │ └── expiry: Fri Mar 21 06:54:21 2025 + └── signed artifact + ├── media type: application/vnd.docker.distribution.manifest.v2+json + ├── digest: sha256:db16cd196b8a37ba5f08414e6f6e71003d76665a5eac160cb75ad3759d8b3e29 + └── size: 527 + +``` + +## Using Skopeo to move signatures from one registry to another + +Originally, tool investigated for packaging was Skopeo, but +[Skopeo is not supporting Notation signatures yet](https://github.com/containers/skopeo/issues/2227)