From 6996653a951c752f921ecc09d61274038e7f2b1a Mon Sep 17 00:00:00 2001 From: Nick Baker Date: Thu, 18 Jan 2024 01:45:31 +0000 Subject: [PATCH] fix nodeadm e2e tests --- .../e2e/cases/containerd-config/config.yaml | 6 --- .../expected-containerd-config.toml | 38 +++++++-------- .../test/e2e/cases/containerd-config/run.sh | 1 + .../test/e2e/cases/kubelet-config-dir/run.sh | 3 +- .../expected-kubelet-config.json | 46 +++++++++++++------ nodeadm/test/e2e/cases/kubelet-config/run.sh | 3 +- nodeadm/test/e2e/helpers.sh | 18 ++++++++ nodeadm/test/e2e/infra/.dockerignore | 1 + nodeadm/test/e2e/infra/Dockerfile | 12 +++-- .../test/e2e/infra/aemm-default-config.json | 27 +++++++++++ .../test/e2e/infra/systemd/imds-mock.service | 8 ---- nodeadm/test/e2e/run.sh | 2 - 12 files changed, 111 insertions(+), 54 deletions(-) create mode 100644 nodeadm/test/e2e/infra/.dockerignore create mode 100644 nodeadm/test/e2e/infra/aemm-default-config.json delete mode 100644 nodeadm/test/e2e/infra/systemd/imds-mock.service diff --git a/nodeadm/test/e2e/cases/containerd-config/config.yaml b/nodeadm/test/e2e/cases/containerd-config/config.yaml index 20e639066..218d59844 100644 --- a/nodeadm/test/e2e/cases/containerd-config/config.yaml +++ b/nodeadm/test/e2e/cases/containerd-config/config.yaml @@ -9,9 +9,3 @@ spec: apiServerEndpoint: https://example.com certificateAuthority: Y2VydGlmaWNhdGVBdXRob3JpdHk= cidr: 10.100.0.0/16 - containerd: - config: - mergeWithDefaults: true - inline: | - [foo] - bar = 'baz' diff --git a/nodeadm/test/e2e/cases/containerd-config/expected-containerd-config.toml b/nodeadm/test/e2e/cases/containerd-config/expected-containerd-config.toml index 4e6003346..746a9732f 100644 --- a/nodeadm/test/e2e/cases/containerd-config/expected-containerd-config.toml +++ b/nodeadm/test/e2e/cases/containerd-config/expected-containerd-config.toml @@ -1,28 +1,30 @@ -root = '/var/lib/containerd' -state = '/run/containerd' version = 2 +root = "/var/lib/containerd" +state = "/run/containerd" +# Users can use the following import directory to add additional +# configuration to containerd. The imports do not behave exactly like overrides. +# see: https://github.com/containerd/containerd/blob/main/docs/man/containerd-config.toml.5.md#format +imports = ["/etc/containerd/config.d/*.toml"] [grpc] -address = '/run/containerd/containerd.sock' +address = "/run/containerd/containerd.sock" -[plugins] -[plugins.'io.containerd.grpc.v1.cri'] -sandbox_image = 'SANDBOX_IMAGE' +[plugins."io.containerd.grpc.v1.cri".containerd] +default_runtime_name = "runc" +discard_unpacked_layers = true -[plugins.'io.containerd.grpc.v1.cri'.cni] -bin_dir = '/opt/cni/bin' -conf_dir = '/etc/cni/net.d' +[plugins."io.containerd.grpc.v1.cri"] +sandbox_image = "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/pause:3.5" -[plugins.'io.containerd.grpc.v1.cri'.containerd] -default_runtime_name = 'runc' -discard_unpacked_layers = true +[plugins."io.containerd.grpc.v1.cri".registry] +config_path = "/etc/containerd/certs.d:/etc/docker/certs.d" -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes] -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc] -runtime_type = 'io.containerd.runc.v2' +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] +runtime_type = "io.containerd.runc.v2" -[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc.options] +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] SystemdCgroup = true -[plugins.'io.containerd.grpc.v1.cri'.registry] -config_path = '/etc/containerd/certs.d:/etc/docker/certs.d' +[plugins."io.containerd.grpc.v1.cri".cni] +bin_dir = "/opt/cni/bin" +conf_dir = "/etc/cni/net.d" diff --git a/nodeadm/test/e2e/cases/containerd-config/run.sh b/nodeadm/test/e2e/cases/containerd-config/run.sh index 6f4e46cd3..eef9e2c17 100755 --- a/nodeadm/test/e2e/cases/containerd-config/run.sh +++ b/nodeadm/test/e2e/cases/containerd-config/run.sh @@ -6,6 +6,7 @@ set -o pipefail source /helpers.sh +mock::imds mock::kubelet 1.27.0 wait::dbus-ready diff --git a/nodeadm/test/e2e/cases/kubelet-config-dir/run.sh b/nodeadm/test/e2e/cases/kubelet-config-dir/run.sh index a20e6bbce..179d6636e 100755 --- a/nodeadm/test/e2e/cases/kubelet-config-dir/run.sh +++ b/nodeadm/test/e2e/cases/kubelet-config-dir/run.sh @@ -6,10 +6,11 @@ set -o pipefail source /helpers.sh +mock::imds mock::kubelet 1.28.0 wait::dbus-ready nodeadm init --skip run --config-source file://config.yaml assert::files-equal /var/lib/kubelet/kubeconfig expected-kubeconfig.yaml -assert::files-equal /etc/kubernetes/kubelet/config.json.d/10-defaults.conf expected-kubelet-config.json +assert::json-files-equal /etc/kubernetes/kubelet/config.json.d/10-defaults.conf expected-kubelet-config.json diff --git a/nodeadm/test/e2e/cases/kubelet-config/expected-kubelet-config.json b/nodeadm/test/e2e/cases/kubelet-config/expected-kubelet-config.json index 89938102e..3f6ae5210 100644 --- a/nodeadm/test/e2e/cases/kubelet-config/expected-kubelet-config.json +++ b/nodeadm/test/e2e/cases/kubelet-config/expected-kubelet-config.json @@ -1,16 +1,17 @@ { - "address": "0.0.0.0", + "kind": "KubeletConfiguration", "apiVersion": "kubelet.config.k8s.io/v1beta1", + "address": "0.0.0.0", "authentication": { - "anonymous": { - "enabled": false + "x509": { + "clientCAFile": "/etc/kubernetes/pki/ca.crt" }, "webhook": { - "cacheTTL": "2m0s", - "enabled": true + "enabled": true, + "cacheTTL": "2m0s" }, - "x509": { - "clientCAFile": "/etc/kubernetes/pki/ca.crt" + "anonymous": { + "enabled": false } }, "authorization": { @@ -20,26 +21,41 @@ "cacheUnauthorizedTTL": "30s" } }, - "cgroupDriver": "cgroupfs", + "cgroupDriver": "systemd", "cgroupRoot": "/", "clusterDomain": "cluster.local", + "containerRuntimeEndpoint": "unix:///run/containerd/containerd.sock", "featureGates": { "RotateKubeletServerCertificate": true }, "hairpinMode": "hairpin-veth", - "kind": "KubeletConfiguration", "protectKernelDefaults": true, "readOnlyPort": 0, + "logging": { + "flushFrequency": 0, + "verbosity": 2, + "options": { + "json": { + "infoBufferSize": "0" + } + } + }, "serializeImagePulls": false, "serverTLSBootstrap": true, "tlsCipherSuites": [ "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", - "TLS_RSA_WITH_AES_256_GCM_SHA384", - "TLS_RSA_WITH_AES_128_GCM_SHA256" - ] -} \ No newline at end of file + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_RSA_WITH_AES_256_GCM_SHA384" + ], + "clusterDNS": [ + "10.100.0.10" + ], + "systemReservedCgroup": "/system", + "kubeReservedCgroup": "/runtime", + "providerID": "aws:///us-east-1f/i-1234567890abcdef0" +} diff --git a/nodeadm/test/e2e/cases/kubelet-config/run.sh b/nodeadm/test/e2e/cases/kubelet-config/run.sh index 823721287..b1848364c 100755 --- a/nodeadm/test/e2e/cases/kubelet-config/run.sh +++ b/nodeadm/test/e2e/cases/kubelet-config/run.sh @@ -6,10 +6,11 @@ set -o pipefail source /helpers.sh +mock::imds mock::kubelet 1.27.0 wait::dbus-ready nodeadm init --skip run --config-source file://config.yaml assert::files-equal /var/lib/kubelet/kubeconfig expected-kubeconfig.yaml -assert::files-equal /etc/kubernetes/kubelet/config.json expected-kubelet-config.json +assert::json-files-equal /etc/kubernetes/kubelet/config.json expected-kubelet-config.json diff --git a/nodeadm/test/e2e/helpers.sh b/nodeadm/test/e2e/helpers.sh index 83b6befd0..e960fce4f 100644 --- a/nodeadm/test/e2e/helpers.sh +++ b/nodeadm/test/e2e/helpers.sh @@ -17,6 +17,19 @@ function assert::files-equal() { fi } +function assert::json-files-equal() { + if [ "$#" -ne 2 ]; then + echo "Usage: assert::json-files-equal FILE1 FILE2" + exit 1 + fi + local FILE1=$1 + local FILE2=$2 + if ! jd $FILE1 $FILE2; then + echo "Files $FILE1 and $FILE2 are not equal" + exit 1 + fi +} + function mock::kubelet() { if [ "$#" -ne 1 ]; then echo "Usage: mock::kubelet VERSION" @@ -48,3 +61,8 @@ function wait::path-exists() { function wait::dbus-ready() { wait::path-exists /run/systemd/private } + +function mock::imds() { + imds-mock --config-file ${1:-/etc/aemm-default-config.json} & + export AWS_EC2_METADATA_SERVICE_ENDPOINT=http://localhost:1338 +} diff --git a/nodeadm/test/e2e/infra/.dockerignore b/nodeadm/test/e2e/infra/.dockerignore new file mode 100644 index 000000000..c975b44a1 --- /dev/null +++ b/nodeadm/test/e2e/infra/.dockerignore @@ -0,0 +1 @@ +test/e2e/cases/ diff --git a/nodeadm/test/e2e/infra/Dockerfile b/nodeadm/test/e2e/infra/Dockerfile index 632ab4a53..33960efb7 100644 --- a/nodeadm/test/e2e/infra/Dockerfile +++ b/nodeadm/test/e2e/infra/Dockerfile @@ -3,6 +3,10 @@ RUN go env -w GOPROXY=direct RUN GOBIN=/bin go install github.com/aws/amazon-ec2-metadata-mock/cmd@v1.11.2 RUN mv /bin/cmd /imds-mock +FROM golang:1.21 AS go-utils +RUN go env -w GOPROXY=direct +RUN GOBIN=/bin go install github.com/josephburnett/jd@latest + FROM golang:1.21 AS nodeadm-build WORKDIR /go/src/github.com/awslabs/amazon-eks-ami/nodeadm RUN go env -w GOPROXY=direct @@ -14,14 +18,16 @@ COPY . . RUN make build RUN mv _bin/nodeadm /nodeadm -FROM amazonlinux:2023 +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 RUN dnf -y update && \ dnf -y install systemd containerd && \ + dnf -y install crypto-policies crypto-policies-scripts && \ dnf clean all +COPY --from=go-utils /bin/jd /usr/local/bin/jd COPY --from=imds-mock-build /imds-mock /usr/local/bin/imds-mock +COPY test/e2e/infra/aemm-default-config.json /etc/aemm-default-config.json COPY --from=nodeadm-build /nodeadm /usr/local/bin/nodeadm -COPY test/e2e/infra/systemd/imds-mock.service /usr/lib/systemd/system/imds-mock.service -RUN systemctl enable imds-mock.service +COPY test/e2e/infra/aemm-default-config.json /etc/aemm-default-config.json COPY test/e2e/infra/systemd/kubelet.service /usr/lib/systemd/system/kubelet.service COPY test/e2e/infra/systemd/containerd.service /usr/lib/systemd/system/containerd.service COPY test/e2e/helpers.sh /helpers.sh diff --git a/nodeadm/test/e2e/infra/aemm-default-config.json b/nodeadm/test/e2e/infra/aemm-default-config.json new file mode 100644 index 000000000..48be9c06c --- /dev/null +++ b/nodeadm/test/e2e/infra/aemm-default-config.json @@ -0,0 +1,27 @@ +{ + "imdsv2": true, + "metadata": { + "values": { + "hostname": "ip-172-16-34-43.ec2.internal", + "instance-id": "i-1234567890abcdef0", + "instance-type": "m4.xlarge", + "local-hostname": "ip-172-16-34-43.ec2.internal", + "local-ipv4": "172.16.34.43", + "mac": "0e:49:61:0f:c3:11", + "mac-network-interface-id": "eni-0f95d3625f5c521cc" + } + }, + "dynamic": { + "values": { + "instance-identity-document": { + "accountId": "0123456789", + "availabilityZone": "us-east-1f", + "privateIp": "10.0.7.10", + "instanceId": "i-1234567890abcdef0", + "architecture": "x86_64", + "instanceType": "m4.xlarge", + "region": "us-west-2" + } + } + } +} diff --git a/nodeadm/test/e2e/infra/systemd/imds-mock.service b/nodeadm/test/e2e/infra/systemd/imds-mock.service deleted file mode 100644 index a634d93c5..000000000 --- a/nodeadm/test/e2e/infra/systemd/imds-mock.service +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -Description=Run the mock IMDS server - -[Service] -ExecStart=imds-mock - -[Install] -WantedBy=multi-user.target diff --git a/nodeadm/test/e2e/run.sh b/nodeadm/test/e2e/run.sh index cd5a55b99..358ab4f36 100755 --- a/nodeadm/test/e2e/run.sh +++ b/nodeadm/test/e2e/run.sh @@ -5,7 +5,6 @@ set -o nounset set -o pipefail cd $(dirname $0)/../.. - printf "🛠️ Building test infra image..." TEST_IMAGE=$(docker build -q -f test/e2e/infra/Dockerfile .) echo "done! Test image: $TEST_IMAGE" @@ -19,7 +18,6 @@ for CASE_DIR in $(ls -d test/e2e/cases/*); do -d \ --rm \ --privileged \ - -v /sys/fs/cgroup:/sys/fs/cgroup \ -v $PWD/$CASE_DIR:/test-case \ $TEST_IMAGE) LOG_FILE=$(mktemp)