fix nodeadm e2e tests

awslabs · Jan 18, 2024 · 6996653 · 6996653
1 parent 2aa9a41
commit 6996653
Show file tree

Hide file tree

Showing 12 changed files with 111 additions and 54 deletions.
diff --git a/nodeadm/test/e2e/cases/containerd-config/config.yaml b/nodeadm/test/e2e/cases/containerd-config/config.yaml
@@ -9,9 +9,3 @@ spec:
     apiServerEndpoint: https://example.com
     certificateAuthority: Y2VydGlmaWNhdGVBdXRob3JpdHk=
     cidr: 10.100.0.0/16
-  containerd:
-    config:
-      mergeWithDefaults: true
-      inline: |
-        [foo]
-        bar = 'baz'
diff --git a/nodeadm/test/e2e/cases/containerd-config/expected-containerd-config.toml b/nodeadm/test/e2e/cases/containerd-config/expected-containerd-config.toml
@@ -1,28 +1,30 @@
-root = '/var/lib/containerd'
-state = '/run/containerd'
 version = 2
+root = "/var/lib/containerd"
+state = "/run/containerd"
+# Users can use the following import directory to add additional
+# configuration to containerd. The imports do not behave exactly like overrides.
+# see: https://github.com/containerd/containerd/blob/main/docs/man/containerd-config.toml.5.md#format
+imports = ["/etc/containerd/config.d/*.toml"]
 
 [grpc]
-address = '/run/containerd/containerd.sock'
+address = "/run/containerd/containerd.sock"
 
-[plugins]
-[plugins.'io.containerd.grpc.v1.cri']
-sandbox_image = 'SANDBOX_IMAGE'
+[plugins."io.containerd.grpc.v1.cri".containerd]
+default_runtime_name = "runc"
+discard_unpacked_layers = true
 
-[plugins.'io.containerd.grpc.v1.cri'.cni]
-bin_dir = '/opt/cni/bin'
-conf_dir = '/etc/cni/net.d'
+[plugins."io.containerd.grpc.v1.cri"]
+sandbox_image = "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/pause:3.5"
 
-[plugins.'io.containerd.grpc.v1.cri'.containerd]
-default_runtime_name = 'runc'
-discard_unpacked_layers = true
+[plugins."io.containerd.grpc.v1.cri".registry]
+config_path = "/etc/containerd/certs.d:/etc/docker/certs.d"
 
-[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes]
-[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc]
-runtime_type = 'io.containerd.runc.v2'
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+runtime_type = "io.containerd.runc.v2"
 
-[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc.options]
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
 SystemdCgroup = true
 
-[plugins.'io.containerd.grpc.v1.cri'.registry]
-config_path = '/etc/containerd/certs.d:/etc/docker/certs.d'
+[plugins."io.containerd.grpc.v1.cri".cni]
+bin_dir = "/opt/cni/bin"
+conf_dir = "/etc/cni/net.d"
diff --git a/nodeadm/test/e2e/cases/containerd-config/run.sh b/nodeadm/test/e2e/cases/containerd-config/run.sh
@@ -6,6 +6,7 @@ set -o pipefail
 
 source /helpers.sh
 
+mock::imds
 mock::kubelet 1.27.0
 wait::dbus-ready
 

diff --git a/nodeadm/test/e2e/cases/kubelet-config-dir/run.sh b/nodeadm/test/e2e/cases/kubelet-config-dir/run.sh
@@ -6,10 +6,11 @@ set -o pipefail
 
 source /helpers.sh
 
+mock::imds
 mock::kubelet 1.28.0
 wait::dbus-ready
 
 nodeadm init --skip run --config-source file://config.yaml
 
 assert::files-equal /var/lib/kubelet/kubeconfig expected-kubeconfig.yaml
-assert::files-equal /etc/kubernetes/kubelet/config.json.d/10-defaults.conf expected-kubelet-config.json
+assert::json-files-equal /etc/kubernetes/kubelet/config.json.d/10-defaults.conf expected-kubelet-config.json
diff --git a/nodeadm/test/e2e/cases/kubelet-config/expected-kubelet-config.json b/nodeadm/test/e2e/cases/kubelet-config/expected-kubelet-config.json
@@ -1,16 +1,17 @@
 {
-    "address": "0.0.0.0",
+    "kind": "KubeletConfiguration",
     "apiVersion": "kubelet.config.k8s.io/v1beta1",
+    "address": "0.0.0.0",
     "authentication": {
-        "anonymous": {
-            "enabled": false
+        "x509": {
+            "clientCAFile": "/etc/kubernetes/pki/ca.crt"
         },
         "webhook": {
-            "cacheTTL": "2m0s",
-            "enabled": true
+            "enabled": true,
+            "cacheTTL": "2m0s"
         },
-        "x509": {
-            "clientCAFile": "/etc/kubernetes/pki/ca.crt"
+        "anonymous": {
+            "enabled": false
         }
     },
     "authorization": {
@@ -20,26 +21,41 @@
             "cacheUnauthorizedTTL": "30s"
         }
     },
-    "cgroupDriver": "cgroupfs",
+    "cgroupDriver": "systemd",
     "cgroupRoot": "/",
     "clusterDomain": "cluster.local",
+    "containerRuntimeEndpoint": "unix:///run/containerd/containerd.sock",
     "featureGates": {
         "RotateKubeletServerCertificate": true
     },
     "hairpinMode": "hairpin-veth",
-    "kind": "KubeletConfiguration",
     "protectKernelDefaults": true,
     "readOnlyPort": 0,
+    "logging": {
+        "flushFrequency": 0,
+        "verbosity": 2,
+        "options": {
+            "json": {
+                "infoBufferSize": "0"
+            }
+        }
+    },
     "serializeImagePulls": false,
     "serverTLSBootstrap": true,
     "tlsCipherSuites": [
         "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
-        "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+        "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
         "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305",
+        "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
         "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
         "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305",
-        "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
-        "TLS_RSA_WITH_AES_256_GCM_SHA384",
-        "TLS_RSA_WITH_AES_128_GCM_SHA256"
-    ]
-}
+        "TLS_RSA_WITH_AES_128_GCM_SHA256",
+        "TLS_RSA_WITH_AES_256_GCM_SHA384"
+    ],
+    "clusterDNS": [
+        "10.100.0.10"
+    ],
+    "systemReservedCgroup": "/system",
+    "kubeReservedCgroup": "/runtime",
+    "providerID": "aws:///us-east-1f/i-1234567890abcdef0"
+}
diff --git a/nodeadm/test/e2e/cases/kubelet-config/run.sh b/nodeadm/test/e2e/cases/kubelet-config/run.sh
@@ -6,10 +6,11 @@ set -o pipefail
 
 source /helpers.sh
 
+mock::imds
 mock::kubelet 1.27.0
 wait::dbus-ready
 
 nodeadm init --skip run --config-source file://config.yaml
 
 assert::files-equal /var/lib/kubelet/kubeconfig expected-kubeconfig.yaml
-assert::files-equal /etc/kubernetes/kubelet/config.json expected-kubelet-config.json
+assert::json-files-equal /etc/kubernetes/kubelet/config.json expected-kubelet-config.json
diff --git a/nodeadm/test/e2e/helpers.sh b/nodeadm/test/e2e/helpers.sh
@@ -17,6 +17,19 @@ function assert::files-equal() {
   fi
 }
 
+function assert::json-files-equal() {
+  if [ "$#" -ne 2 ]; then
+    echo "Usage: assert::json-files-equal FILE1 FILE2"
+    exit 1
+  fi
+  local FILE1=$1
+  local FILE2=$2
+  if ! jd $FILE1 $FILE2; then
+    echo "Files $FILE1 and $FILE2 are not equal"
+    exit 1
+  fi
+}
+
 function mock::kubelet() {
   if [ "$#" -ne 1 ]; then
     echo "Usage: mock::kubelet VERSION"
@@ -48,3 +61,8 @@ function wait::path-exists() {
 function wait::dbus-ready() {
   wait::path-exists /run/systemd/private
 }
+
+function mock::imds() {
+  imds-mock --config-file ${1:-/etc/aemm-default-config.json} &
+  export AWS_EC2_METADATA_SERVICE_ENDPOINT=http://localhost:1338
+}
diff --git a/nodeadm/test/e2e/infra/.dockerignore b/nodeadm/test/e2e/infra/.dockerignore
@@ -0,0 +1 @@
+test/e2e/cases/
diff --git a/nodeadm/test/e2e/infra/Dockerfile b/nodeadm/test/e2e/infra/Dockerfile
@@ -3,6 +3,10 @@ RUN go env -w GOPROXY=direct
 RUN GOBIN=/bin go install github.com/aws/amazon-ec2-metadata-mock/cmd@v1.11.2
 RUN mv /bin/cmd /imds-mock
 
+FROM golang:1.21 AS go-utils
+RUN go env -w GOPROXY=direct
+RUN GOBIN=/bin go install github.com/josephburnett/jd@latest
+
 FROM golang:1.21 AS nodeadm-build
 WORKDIR /go/src/github.com/awslabs/amazon-eks-ami/nodeadm
 RUN go env -w GOPROXY=direct
@@ -14,14 +18,16 @@ COPY . .
 RUN make build
 RUN mv _bin/nodeadm /nodeadm
 
-FROM amazonlinux:2023
+FROM public.ecr.aws/amazonlinux/amazonlinux:2023
 RUN dnf -y update && \
     dnf -y install systemd containerd && \
+    dnf -y install crypto-policies crypto-policies-scripts && \
     dnf clean all
+COPY --from=go-utils /bin/jd /usr/local/bin/jd
 COPY --from=imds-mock-build /imds-mock /usr/local/bin/imds-mock
+COPY test/e2e/infra/aemm-default-config.json /etc/aemm-default-config.json
 COPY --from=nodeadm-build /nodeadm /usr/local/bin/nodeadm
-COPY test/e2e/infra/systemd/imds-mock.service /usr/lib/systemd/system/imds-mock.service
-RUN systemctl enable imds-mock.service
+COPY test/e2e/infra/aemm-default-config.json /etc/aemm-default-config.json
 COPY test/e2e/infra/systemd/kubelet.service /usr/lib/systemd/system/kubelet.service
 COPY test/e2e/infra/systemd/containerd.service /usr/lib/systemd/system/containerd.service
 COPY test/e2e/helpers.sh /helpers.sh

diff --git a/nodeadm/test/e2e/infra/aemm-default-config.json b/nodeadm/test/e2e/infra/aemm-default-config.json
@@ -0,0 +1,27 @@
+{
+  "imdsv2": true,
+  "metadata": {
+    "values": {
+      "hostname": "ip-172-16-34-43.ec2.internal",
+      "instance-id": "i-1234567890abcdef0",
+      "instance-type": "m4.xlarge",
+      "local-hostname": "ip-172-16-34-43.ec2.internal",
+      "local-ipv4": "172.16.34.43",
+      "mac": "0e:49:61:0f:c3:11",
+      "mac-network-interface-id": "eni-0f95d3625f5c521cc"
+    }
+  },
+  "dynamic": {
+    "values": {
+      "instance-identity-document": {
+        "accountId": "0123456789",
+        "availabilityZone": "us-east-1f",
+        "privateIp": "10.0.7.10",
+        "instanceId": "i-1234567890abcdef0",
+        "architecture": "x86_64",
+        "instanceType": "m4.xlarge",
+        "region": "us-west-2"
+      }
+    }
+  }
+}
diff --git a/nodeadm/test/e2e/infra/systemd/imds-mock.service b/nodeadm/test/e2e/infra/systemd/imds-mock.service
diff --git a/nodeadm/test/e2e/run.sh b/nodeadm/test/e2e/run.sh
@@ -5,7 +5,6 @@ set -o nounset
 set -o pipefail
 
 cd $(dirname $0)/../..
-
 printf "🛠️ Building test infra image..."
 TEST_IMAGE=$(docker build -q -f test/e2e/infra/Dockerfile .)
 echo "done! Test image: $TEST_IMAGE"
@@ -19,7 +18,6 @@ for CASE_DIR in $(ls -d test/e2e/cases/*); do
     -d \
     --rm \
     --privileged \
-    -v /sys/fs/cgroup:/sys/fs/cgroup \
     -v $PWD/$CASE_DIR:/test-case \
     $TEST_IMAGE)
   LOG_FILE=$(mktemp)