Merge pull request #1932 from Sneha-at/data-cache-support

Add support for data cache

Dockerfile

@@ -27,7 +27,7 @@ FROM gke.gcr.io/debian-base:bookworm-v1.0.4-gke.2 AS debian
 
 # Install necessary dependencies
 # google_nvme_id script depends on the following packages: nvme-cli, xxd, bash
-RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash
+RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash kmod lvm2 mdadm
 
 # Since we're leveraging apt to pull in dependencies, we use `gcr.io/distroless/base` because it includes glibc.
 FROM gcr.io/distroless/base-debian12 AS distroless-base
@@ -56,6 +56,35 @@ COPY --from=debian /sbin/e2fsck /sbin/e2fsck
 COPY --from=debian /sbin/fsck /sbin/fsck
 COPY --from=debian /sbin/fsck* /sbin/
 COPY --from=debian /sbin/fsck.xfs /sbin/fsck.xfs
+# Add dependencies for LVM
+COPY --from=debian /etc/lvm /lvm-tmp/lvm
+COPY --from=debian /lib/systemd/system/blk-availability.service /lib/systemd/system/blk-availability.service
+COPY --from=debian /lib/systemd/system/lvm2-lvmpolld.service /lib/systemd/system/lvm2-lvmpolld.service
+COPY --from=debian /lib/systemd/system/lvm2-lvmpolld.socket /lib/systemd/system/lvm2-lvmpolld.socket
+COPY --from=debian /lib/systemd/system/lvm2-monitor.service /lib/systemd/system/lvm2-monitor.service
+COPY --from=debian /lib/udev/rules.d/56-lvm.rules /lib/udev/rules.d/56-lvm.rules
+COPY --from=debian /sbin/fsadm /sbin/fsadm
+COPY --from=debian /sbin/lvm /sbin/lvm
+COPY --from=debian /sbin/lvmdump /sbin/lvmdump
+COPY --from=debian /sbin/lvmpolld /sbin/lvmpolld
+COPY --from=debian /usr/lib/tmpfiles.d /usr/lib/tmpfiles.d
+COPY --from=debian /usr/lib/tmpfiles.d/lvm2.conf /usr/lib/tmpfiles.d/lvm2.conf
+COPY --from=debian /sbin/lv* /sbin/
+COPY --from=debian /sbin/pv* /sbin/
+COPY --from=debian /sbin/vg* /sbin/
+COPY --from=debian /bin/lsblk /bin/lsblk
+COPY --from=debian /sbin/modprobe /sbin/modprobe
+COPY --from=debian /lib/udev /lib/udev
+COPY --from=debian /lib/udev/rules.d /lib/udev/rules.d
+COPY --from=debian /lib/udev/rules.d/55-dm.rules /lib/udev/rules.d/55-dm.rules
+COPY --from=debian /lib/udev/rules.d/60-persistent-storage-dm.rules /lib/udev/rules.d/60-persistent-storage-dm.rules
+COPY --from=debian /lib/udev/rules.d/95-dm-notify.rules /lib/udev/rules.d/95-dm-notify.rules
+COPY --from=debian /sbin/blkdeactivate /sbin/blkdeactivate
+COPY --from=debian /sbin/dmsetup /sbin/dmsetup
+COPY --from=debian /sbin/dmstats /sbin/dmstats
+COPY --from=debian /bin/ls /bin/ls
+# End of dependencies for LVM
+COPY --from=debian /sbin/mdadm /sbin/mdadm
 COPY --from=debian /sbin/mke2fs /sbin/mke2fs
 COPY --from=debian /sbin/mkfs* /sbin/
 COPY --from=debian /sbin/resize2fs /sbin/resize2fs
@@ -71,14 +100,20 @@ COPY --from=debian /bin/date /bin/date
 COPY --from=debian /bin/grep /bin/grep
 COPY --from=debian /bin/sed /bin/sed
 COPY --from=debian /bin/ln /bin/ln
+COPY --from=debian /bin/cp /bin/cp
 COPY --from=debian /bin/udevadm /bin/udevadm
 
 # Copy shared libraries into distroless base.
 COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libdl.so.2 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libpthread.so.0 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libe2p.so.2 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libcom_err.so.2 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libdevmapper.so.1.02.1 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libm.so.6 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libc.so.6 \
+                   /lib/${LIB_DIR_PREFIX}-linux-gnu/libdevmapper-event.so.1.02.1 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libext2fs.so.2 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libgcc_s.so.1 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/liblzma.so.5 \
@@ -99,11 +134,17 @@ COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \
                    /lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/
 
 COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libsmartcols.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libbsd.so.0 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libinih.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmount.so.1 \         
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libudev.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libuuid.so.1 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libaio.so.1 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libgcrypt.so.20 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libsystemd.so.0 \
+                   /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/liblz4.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libacl.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libattr.so.1 \
                    /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libedit.so.2 \
@@ -118,4 +159,5 @@ COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \
 # Copy NVME support required script and rules into distroless base.
 COPY deploy/kubernetes/udev/google_nvme_id /lib/udev_containerized/google_nvme_id
 
-ENTRYPOINT ["/gce-pd-csi-driver"]
+COPY --from=builder /go/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/initialize-driver.sh /initialize-driver.sh
+ENTRYPOINT ["/initialize-driver.sh"]

cmd/gce-pd-csi-driver/main.go

@@ -27,9 +27,12 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/strings/slices"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/common"
 	"sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/deviceutils"
 	gce "sigs.k8s.io/gcp-compute-persistent-disk-csi-driver/pkg/gce-cloud-provider/compute"
@@ -76,6 +79,8 @@ var (
 	fallbackRequisiteZonesFlag  = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk")
 	enableStoragePoolsFlag      = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools")
 	enableHdHAFlag              = flag.Bool("allow-hdha-provisioning", false, "If set to true, will allow the driver to provision Hyperdisk-balanced High Availability disks")
+	enableDataCacheFlag         = flag.Bool("enable-data-cache", false, "If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration")
+	nodeName                    = flag.String("node-name", "", "The node this driver is running on")
 
 	multiZoneVolumeHandleDiskTypesFlag = flag.String("multi-zone-volume-handle-disk-types", "", "Comma separated list of allowed disk types that can use the multi-zone volumeHandle. Used only if --multi-zone-volume-handle-enable")
 	multiZoneVolumeHandleEnableFlag    = flag.Bool("multi-zone-volume-handle-enable", false, "If set to true, the multi-zone volumeHandle feature will be enabled")
@@ -97,7 +102,9 @@ var (
 )
 
 const (
-	driverName = "pd.csi.storage.gke.io"
+	driverName          = "pd.csi.storage.gke.io"
+	dataCacheLabel      = "datacache-storage-gke-io"
+	dataCacheLabelValue = "enabled"
 )
 
 func init() {
@@ -226,7 +233,7 @@ func handle() {
 		}
 		initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond
 		maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond
-		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig, provisionableDisksConfig, *enableHdHAFlag)
+		controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, *enableDataCacheFlag, multiZoneVolumeHandleConfig, listVolumesConfig, provisionableDisksConfig, *enableHdHAFlag)
 	} else if *cloudConfigFilePath != "" {
 		klog.Warningf("controller service is disabled but cloud config given - it has no effect")
 	}
@@ -247,13 +254,23 @@ func handle() {
 		nsArgs := driver.NodeServerArgs{
 			EnableDeviceInUseCheck: *enableDeviceInUseCheck,
 			DeviceInUseTimeout:     *deviceInUseTimeout,
+			EnableDataCache:        *enableDataCacheFlag,
 		}
 		nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter, nsArgs)
 		if *maxConcurrentFormatAndMount > 0 {
 			nodeServer = nodeServer.WithSerializedFormatAndMount(*formatAndMountTimeout, *maxConcurrentFormatAndMount)
 		}
 	}
 
+	if *enableDataCacheFlag {
+		if nodeName == nil || *nodeName == "" {
+			klog.Errorf("Data cache enabled, but --node-name not passed")
+		}
+		if err := setupDataCache(ctx, *nodeName); err != nil {
+			klog.Errorf("DataCache setup failed: %v", err)
+		}
+	}
+
 	err = gceDriver.SetupGCEDriver(driverName, version, extraVolumeLabels, extraTags, identityServer, controllerServer, nodeServer)
 	if err != nil {
 		klog.Fatalf("Failed to initialize GCE CSI Driver: %v", err.Error())
@@ -332,3 +349,33 @@ func urlFlag(target **url.URL, name string, usage string) {
 		return err
 	})
 }
+
+func setupDataCache(ctx context.Context, nodeName string) error {
+	klog.V(2).Infof("Setting up data cache for node %s", nodeName)
+	if nodeName != common.TestNode {
+		cfg, err := rest.InClusterConfig()
+		if err != nil {
+			return err
+		}
+		kubeClient, err := kubernetes.NewForConfig(cfg)
+		if err != nil {
+			return err
+		}
+		node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
+		if err != nil {
+			// We could retry, but this error will also crashloop the driver which may be as good a way to retry as any.
+			return err
+		}
+		if val, found := node.GetLabels()[dataCacheLabel]; !found || val != dataCacheLabelValue {
+			klog.V(2).Infof("Datacache not enabled for node %s; node label %s=%s and not %s", nodeName, dataCacheLabel, val, dataCacheLabelValue)
+			return nil
+		}
+	}
+	klog.V(2).Info("Raiding local ssds to setup data cache")
+	if err := driver.RaidLocalSsds(); err != nil {
+		return fmt.Errorf("Failed to Raid local SSDs, unable to setup data caching, got error %v", err)
+	}
+
+	klog.V(2).Infof("Datacache enabled for node %s", nodeName)
+	return nil
+}

deploy/kubernetes/base/controller/cluster_setup.yaml

@@ -205,6 +205,9 @@ rules:
     verbs:     ['use']
     resourceNames:
     - csi-gce-pd-node-psp
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list"]
 ---
 
 kind: ClusterRole

deploy/kubernetes/base/node_linux/node.yaml

@@ -46,8 +46,15 @@ spec:
             - "--v=5"
             - "--endpoint=unix:/csi/csi.sock"
             - "--run-controller-service=false"
+            - "--enable-data-cache"
+            - "--node-name=$(KUBE_NODE_NAME)"
           securityContext:
             privileged: true
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
           volumeMounts:
             - name: kubelet-dir
               mountPath: /var/lib/kubelet
@@ -66,6 +73,8 @@ spec:
               mountPath: /run/udev
             - name: sys
               mountPath: /sys
+            - name: lib-modules
+              mountPath: /lib/modules
       volumes:
         - name: registration-dir
           hostPath:
@@ -101,6 +110,10 @@ spec:
           hostPath:
             path: /sys
             type: Directory
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+            type: Directory
       # https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
       # See "special case". This will tolerate everything. Node component should
       # be scheduled on all nodes.

initialize-driver.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+/bin/cp -r /lvm-tmp/lvm /etc/
+/bin/sed -i -e "s/.*allow_mixed_block_sizes = 0.*/	allow_mixed_block_sizes = 1/" /etc/lvm/lvm.conf
+/bin/sed -i -e "s/.*udev_sync = 1.*/	udev_sync = 0/" /etc/lvm/lvm.conf
+/bin/sed -i -e "s/.*udev_rules = 1.*/	udev_rules = 0/" /etc/lvm/lvm.conf
+/bin/sed -i -e "s/.*locking_dir = .*/	 locking_dir = \"\/tmp\"/" /etc/lvm/lvm.conf
+
+/gce-pd-csi-driver "$@"

pkg/common/constants.go

@@ -32,4 +32,16 @@ const (
 
 	// Label that is set on a disk when it is used by a 'multi-zone' VolumeHandle
 	MultiZoneLabel = "goog-gke-multi-zone"
+
+	// Data cache mode
+	DataCacheModeWriteBack    = "writeback"
+	DataCacheModeWriteThrough = "writethrough"
+
+	ContextDataCacheSize = "data-cache-size"
+	ContextDataCacheMode = "data-cache-mode"
+
+	// Keys in the publish context
+	ContexLocalSsdCacheSize = "local-ssd-cache-size"
+	// Node name for E2E tests
+	TestNode = "test-node-csi-e2e"
 )