Merge pull request rook#15433 from SkalaNetworks/master

fix(csi-addons): bind cephfs and rbd provisionners on non-colliding p…

Author: travisn

Documentation/Helm-Charts/operator-chart.md

@@ -67,7 +67,9 @@ The following table lists the configurable parameters of the rook-operator chart
 | `csi.csiAddons.enabled` | Enable CSIAddons | `false` |
 | `csi.csiAddons.repository` | CSIAddons sidecar image repository | `"quay.io/csiaddons/k8s-sidecar"` |
 | `csi.csiAddons.tag` | CSIAddons sidecar image tag | `"v0.11.0"` |
+| `csi.csiAddonsCephFSProvisionerPort` | CSI Addons server port for the Ceph FS provisioner | `9070` |
 | `csi.csiAddonsPort` | CSI Addons server port | `9070` |
+| `csi.csiAddonsRBDProvisionerPort` | CSI Addons server port for the RBD provisioner | `9070` |
 | `csi.csiCephFSPluginResource` | CEPH CSI CephFS plugin resource requirement list | see values.yaml |
 | `csi.csiCephFSPluginVolume` | The volume of the CephCSI CephFS plugin DaemonSet | `nil` |
 | `csi.csiCephFSPluginVolumeMount` | The volume mounts of the CephCSI CephFS plugin DaemonSet | `nil` |

deploy/charts/rook-ceph/templates/configmap.yaml

@@ -208,6 +208,12 @@ data:
 {{- if .Values.csi.csiAddonsPort }}
   CSIADDONS_PORT: {{ .Values.csi.csiAddonsPort | quote }}
 {{- end }}
+{{- if .Values.csi.csiAddonsRBDProvisionerPort }}
+  CSIADDONS_RBD_PROVISIONER_PORT: {{ .Values.csi.csiAddonsRBDProvisionerPort | quote }}
+{{- end }}
+{{- if .Values.csi.csiAddonsCephFSProvisionerPort }}
+  CSIADDONS_CEPHFS_PROVISIONER_PORT: {{ .Values.csi.csiAddonsCephFSProvisionerPort | quote }}
+{{- end }}
 {{- if .Values.csi.forceCephFSKernelClient }}
   CSI_FORCE_CEPHFS_KERNEL_CLIENT: {{ .Values.csi.forceCephFSKernelClient | quote }}
 {{- end }}

deploy/charts/rook-ceph/values.yaml

@@ -443,6 +443,12 @@ csi:
   # -- CSI Addons server port
   # @default -- `9070`
   csiAddonsPort:
+  # -- CSI Addons server port for the RBD provisioner
+  # @default -- `9070`
+  csiAddonsRBDProvisionerPort:
+  # -- CSI Addons server port for the Ceph FS provisioner
+  # @default -- `9070`
+  csiAddonsCephFSProvisionerPort:
 
   # -- Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS
   # you may want to disable this setting. However, this will cause an issue during upgrades

deploy/examples/operator.yaml

@@ -470,7 +470,12 @@ data:
   # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081"
   # Configure CSI RBD liveness metrics port
   # CSI_RBD_LIVENESS_METRICS_PORT: "9080"
+
+  # We can override the ports for each individual component that uses the CSIADDONS sidecar
+  # This is useful if we're running in hostNetwork, where ports may conflict on the same host
   # CSIADDONS_PORT: "9070"
+  # CSIADDONS_RBD_PROVISIONER_PORT: "9070"
+  # CSIADDONS_CEPHFS_PROVISIONER_PORT: "9070"
 
   # Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options
   # Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR

pkg/operator/ceph/csi/csi.go

@@ -97,10 +97,22 @@ func (r *ReconcileCSI) setParams() error {
 	if err != nil {
 		return errors.Wrap(err, "error getting CSI CephFS liveness metrics port.")
 	}
+
 	CSIParam.CSIAddonsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_PORT", DefaultCSIAddonsPort)
 	if err != nil {
 		return errors.Wrap(err, "failed to get CSI Addons port")
 	}
+
+	CSIParam.CSIAddonsRBDProvisionerPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_RBD_PROVISIONER_PORT", DefaultCSIAddonsRBDProvisionerPort)
+	if err != nil {
+		return errors.Wrap(err, "failed to get CSI Addons port for RBD provisioner")
+	}
+
+	CSIParam.CSIAddonsCephFSProvisionerPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_CEPHFS_PROVISIONER_PORT", DefaultCSIAddonsCephFSProvisionerPort)
+	if err != nil {
+		return errors.Wrap(err, "failed to get CSI Addons port for Ceph FS provisioner")
+	}
+
 	CSIParam.RBDLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_LIVENESS_METRICS_PORT", DefaultRBDLivenessMerticsPort)
 	if err != nil {
 		return errors.Wrap(err, "error getting CSI RBD liveness metrics port.")

pkg/operator/ceph/csi/spec.go

@@ -83,6 +83,8 @@ type Param struct {
 	SidecarLogLevel                          uint8
 	CephFSLivenessMetricsPort                uint16
 	CSIAddonsPort                            uint16
+	CSIAddonsRBDProvisionerPort              uint16
+	CSIAddonsCephFSProvisionerPort           uint16
 	RBDLivenessMetricsPort                   uint16
 	KubeApiBurst                             uint16
 	KubeApiQPS                               float32
@@ -219,12 +221,14 @@ const (
 	// kubelet directory path
 	DefaultKubeletDirPath = "/var/lib/kubelet"
 
-	// grpc metrics and liveness port for cephfs  and rbd
-	DefaultCephFSGRPCMerticsPort     uint16 = 9091
-	DefaultCephFSLivenessMerticsPort uint16 = 9081
-	DefaultRBDGRPCMerticsPort        uint16 = 9090
-	DefaultRBDLivenessMerticsPort    uint16 = 9080
-	DefaultCSIAddonsPort             uint16 = 9070
+	// gRPC metrics and liveness port for CephFS and RBD
+	DefaultCephFSGRPCMerticsPort          uint16 = 9091
+	DefaultCephFSLivenessMerticsPort      uint16 = 9081
+	DefaultRBDGRPCMerticsPort             uint16 = 9090
+	DefaultRBDLivenessMerticsPort         uint16 = 9080
+	DefaultCSIAddonsPort                  uint16 = 9070
+	DefaultCSIAddonsRBDProvisionerPort    uint16 = 9070
+	DefaultCSIAddonsCephFSProvisionerPort uint16 = 9070
 
 	// default log level for csi containers
 	defaultLogLevel        uint8 = 0

pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml

@@ -216,7 +216,7 @@ spec:
             - "--node-id=$(NODE_ID)"
             - "--v={{ .LogLevel }}"
             - "--csi-addons-address=$(CSIADDONS_ENDPOINT)"
-            - "--controller-port={{ .CSIAddonsPort }}"
+            - "--controller-port={{ .CSIAddonsCephFSProvisionerPort }}"
             - "--pod=$(POD_NAME)"
             - "--namespace=$(POD_NAMESPACE)"
             - "--pod-uid=$(POD_UID)"
@@ -231,7 +231,7 @@ spec:
             - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log"
             {{ end }}
           ports:
-            - containerPort: {{ .CSIAddonsPort }}
+            - containerPort: {{ .CSIAddonsCephFSProvisionerPort }}
           env:
             - name: NODE_ID
               valueFrom:

pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml

@@ -169,7 +169,7 @@ spec:
             - "--node-id=$(NODE_ID)"
             - "--v={{ .LogLevel }}"
             - "--csi-addons-address=$(CSIADDONS_ENDPOINT)"
-            - "--controller-port={{ .CSIAddonsPort }}"
+            - "--controller-port={{ .CSIAddonsRBDProvisionerPort }}"
             - "--pod=$(POD_NAME)"
             - "--namespace=$(POD_NAMESPACE)"
             - "--pod-uid=$(POD_UID)"
@@ -184,7 +184,7 @@ spec:
             - "--log_file={{ .CsiLogRootPath }}/log/{{ .CsiComponentName }}/csi-addons.log"
             {{ end }}
           ports:
-            - containerPort: {{ .CSIAddonsPort }}
+            - containerPort: {{ .CSIAddonsRBDProvisionerPort }}
           env:
             - name: NODE_ID
               valueFrom: