Syncing latest changes from upstream master for rook

.github/workflows/build.yml

@@ -90,7 +90,7 @@ jobs:
           go-version: ${{ matrix.go-version }}
 
       - name: set up QEMU
-        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # master
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # master
         with:
           platforms: all
 

.github/workflows/linters.yaml

@@ -61,4 +61,4 @@ jobs:
         run: make pylint
 
       - name: Setup black for py
-        uses: psf/black@1b2427a2b785cc4aac97c19bb4b9a0de063f9547 # stable
+        uses: psf/black@8a737e727ac5ab2f1d4cf5876720ed276dc8dc4b # stable

.github/workflows/push-build.yaml

@@ -14,6 +14,7 @@ defaults:
 
 permissions:
   contents: read
+  packages: write
 
 jobs:
   push-image-to-container-registry:
@@ -31,7 +32,7 @@ jobs:
 
         # docker/setup-qemu action installs QEMU static binaries, which are used to run builders for architectures other than the host.
       - name: set up QEMU
-        uses: docker/setup-qemu-action@4574d27a4764455b42196d70a065bc6853246a25 # master
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # master
         with:
           platforms: all
 
@@ -41,6 +42,20 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      - name: Login to Quay.io
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
         with:
@@ -54,6 +69,10 @@ jobs:
           echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
           echo "GITHUB_REF"=${GITHUB_REF} >> $GITHUB_ENV
 
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: 3.13
+
       - name: Install dependencies
         run: |
           python3 -m pip install --upgrade pip setuptools wheel

Documentation/CRDs/Cluster/ceph-cluster-crd.md

@@ -150,6 +150,13 @@ A specific will contain a specific release of Ceph as well as security fixes fro
 * `failureDomainLabel`: The label that is expected on each node where the mons
     are expected to be deployed. The labels must be found in the list of
     well-known [topology labels](#osd-topology).
+* `externalMonIDs`: ID list of external mons deployed outside of Rook cluster
+    and not managed by Rook. If set, Rook will not remove external mons from quorum
+    and populate external mons addresses to mon endpoints for CSI.
+    This parameter is supported only for local Rook cluster running in normal mode,
+    meaning that it will be ignored for external cluster (`spec.external.enabled: true`)
+    or for `stretchedCluster`.
+    For more details see [external mons](../../Storage-Configuration/Advanced/ceph-mon-health.md#external-monitors).
 * `zones`: The failure domain names where the Mons are expected to be deployed.
     There must be **at least three zones** specified in the list. Each zone can be
     backed by a different storage class by specifying the `volumeClaimTemplate`.

Documentation/CRDs/Object-Storage/ceph-object-store-crd.md

@@ -390,19 +390,24 @@ spec:
       debug_rgw: "10" # int
       # debug-rgw: "20" # equivalent config keys can have dashes or underscores
       rgw_s3_auth_use_ldap: "true" # bool
+    rgwConfigFromSecret:
+      rgw_keystone_barbican_password: # name of rgw option with secret value
+        name: "barbican-secret" # name of K8s secret 
+        key: "password" # key of secret value in K8s secret.Data map[string]string
     rgwCommandFlags:
       rgw_dmclock_auth_res: "100.0" # float
       rgw_d4n_l1_datacache_persistent_path: /var/log/rook/rgwd4ncache # string
       rgw_d4n_address: "127.0.0.1:6379" # IP string
 ```
 
 * `rgwConfig` - These configurations are applied and modified at runtime, without RGW restart.
+* `rgwConfigFromSecret` - same as `rgwConfig` but config value is referenced from k8s secret.
 * `rgwCommandFlags` - These configurations are applied as CLI arguments and result in RGW daemons
     restarting when updates are applied. Restarts are desired behavior for some RGW configs.
 
 !!! note
-    Once an `rgwConfig` is set, it will not be removed from Ceph's central config store when removed
-    from the `rgwConfig` spec. Be sure to specifically set values back to their defaults once done.
+    Once an `rgwConfig` or `rgwConfigFromSecret` is set, it will not be removed from Ceph's central config store when removed
+    from the `rgwConfig` or `rgwConfigFromSecret` spec. Be sure to specifically set values back to their defaults once done.
     With this in mind, `rgwCommandFlags` may be a better choice for temporary config values like
     debug levels.
 
@@ -455,6 +460,34 @@ data:
     "bindpass.secret": aGVsbG8ud29ybGQK # hello.world
 ```
 
+### Example - usage with `rgwConfigFromSecret`
+
+The sample configuration below demonstrates how to securely handle secret configuration parameters when setting up RGW for [Barbican integration](https://docs.ceph.com/en/latest/radosgw/config-ref/#barbican-settings).
+
+```yaml
+  # ...
+  gateway:
+    # ...
+    rgwConfig:
+      rgw_barbican_url: "http://barbican.example.com:9311"
+      rgw_keystone_barbican_domain: "domain"
+      rgw_keystone_barbican_project: "project" 
+      rgw_keystone_barbican_user: "user"
+    rgwConfigFromSecret:
+      rgw_keystone_barbican_password:
+        name: "barbican-secret"
+        key: "password"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: barbican-secret
+  namespace: rook-ceph
+type: Opaque
+data:
+  "password": aGVsbG8ud29ybGQK # hello.world
+```
+
 ## Deleting a CephObjectStore
 
 During deletion of a CephObjectStore resource, Rook protects against accidental or premature

Documentation/CRDs/specification.md

@@ -7203,6 +7203,23 @@ applied. Use with caution.</p>
 </tr>
 <tr>
 <td>
+<code>rgwConfigFromSecret</code><br/>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core">
+map[string]k8s.io/api/core/v1.SecretKeySelector
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>RgwConfigFromSecret works exactly like RgwConfig but takes config value from Secret Key reference.
+Values are modified at runtime without RGW restart.
+This feature is intended for advanced users. It allows breaking configurations to be easily
+applied. Use with caution.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>rgwCommandFlags</code><br/>
 <em>
 map[string]string
@@ -8777,6 +8794,22 @@ VolumeClaimTemplate
 <p>VolumeClaimTemplate is the PVC definition</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>externalMonIDs</code><br/>
+<em>
+[]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>ExternalMonIDs - optional list of monitor IDs which are deployed externally and not managed by Rook.
+If set, Rook will not remove mons with given IDs from quorum.
+This parameter is used only for local Rook cluster running in normal mode
+and will be ignored if external or stretched mode is used.
+leading</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="ceph.rook.io/v1.MonZoneSpec">MonZoneSpec

Documentation/Helm-Charts/operator-chart.md

@@ -66,7 +66,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `csi.clusterName` | Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster | `nil` |
 | `csi.csiAddons.enabled` | Enable CSIAddons | `false` |
 | `csi.csiAddons.repository` | CSIAddons sidecar image repository | `"quay.io/csiaddons/k8s-sidecar"` |
-| `csi.csiAddons.tag` | CSIAddons sidecar image tag | `"v0.11.0"` |
+| `csi.csiAddons.tag` | CSIAddons sidecar image tag | `"v0.12.0"` |
 | `csi.csiAddonsCephFSProvisionerPort` | CSI Addons server port for the Ceph FS provisioner | `9070` |
 | `csi.csiAddonsPort` | CSI Addons server port | `9070` |
 | `csi.csiAddonsRBDProvisionerPort` | CSI Addons server port for the RBD provisioner | `9070` |

Documentation/Storage-Configuration/Advanced/ceph-mon-health.md

@@ -124,3 +124,43 @@ CephCluster CR:
 - `spec.network.hostNetwork`: When enabled or disabled, Rook fails over all monitors, configuring them to enable or disable host networking.
 - `spec.network.Provider` : When updated from being empty to "host", Rook fails over all monitors, configuring them to enable or disable host networking.
 - `spec.network.multiClusterService`: When enabled or disabled, Rook fails over all monitors, configuring them to start (or stop) using service IPs compatible with the multi-cluster service.
+
+## External Monitors
+
+!!! attention
+    This feature is experimental.
+
+It is possible to have both Rook-managed and external monitors in the same Rook cluster.
+One use case for this is 2 datacenter aka 2-AZ (Availability Zone) setup. For 2-AZ setup, Zone outage will lead to loss of the k8s control plane and half of the worker nodes hosting Rook mons.
+In this case remaining half of the Rook cluster will not be able to form quorum and will be in a stuck state even if the other half of worker nodes are still up.
+To avoid this situation, external mons can be used to form quorum and keep the cluster running.
+
+If there are external monitors, Rook must be aware of them, otherwise Rook will remove the unknown mons from the quorum. This is done by setting the `mon.externalMonIDs` field in the CephCluster CR. The `mon.count` ignores the number of `mon.externalMonIDs`. For example, if `mon.count = 2`, Rook will create two internal mons no matter how many external mons are in the cluster and no matter what their health state might be. External monitors are supported only for the local Rook cluster running in normal mode. The external mons will be ignored for external clusters and stretch clusters.
+
+Here is a step-by-step guide on how to add external monitors to a Rook cluster:
+
+1. Create a CephCluster CR with the `mon.externalMonIDs` field set to the external monitor IDs. For example:
+
+    ```yaml
+    spec:
+      mon:
+        # Spawn 2 Mons - one Mon in each AZ managed by Rook
+        count: 2
+        allowMultiplePerNode: false
+        # ID of external Mon
+        externalMonIDs:
+        - ext-mon-1 
+    ```
+
+    This will tell Rook to create two internal monitors in the cluster and to keep the external monitor with the ID `ext-mon-1` if it is found in the quorum.
+    It is also possible to add `externalMonIDs` to an existing Cluster.
+2. Wait until the Rook cluster and internal mons are up and running.
+3. Manually deploy an external monitor with the ID `ext-mon-1` outside of the Rook cluster and its availability zones. See [ceph guide](https://docs.ceph.com/en/latest/rados/operations/add-or-rm-mons/#adding-removing-monitors) on deploying monitors.
+4. Move the external Mon to [disallow-mode](https://docs.ceph.com/en/reef/rados/operations/change-mon-elections/#rados-operations-disallow-mode) to make sure that it won't be elected as a leader. The purpose of external mode is maintaining quorum to elect a new leader in case of a zone outage. The external Mon will likely have higher latency to the cluster so it should not be elected.
+5. Check that the external monitor is in the quorum by running `ceph status` or `ceph quorum_status` from the toolbox.
+6. Check that the external mon is added to the Rook mon endpoints:
+
+    ```console
+    $ kubectl -n rook-ceph get cm rook-ceph-mon-endpoints -o jsonpath='{.data.data}'
+    a=10.100.68.61:6789,b=10.103.201.172:6789,ext-mon-1=10.102.136.102:6789
+    ```

Documentation/Storage-Configuration/Advanced/ceph-osd-mgmt.md

@@ -24,7 +24,7 @@ ceph osd utilization
 ```
 
 ```console
-kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') bash
+kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[0].metadata.name}') -- bash
 ```
 
 ## Add an OSD

Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md

@@ -166,9 +166,9 @@ that the controller inspects and forwards to one or more CSI-Addons sidecars for
 Deploy the controller by running the following commands:
 
 ```console
-kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/v0.11.0/crds.yaml
-kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/v0.11.0/rbac.yaml
-kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/v0.11.0/setup-controller.yaml
+kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/v0.12.0/crds.yaml
+kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/v0.12.0/rbac.yaml
+kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/v0.12.0/setup-controller.yaml
 ```
 
 This creates the required CRDs and configures permissions.
@@ -196,22 +196,22 @@ Execute the following to enable the CSI-Addons sidecars:
 CSI-Addons supports the following operations:
 
 * Reclaim Space
-    * [Creating a ReclaimSpaceJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/reclaimspace.md#reclaimspacejob)
-    * [Creating a ReclaimSpaceCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/reclaimspace.md#reclaimspacecronjob)
-    * [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/reclaimspace.md#annotating-perstentvolumeclaims)
-    * [Annotating Namespace](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/reclaimspace.md#annotating-namespace)
-    * [Annotating StorageClass](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/reclaimspace.md#annotating-storageclass)
+    * [Creating a ReclaimSpaceJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/reclaimspace.md#reclaimspacejob)
+    * [Creating a ReclaimSpaceCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/reclaimspace.md#reclaimspacecronjob)
+    * [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/reclaimspace.md#annotating-perstentvolumeclaims)
+    * [Annotating Namespace](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/reclaimspace.md#annotating-namespace)
+    * [Annotating StorageClass](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/reclaimspace.md#annotating-storageclass)
 * Network Fencing
-    * [Creating a NetworkFence](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/networkfence.md)
+    * [Creating a NetworkFence](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/networkfence.md)
 * Volume Replication
-    * [Creating VolumeReplicationClass](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/volumereplicationclass.md)
-    * [Creating VolumeReplication CR](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/volumereplication.md)
+    * [Creating VolumeReplicationClass](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/volumereplicationclass.md)
+    * [Creating VolumeReplication CR](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/volumereplication.md)
 * Key Rotation Job for PV encryption
-    * [Creating EncryptionKeyRotationJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/encryptionkeyrotation.md#encryptionkeyrotationjob)
-    * [Creating EncryptionKeyRotationCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/encryptionkeyrotation.md#encryptionkeyrotationcronjob)
-    * [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/encryptionkeyrotation.md#annotating-persistentvolumeclaims)
-    * [Annotating Namespace](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/encryptionkeyrotation.md#annotating-namespace)
-    * [Annotating StorageClass](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.11.0/docs/encryptionkeyrotation.md#annotating-storageclass)
+    * [Creating EncryptionKeyRotationJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/encryptionkeyrotation.md#encryptionkeyrotationjob)
+    * [Creating EncryptionKeyRotationCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/encryptionkeyrotation.md#encryptionkeyrotationcronjob)
+    * [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/encryptionkeyrotation.md#annotating-persistentvolumeclaims)
+    * [Annotating Namespace](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/encryptionkeyrotation.md#annotating-namespace)
+    * [Annotating StorageClass](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.12.0/docs/encryptionkeyrotation.md#annotating-storageclass)
 
 ## Enable RBD and CephFS Encryption Support
 

Documentation/Storage-Configuration/Ceph-CSI/custom-images.md

@@ -24,7 +24,7 @@ ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v5.1.0"
 ROOK_CSI_ATTACHER_IMAGE: "registry.k8s.io/sig-storage/csi-attacher:v4.8.0"
 ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v1.13.1"
 ROOK_CSI_SNAPSHOTTER_IMAGE: "registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0"
-ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.11.0"
+ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.12.0"
 ```
 
 ### **Use private repository**

Documentation/Troubleshooting/ceph-common-issues.md

@@ -99,7 +99,7 @@ After you verify the basic health of the running pods, next you will want to run
 The [rook-ceph-tools pod](ceph-toolbox.md) provides a simple environment to run Ceph tools. Once the pod is up and running, connect to the pod to execute Ceph commands to evaluate that current state of the cluster.
 
 ```console
-kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[*].metadata.name}') bash
+kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -o jsonpath='{.items[*].metadata.name}') -- bash
 ```
 
 #### Ceph Commands

Documentation/Troubleshooting/direct-tools.md

@@ -19,7 +19,7 @@ After the pod is started, connect to it like this:
 
 ```console
 kubectl -n rook-ceph get pod -l app=rook-direct-mount
-$ kubectl -n rook-ceph exec -it <pod> bash
+$ kubectl -n rook-ceph exec -it <pod> -- bash
 ```
 
 ## Block Storage Tools

PendingReleaseNotes.md

@@ -13,3 +13,5 @@ Object:
     See https://github.com/rook/rook/pull/15376 for more information.
 
 ## Features
+
+- Support external mons for local Rook cluster (see [#14733](https://github.com/rook/rook/issues/14733)).

build/csv/ceph/ceph.rook.io_cephclusters.yaml

@@ -468,6 +468,10 @@ spec:
                     maximum: 9
                     minimum: 0
                     type: integer
+                  externalMonIDs:
+                    items:
+                      type: string
+                    type: array
                   failureDomainLabel:
                     type: string
                   stretchCluster:

build/csv/ceph/ceph.rook.io_cephobjectstores.yaml

@@ -1094,6 +1094,22 @@ spec:
                       type: string
                     nullable: true
                     type: object
+                  rgwConfigFromSecret:
+                    additionalProperties:
+                      properties:
+                        key:
+                          type: string
+                        name:
+                          default: ""
+                          type: string
+                        optional:
+                          type: boolean
+                      required:
+                      - key
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    nullable: true
+                    type: object
                   securePort:
                     format: int32
                     maximum: 65535