From fc08e87d4493e1289417086220984a2b52fe9441 Mon Sep 17 00:00:00 2001
From: Blaine Gardner <blaine.gardner@ibm.com>
Date: Wed, 20 Nov 2024 10:54:44 -0700
Subject: [PATCH 01/24] Revert "object: create cosi user for each object store"

This reverts commit a941b3c33f452fe7e4c1af34b5e61bc28ffb6cbe.

Stop creating the 'cosi' user in the CephObjectStore reconcile. This
step often fails for some amount of time during initial object store
creation, causing frequent user concern. It has also been the source of
some reported failures that would otherwise be non-breaking for certain
users.

Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>
---
 .../Object-Storage-RGW/cosi.md                | 24 +++++-
 deploy/examples/cosi/cephcosidriver.yaml      | 14 ++++
 pkg/operator/ceph/object/controller.go        | 73 +------------------
 pkg/operator/ceph/object/controller_test.go   | 58 +--------------
 pkg/operator/ceph/object/objectstore.go       |  2 -
 pkg/operator/ceph/object/user.go              | 60 ---------------
 pkg/operator/ceph/object/user/controller.go   | 55 +++++++++++++-
 tests/integration/ceph_cosi_test.go           |  4 +-
 8 files changed, 93 insertions(+), 197 deletions(-)

diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md
index a9541f6848e5..f3635d6a5c9f 100644
--- a/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md
+++ b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md
@@ -33,6 +33,20 @@ metadata:
   namespace: rook-ceph
 spec:
   deploymentStrategy: "Auto"
+---
+# The Ceph-COSI driver needs a privileged user for each CephObjectStore
+# in order to provision buckets and users
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: cosi
+  namespace: rook-ceph # rook operator namespace
+spec:
+  displayName: "cosi user"
+  store: my-store # name of the CephObjectStore
+  capabilities:
+    bucket: "*"
+    user: "*"
 ```
 
 ```console
@@ -40,13 +54,11 @@ cd deploy/examples/cosi
 kubectl create -f cephcosidriver.yaml
 ```
 
-The driver is created in the same namespace as Rook operator.
-
 ## Admin Operations
 
 ### Create a BucketClass and BucketAccessClass
 
-The BucketClass and BucketAccessClass are CRDs defined by COSI. The BucketClass defines the bucket class for the bucket. The BucketAccessClass defines the access class for the bucket. Rook will automatically create a secret named with `rook-ceph-object-user-<store-name>-cosi` which contains credentials used by the COSI driver. This secret is referred by the BucketClass and BucketAccessClass as defined below:
+The BucketClass and BucketAccessClass are CRDs defined by COSI. The BucketClass defines the storage class for the bucket. The BucketAccessClass defines the access class for the bucket. The BucketClass and BucketAccessClass are defined as below:
 
 ```yaml
 kind: BucketClass
@@ -58,7 +70,9 @@ deletionPolicy: Delete
 parameters:
   objectStoreUserSecretName: rook-ceph-object-user-my-store-cosi
   objectStoreUserSecretNamespace: rook-ceph
----
+```
+
+```yaml
 kind: BucketAccessClass
 apiVersion: objectstorage.k8s.io/v1alpha1
 metadata:
@@ -74,6 +88,8 @@ parameters:
 kubectl create -f bucketclass.yaml -f bucketaccessclass.yaml
 ```
 
+The `objectStoreUserSecretName` and `objectStoreUserSecretNamespace` are the name and namespace of the CephObjectStoreUser created in the previous step.
+
 ## User Operations
 
 ### Create a Bucket
diff --git a/deploy/examples/cosi/cephcosidriver.yaml b/deploy/examples/cosi/cephcosidriver.yaml
index a2157c164b92..d9594357ffef 100644
--- a/deploy/examples/cosi/cephcosidriver.yaml
+++ b/deploy/examples/cosi/cephcosidriver.yaml
@@ -5,3 +5,17 @@ metadata:
   namespace: rook-ceph
 spec:
   deploymentStrategy: "Auto"
+---
+# The Ceph-COSI driver needs a privileged user for each CephObjectStore
+# in order to provision buckets and users
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: cosi
+  namespace: rook-ceph # namespace:cluster
+spec:
+  store: my-store # name of the CephObjectStore
+  displayName: "user for cosi"
+  capabilities:
+    user: "*"
+    bucket: "*"
diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go
index 82e8b128c820..e57f9d8896c2 100644
--- a/pkg/operator/ceph/object/controller.go
+++ b/pkg/operator/ceph/object/controller.go
@@ -24,7 +24,6 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/ceph/go-ceph/rgw/admin"
 	"github.com/coreos/pkg/capnslog"
 	bktclient "github.com/kube-object-storage/lib-bucket-provisioner/pkg/client/clientset/versioned"
 	"github.com/pkg/errors"
@@ -82,9 +81,6 @@ var currentAndDesiredCephVersion = opcontroller.CurrentAndDesiredCephVersion
 // allow this to be overridden for unit tests
 var cephObjectStoreDependents = CephObjectStoreDependents
 
-// newMultisiteAdminOpsCtxFunc help us mocking the admin ops API client in unit test
-var newMultisiteAdminOpsCtxFunc = NewMultisiteAdminOpsContext
-
 // ReconcileCephObjectStore reconciles a cephObjectStore object
 type ReconcileCephObjectStore struct {
 	client           client.Client
@@ -243,15 +239,10 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
 		if err != nil {
 			return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get object context")
 		}
-		opsCtx, err := newMultisiteAdminOpsCtxFunc(objCtx, &cephObjectStore.Spec)
+		opsCtx, err := NewMultisiteAdminOpsContext(objCtx, &cephObjectStore.Spec)
 		if err != nil {
 			return reconcile.Result{}, *cephObjectStore, errors.Wrapf(err, "failed to get admin ops API context")
 		}
-		err = r.deleteCOSIUser(opsCtx)
-		if err != nil {
-			// Allow the object store removal to proceed even if the user deletion fails
-			logger.Warningf("failed to delete COSI user. %v", err)
-		}
 		deps, err := cephObjectStoreDependents(r.context, r.clusterInfo, cephObjectStore, objCtx, opsCtx)
 		if err != nil {
 			return reconcile.Result{}, *cephObjectStore, err
@@ -480,11 +471,9 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
 		if err != nil {
 			return reconcile.Result{}, errors.Wrapf(err, "failed to create object store %q", cephObjectStore.Name)
 		}
-
 	}
 
-	// Create COSI user and secret
-	return r.reconcileCOSIUser(cephObjectStore)
+	return reconcile.Result{}, nil
 }
 
 func (r *ReconcileCephObjectStore) retrieveMultisiteZone(store *cephv1.CephObjectStore, zoneGroupName string, realmName string) (reconcile.Result, error) {
@@ -545,61 +534,3 @@ func (r *ReconcileCephObjectStore) getMultisiteResourceNames(cephObjectStore *ce
 
 	return realm.Name, zonegroup.Name, zone.Name, zone, reconcile.Result{}, nil
 }
-
-func (r *ReconcileCephObjectStore) reconcileCOSIUser(cephObjectStore *cephv1.CephObjectStore) (reconcile.Result, error) {
-	// Create COSI user and secret
-	userConfig := generateCOSIUserConfig()
-	var user admin.User
-
-	// Create COSI user
-	objCtx, err := NewMultisiteContext(r.context, r.clusterInfo, cephObjectStore)
-	if err != nil {
-		return reconcile.Result{}, errors.Wrapf(err, "failed to get object context")
-	}
-
-	adminOpsCtx, err := newMultisiteAdminOpsCtxFunc(objCtx, &cephObjectStore.Spec)
-	if err != nil {
-		return reconcile.Result{}, errors.Wrapf(err, "failed to get admin ops API context")
-	}
-
-	user, err = adminOpsCtx.AdminOpsClient.GetUser(r.opManagerContext, *userConfig)
-	if err != nil {
-		if errors.Is(err, admin.ErrNoSuchUser) {
-			logger.Infof("creating COSI user %q", userConfig.ID)
-			user, err = adminOpsCtx.AdminOpsClient.CreateUser(r.opManagerContext, *userConfig)
-			if err != nil {
-				return reconcile.Result{}, errors.Wrapf(err, "failed to create COSI user %q", userConfig.ID)
-			}
-		} else {
-			return reconcile.Result{}, errors.Wrapf(err, "failed to get COSI user %q", userConfig.ID)
-		}
-	}
-
-	// Create COSI user secret
-	return ReconcileCephUserSecret(r.opManagerContext, r.client, r.scheme, cephObjectStore, &user, objCtx.Endpoint, cephObjectStore.Namespace, cephObjectStore.Name, cephObjectStore.Spec.Gateway.SSLCertificateRef)
-}
-
-func generateCOSIUserConfig() *admin.User {
-	userConfig := admin.User{
-		ID:          cosiUserName,
-		DisplayName: cosiUserName,
-	}
-
-	userConfig.UserCaps = cosiUserCaps
-
-	return &userConfig
-}
-
-func (r *ReconcileCephObjectStore) deleteCOSIUser(adminOpsCtx *AdminOpsContext) error {
-	userConfig := generateCOSIUserConfig()
-	err := adminOpsCtx.AdminOpsClient.RemoveUser(r.opManagerContext, *userConfig)
-	if err != nil {
-		if errors.Is(err, admin.ErrNoSuchUser) {
-			logger.Debugf("COSI user %q not found", userConfig.ID)
-			return nil
-		} else {
-			return errors.Wrapf(err, "failed to delete COSI user %q", userConfig.ID)
-		}
-	}
-	return nil
-}
diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go
index c6e1d2cda31e..50bc736326b7 100644
--- a/pkg/operator/ceph/object/controller_test.go
+++ b/pkg/operator/ceph/object/controller_test.go
@@ -18,10 +18,7 @@ limitations under the License.
 package object
 
 import (
-	"bytes"
 	"context"
-	"io"
-	"net/http"
 	"os"
 	"reflect"
 	"testing"
@@ -29,7 +26,6 @@ import (
 
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 
-	"github.com/ceph/go-ceph/rgw/admin"
 	"github.com/coreos/pkg/capnslog"
 	"github.com/pkg/errors"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
@@ -325,35 +321,6 @@ var (
 	store     = "my-store"
 )
 
-var mockMultisiteAdminOpsCtxFunc = func(objContext *Context, spec *cephv1.ObjectStoreSpec) (*AdminOpsContext, error) {
-	mockClient := &MockClient{
-		MockDo: func(req *http.Request) (*http.Response, error) {
-			if req.Method == http.MethodGet || req.Method == http.MethodPut {
-				return &http.Response{
-					StatusCode: http.StatusOK,
-					Body:       io.NopCloser(bytes.NewReader([]byte(userCreateJSON))),
-				}, nil
-			}
-			if req.Method == http.MethodDelete {
-				return &http.Response{
-					StatusCode: http.StatusOK,
-					Body:       io.NopCloser(bytes.NewReader([]byte(""))),
-				}, nil
-			}
-			return nil, errors.Errorf("unexpected method %q", req.Method)
-		},
-	}
-	context := NewContext(objContext.Context, objContext.clusterInfo, store)
-	adminClient, _ := admin.New("rook-ceph-rgw-my-store.mycluster.svc", "53S6B9S809NUP19IJ2K3", "1bXPegzsGClvoGAiJdHQD1uOW2sQBLAZM9j9VtXR", mockClient)
-
-	return &AdminOpsContext{
-		Context:               *context,
-		AdminOpsUserAccessKey: "EOE7FYCNOBZJ5VFV909G",
-		AdminOpsUserSecretKey: "qmIqpWm8HxCzmynCrD6U6vKWi4hnDBndOnmxXNsV", // notsecret
-		AdminOpsClient:        adminClient,
-	}, nil
-}
-
 func TestCephObjectStoreController(t *testing.T) {
 	ctx := context.TODO()
 	// Set DEBUG logging
@@ -370,13 +337,6 @@ func TestCephObjectStoreController(t *testing.T) {
 		return nil
 	}
 
-	// overwrite adminops context func
-	oldNewMultisiteAdminOpsCtxFunc := newMultisiteAdminOpsCtxFunc
-	newMultisiteAdminOpsCtxFunc = mockMultisiteAdminOpsCtxFunc
-	defer func() {
-		newMultisiteAdminOpsCtxFunc = oldNewMultisiteAdminOpsCtxFunc
-	}()
-
 	setupNewEnvironment := func(additionalObjects ...runtime.Object) *ReconcileCephObjectStore {
 		// reset var we use to check if we have called to commit config changes
 		calledCommitConfigChanges = false
@@ -418,11 +378,9 @@ func TestCephObjectStoreController(t *testing.T) {
 		s := scheme.Scheme
 		s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectStore{})
 		s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{})
-		s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{})
 
 		// Create a fake client to mock API calls.
 		cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objects...).Build()
-
 		// Create a ReconcileCephObjectStore object with the scheme and fake client.
 		r := &ReconcileCephObjectStore{
 			client:           cl,
@@ -738,12 +696,7 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
 
 	commitConfigChangesOrig := commitConfigChanges
 	defer func() { commitConfigChanges = commitConfigChangesOrig }()
-	// overwrite adminops context func
-	oldNewMultisiteAdminOpsCtxFunc := newMultisiteAdminOpsCtxFunc
-	newMultisiteAdminOpsCtxFunc = mockMultisiteAdminOpsCtxFunc
-	defer func() {
-		newMultisiteAdminOpsCtxFunc = oldNewMultisiteAdminOpsCtxFunc
-	}()
+
 	// make sure joining multisite calls to commit config changes
 	calledCommitConfigChanges := false
 	commitConfigChanges = func(c *Context) error {
@@ -766,7 +719,7 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) {
 	// Register operator types with the runtime scheme.
 	s := scheme.Scheme
 	s.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephObjectZone{}, &cephv1.CephObjectZoneList{}, &cephv1.CephCluster{}, &cephv1.CephClusterList{}, &cephv1.CephObjectStore{}, &cephv1.CephObjectStoreList{})
-	s.AddKnownTypes(v1.SchemeGroupVersion, &v1.Secret{})
+
 	// Create a fake client to mock API calls.
 	cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(object...).Build()
 
@@ -971,13 +924,6 @@ func TestCephObjectExternalStoreController(t *testing.T) {
 			rgwAdminOpsUserSecret,
 		}
 
-		// overwrite adminops context func
-		oldNewMultisiteAdminOpsCtxFunc := newMultisiteAdminOpsCtxFunc
-		newMultisiteAdminOpsCtxFunc = mockMultisiteAdminOpsCtxFunc
-		defer func() {
-			newMultisiteAdminOpsCtxFunc = oldNewMultisiteAdminOpsCtxFunc
-		}()
-
 		r := getReconciler(objects)
 
 		t.Run("create an external object store", func(t *testing.T) {
diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go
index e2e79304afd5..054b7b9ec100 100644
--- a/pkg/operator/ceph/object/objectstore.go
+++ b/pkg/operator/ceph/object/objectstore.go
@@ -53,8 +53,6 @@ const (
 	SecretKeyName         = "secret-key"
 	svcDNSSuffix          = "svc"
 	rgwRadosPoolPgNum     = "8"
-	cosiUserName          = "cosi"
-	cosiUserCaps          = "buckets=*;users=*"
 	rgwApplication        = "rgw"
 )
 
diff --git a/pkg/operator/ceph/object/user.go b/pkg/operator/ceph/object/user.go
index 7cb2b670b682..d435c29951f5 100644
--- a/pkg/operator/ceph/object/user.go
+++ b/pkg/operator/ceph/object/user.go
@@ -17,23 +17,13 @@ limitations under the License.
 package object
 
 import (
-	"context"
 	"encoding/json"
-	"fmt"
 	"strings"
 	"syscall"
 
 	"github.com/ceph/go-ceph/rgw/admin"
 	"github.com/pkg/errors"
-	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
-	"github.com/rook/rook/pkg/operator/k8sutil"
 	"github.com/rook/rook/pkg/util/exec"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
 const (
@@ -236,53 +226,3 @@ func DeleteUser(c *Context, id string, opts ...string) (string, error) {
 
 	return result, errors.Wrapf(err, "failed to delete s3 user uid=%q", id)
 }
-
-func GenerateCephUserSecretName(store, username string) string {
-	return fmt.Sprintf("rook-ceph-object-user-%s-%s", store, username)
-}
-
-func generateCephUserSecret(userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) *corev1.Secret {
-	secretName := GenerateCephUserSecretName(storeName, userConfig.ID)
-	// Store the keys in a secret
-	secrets := map[string]string{
-		"AccessKey": userConfig.Keys[0].AccessKey,
-		"SecretKey": userConfig.Keys[0].SecretKey,
-		"Endpoint":  endpoint,
-	}
-	if tlsSecretName != "" {
-		secrets["SSLCertSecretName"] = tlsSecretName
-	}
-	secret := &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      secretName,
-			Namespace: namespace,
-			Labels: map[string]string{
-				"app":               AppName,
-				"user":              userConfig.ID,
-				"rook_cluster":      namespace,
-				"rook_object_store": storeName,
-			},
-		},
-		StringData: secrets,
-		Type:       k8sutil.RookType,
-	}
-	return secret
-}
-
-func ReconcileCephUserSecret(ctx context.Context, k8sclient client.Client, scheme *runtime.Scheme, ownerRef metav1.Object, userConfig *admin.User, endpoint, namespace, storeName, tlsSecretName string) (reconcile.Result, error) {
-	// Generate Kubernetes Secret
-	secret := generateCephUserSecret(userConfig, endpoint, namespace, storeName, tlsSecretName)
-
-	// Set owner ref to the object store user object
-	err := controllerutil.SetControllerReference(ownerRef, secret, scheme)
-	if err != nil {
-		return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of ceph object user secret %q", secret.Name)
-	}
-
-	// Create Kubernetes Secret
-	err = opcontroller.CreateOrUpdateObject(ctx, k8sclient, secret)
-	if err != nil {
-		return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph object user %q secret", secret.Name)
-	}
-	return reconcile.Result{}, nil
-}
diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go
index 2ff73e3d85b2..f4f910d051ac 100644
--- a/pkg/operator/ceph/object/user/controller.go
+++ b/pkg/operator/ceph/object/user/controller.go
@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -265,7 +266,7 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci
 	}
 
 	tlsSecretName := store.Spec.Gateway.SSLCertificateRef
-	reconcileResponse, err = object.ReconcileCephUserSecret(r.opManagerContext, r.client, r.scheme, cephObjectStoreUser, r.userConfig, r.advertiseEndpoint, cephObjectStoreUser.Namespace, cephObjectStoreUser.Spec.Store, tlsSecretName)
+	reconcileResponse, err = r.reconcileCephUserSecret(cephObjectStoreUser, tlsSecretName)
 	if err != nil {
 		r.updateStatus(k8sutil.ObservedGenerationNotAvailable, request.NamespacedName, k8sutil.ReconcileFailedStatus)
 		return reconcileResponse, *cephObjectStoreUser, err
@@ -509,12 +510,62 @@ func generateUserConfig(user *cephv1.CephObjectStoreUser) admin.User {
 	return userConfig
 }
 
+func generateCephUserSecretName(u *cephv1.CephObjectStoreUser) string {
+	return fmt.Sprintf("rook-ceph-object-user-%s-%s", u.Spec.Store, u.Name)
+}
+
 func generateStatusInfo(u *cephv1.CephObjectStoreUser) map[string]string {
 	m := make(map[string]string)
-	m["secretName"] = object.GenerateCephUserSecretName(u.Spec.Store, u.Name)
+	m["secretName"] = generateCephUserSecretName(u)
 	return m
 }
 
+func (r *ReconcileObjectStoreUser) generateCephUserSecret(u *cephv1.CephObjectStoreUser, tlsSecretName string) *corev1.Secret {
+	// Store the keys in a secret
+	secrets := map[string]string{
+		"AccessKey": r.userConfig.Keys[0].AccessKey,
+		"SecretKey": r.userConfig.Keys[0].SecretKey,
+		"Endpoint":  r.objContext.Endpoint,
+	}
+	if tlsSecretName != "" {
+		secrets["SSLCertSecretName"] = tlsSecretName
+	}
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      generateCephUserSecretName(u),
+			Namespace: u.Namespace,
+			Labels: map[string]string{
+				"app":               appName,
+				"user":              u.Name,
+				"rook_cluster":      u.Namespace,
+				"rook_object_store": u.Spec.Store,
+			},
+		},
+		StringData: secrets,
+		Type:       k8sutil.RookType,
+	}
+	return secret
+}
+
+func (r *ReconcileObjectStoreUser) reconcileCephUserSecret(cephObjectStoreUser *cephv1.CephObjectStoreUser, tlsSecretName string) (reconcile.Result, error) {
+	// Generate Kubernetes Secret
+	secret := r.generateCephUserSecret(cephObjectStoreUser, tlsSecretName)
+
+	// Set owner ref to the object store user object
+	err := controllerutil.SetControllerReference(cephObjectStoreUser, secret, r.scheme)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to set owner reference of ceph object user secret %q", secret.Name)
+	}
+
+	// Create Kubernetes Secret
+	err = opcontroller.CreateOrUpdateObject(r.opManagerContext, r.client, secret)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrapf(err, "failed to create or update ceph object user %q secret", secret.Name)
+	}
+
+	return reconcile.Result{}, nil
+}
+
 func (r *ReconcileObjectStoreUser) objectStoreInitialized(cephObjectStoreUser *cephv1.CephObjectStoreUser) error {
 	cephObjectStore, err := r.getObjectStore(cephObjectStoreUser.Spec.Store)
 	if err != nil {
diff --git a/tests/integration/ceph_cosi_test.go b/tests/integration/ceph_cosi_test.go
index a4cfa8050941..3cb4fc73f214 100644
--- a/tests/integration/ceph_cosi_test.go
+++ b/tests/integration/ceph_cosi_test.go
@@ -6,7 +6,6 @@ import (
 	"time"
 
 	"github.com/rook/rook/pkg/daemon/ceph/client"
-	"github.com/rook/rook/pkg/operator/ceph/object"
 	rgw "github.com/rook/rook/pkg/operator/ceph/object"
 	"github.com/rook/rook/tests/framework/clients"
 	"github.com/rook/rook/tests/framework/installer"
@@ -56,7 +55,8 @@ func testCOSIDriver(s *suite.Suite, helper *clients.TestClient, k8sh *utils.K8sH
 		assert.NoError(t, k8sh.WaitForLabeledDeploymentsToBeReady("app=ceph-cosi-driver", operatorNamespace))
 	})
 
-	objectStoreUserSecretName := object.GenerateCephUserSecretName(objectStoreCOSI, cosiUser)
+	createCephObjectUser(s, helper, k8sh, namespace, objectStoreCOSI, cosiUser, true)
+	objectStoreUserSecretName := "rook-ceph-object-user" + "-" + objectStoreCOSI + "-" + cosiUser
 	t.Run("Creating BucketClass", func(t *testing.T) {
 		err := helper.COSIClient.CreateBucketClass(bucketClassName, objectStoreUserSecretName, deletionPolicy)
 		assert.NoError(t, err, "failed to create BucketClass")

From 506407b5eab0598c5ee9e06358b136a6ec570f5f Mon Sep 17 00:00:00 2001
From: Travis Nielsen <tnielsen@redhat.com>
Date: Fri, 22 Nov 2024 09:41:43 -0700
Subject: [PATCH 02/24] tests: upgrade from rook 1.15 to master

The upgrade tests in master have been upgrading from 1.14 to
master. In anticipation of the v1.16 release, we change
the upgrade tests to start from v1.15 to test if there
are any regressions in the supported upgrades.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
---
 tests/framework/installer/ceph_installer.go          |  4 ++--
 tests/framework/installer/ceph_manifests.go          |  2 +-
 tests/framework/installer/ceph_manifests_previous.go |  2 +-
 tests/integration/ceph_upgrade_test.go               | 10 +++++-----
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go
index 42adcebe7d2d..f1f89b5d9a0f 100644
--- a/tests/framework/installer/ceph_installer.go
+++ b/tests/framework/installer/ceph_installer.go
@@ -267,8 +267,8 @@ func (h *CephInstaller) CreateCephCluster() error {
 
 func (h *CephInstaller) waitForCluster() error {
 	monWaitLabel := "app=rook-ceph-mon,mon_daemon=true"
-	if h.Manifests.Settings().RookVersion == Version1_14 {
-		// TODO: Remove this when upgrade test is from v1.15 since v1.14 does not have the mon_daemon label
+	if h.Manifests.Settings().RookVersion == Version1_15 {
+		// TODO: Remove this when upgrade test is from v1.15.7 since prior releases do not have the mon_daemon label
 		monWaitLabel = "app=rook-ceph-mon"
 	}
 	if err := h.k8shelper.WaitForPodCount(monWaitLabel, h.settings.Namespace, h.settings.Mons); err != nil {
diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go
index 43a72d9bd830..60b1ddac77a6 100644
--- a/tests/framework/installer/ceph_manifests.go
+++ b/tests/framework/installer/ceph_manifests.go
@@ -71,7 +71,7 @@ func NewCephManifests(settings *TestCephSettings) CephManifests {
 	switch settings.RookVersion {
 	case LocalBuildTag:
 		return &CephManifestsMaster{settings}
-	case Version1_14:
+	case Version1_15:
 		return &CephManifestsPreviousVersion{settings, &CephManifestsMaster{settings}}
 	}
 	panic(fmt.Errorf("unrecognized ceph manifest version: %s", settings.RookVersion))
diff --git a/tests/framework/installer/ceph_manifests_previous.go b/tests/framework/installer/ceph_manifests_previous.go
index 3d48fb66c67d..0b675c8ffe85 100644
--- a/tests/framework/installer/ceph_manifests_previous.go
+++ b/tests/framework/installer/ceph_manifests_previous.go
@@ -24,7 +24,7 @@ import (
 
 const (
 	// The version from which the upgrade test will start
-	Version1_14 = "v1.14.8"
+	Version1_15 = "v1.15.6"
 )
 
 // CephManifestsPreviousVersion wraps rook yaml definitions
diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go
index 849ad06eb148..6bd9793a8572 100644
--- a/tests/integration/ceph_upgrade_test.go
+++ b/tests/integration/ceph_upgrade_test.go
@@ -105,7 +105,7 @@ func (s *UpgradeSuite) TestUpgradeHelm() {
 }
 
 func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersionSpec) {
-	baseRookImage := installer.Version1_14
+	baseRookImage := installer.Version1_15
 	s.baseSetup(useHelm, baseRookImage, initialCephVersion)
 
 	objectUserID := "upgraded-user"
@@ -128,9 +128,9 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi
 	_ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, installer.ObjectStoreName, installer.ObjectStoreSCName, "Delete")
 
 	//
-	// Upgrade Rook from v1.13 to master
+	// Upgrade Rook from the last release branch to master
 	//
-	logger.Infof("*** UPGRADING ROOK FROM %s to master ***", installer.Version1_14)
+	logger.Infof("*** UPGRADING ROOK FROM %s to master ***", baseRookImage)
 	s.gatherLogs(s.settings.OperatorNamespace, "_before_master_upgrade")
 	s.upgradeToMaster()
 
@@ -139,7 +139,7 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi
 	err := s.installer.WaitForToolbox(s.namespace)
 	assert.NoError(s.T(), err)
 
-	logger.Infof("Done with automatic upgrade from %s to master", installer.Version1_14)
+	logger.Infof("Done with automatic upgrade from %s to master", baseRookImage)
 	newFile := "post-upgrade-previous-to-master-file"
 	s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead)
 	rbdFilesToRead = append(rbdFilesToRead, newFile)
@@ -151,7 +151,7 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi
 	// do not need retry b/c the OBC controller runs parallel to Rook-Ceph orchestration
 	assert.True(s.T(), s.helper.BucketClient.CheckOBC(obcName, "bound"))
 
-	logger.Infof("Verified upgrade from %s to master", installer.Version1_14)
+	logger.Infof("Verified upgrade from %s to master", baseRookImage)
 
 	// SKIP the Ceph version upgrades for the helm test
 	if s.settings.UseHelm {

From fb682e2e24970ecbf60c87f0dc77a0c8e522342d Mon Sep 17 00:00:00 2001
From: Zac Dover <zac.dover@proton.me>
Date: Wed, 4 Dec 2024 13:23:28 +0100
Subject: [PATCH 03/24] doc: block-storage.md: Add formatting to namespaces

Add formatting to some namespace names and provisioner values in order
to make the
Storage-Configuration/Block-Storage-RBD/block-storage/provision-storage
section more legible.

Signed-off-by: Zac Dover <zac.dover@proton.me>
---
 .../Block-Storage-RBD/block-storage.md                    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md b/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md
index 4e1ecfc5b3d8..21e3676ba3f9 100644
--- a/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md
+++ b/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md
@@ -93,10 +93,10 @@ reclaimPolicy: Delete
 allowVolumeExpansion: true
 ```
 
-If you've deployed the Rook operator in a namespace other than "rook-ceph",
-change the prefix in the provisioner to match the namespace
-you used. For example, if the Rook operator is running in the namespace "my-namespace" the
-provisioner value should be "my-namespace.rbd.csi.ceph.com".
+If you've deployed the Rook operator in a namespace other than `rook-ceph`,
+change the prefix in the provisioner to match the namespace you used. For
+example, if the Rook operator is running in the namespace `my-namespace` the
+provisioner value should be `my-namespace.rbd.csi.ceph.com`.
 
 Create the storage class.
 

From 675678fad62807b5d2356e16c399a0251ccb6553 Mon Sep 17 00:00:00 2001
From: Tobias Wolf <wolf@b1-systems.de>
Date: Mon, 25 Nov 2024 16:45:48 +0100
Subject: [PATCH 04/24] file: add support for named mds metadata pool names

This commit adds optional support to specify the MDS metadata pool name. It defaults to `<fsName>-metadata` as current implementation expects but allows customization if needed e.g. if exisiting naming conventions used `<fsName>_metadata`. Additionally an "preservePoolNames" boolean has been added to indicate that no generated pool names should be used.

Signed-off-by: Tobias Wolf <wolf@b1-systems.de>
---
 .../Shared-Filesystem/ceph-filesystem-crd.md  |  8 +++-
 Documentation/CRDs/specification.md           | 34 ++++++++++++--
 .../charts/rook-ceph/templates/resources.yaml |  6 +++
 deploy/examples/crds.yaml                     |  6 +++
 pkg/apis/ceph.rook.io/v1/types.go             |  6 ++-
 .../disruption/clusterdisruption/pools.go     |  2 +-
 pkg/operator/ceph/file/filesystem.go          | 47 +++++++++++++------
 pkg/operator/ceph/file/filesystem_test.go     | 26 +++++++++-
 8 files changed, 111 insertions(+), 24 deletions(-)

diff --git a/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md b/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md
index cc87c6658b29..157f90181daf 100644
--- a/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md
+++ b/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md
@@ -111,8 +111,10 @@ Also see an example in the [`storageclass-ec.yaml`](https://github.com/rook/rook
 The pools allow all of the settings defined in the Pool CRD spec. For more details, see the [Pool CRD](../Block-Storage/ceph-block-pool-crd.md) settings. In the example above, there must be at least three hosts (size 3) and at least eight devices (6 data + 2 coding chunks) in the cluster.
 
 * `metadataPool`: The settings used to create the filesystem metadata pool. Must use replication.
+    * `name`: (optional) Override the default generated name of the metadata pool.
 * `dataPools`: The settings to create the filesystem data pools. Optionally (and we highly recommend), a pool name can be specified with the `name` field to override the default generated name; see more below. If multiple pools are specified, Rook will add the pools to the filesystem. Assigning users or files to a pool is left as an exercise for the reader with the [CephFS documentation](http://docs.ceph.com/docs/master/cephfs/file-layouts/). The data pools can use replication or erasure coding. If erasure coding pools are specified, the cluster must be running with bluestore enabled on the OSDs.
-    * `name`: (optional, and highly recommended) Override the default generated name of the pool. The final pool name will consist of the filesystem name and pool name, e.g., `<fsName>-<poolName>`. We highly recommend to specify `name` to prevent issues that can arise from modifying the spec in a way that causes Rook to lose the original pool ordering.
+    * `name`: (optional, and highly recommended) Override the default generated name of the pool. We highly recommend to specify `name` to prevent issues that can arise from modifying the spec in a way that causes Rook to lose the original pool ordering.
+* `preservePoolNames`: Preserve pool names as specified.
 * `preserveFilesystemOnDelete`: If it is set to 'true' the filesystem will remain when the
     CephFilesystem resource is deleted. This is a security measure to avoid loss of data if the
     CephFilesystem resource is deleted accidentally. The default value is 'false'. This option
@@ -121,6 +123,10 @@ The pools allow all of the settings defined in the Pool CRD spec. For more detai
     `preserveFilesystemOnDelete`. For backwards compatibility and upgradeability, if this is set to
     'true', Rook will treat `preserveFilesystemOnDelete` as being set to 'true'.
 
+### Generated Pool Names
+
+Both `metadataPool` and `dataPools` support defining names as required. The final pool name will consist of the filesystem name and pool name, e.g., `<fsName>-<poolName>` or `<fsName>-metadata` for `metadataPool`. For more granular configuration you may want to set `preservePoolNames` to `true` in `pools` to disable generation of names. In that case all pool names defined are used as given.
+
 ## Metadata Server Settings
 
 The metadata server settings correspond to the MDS daemon settings.
diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
index 4bbadf471af3..a6dadccec601 100644
--- a/Documentation/CRDs/specification.md
+++ b/Documentation/CRDs/specification.md
@@ -1127,8 +1127,8 @@ FilesystemSpec
 <td>
 <code>metadataPool</code><br/>
 <em>
-<a href="#ceph.rook.io/v1.PoolSpec">
-PoolSpec
+<a href="#ceph.rook.io/v1.NamedPoolSpec">
+NamedPoolSpec
 </a>
 </em>
 </td>
@@ -1151,6 +1151,18 @@ PoolSpec
 </tr>
 <tr>
 <td>
+<code>preservePoolNames</code><br/>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Preserve pool names as specified</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>preservePoolsOnDelete</code><br/>
 <em>
 bool
@@ -6582,8 +6594,8 @@ FilesystemSnapshotScheduleStatusRetention
 <td>
 <code>metadataPool</code><br/>
 <em>
-<a href="#ceph.rook.io/v1.PoolSpec">
-PoolSpec
+<a href="#ceph.rook.io/v1.NamedPoolSpec">
+NamedPoolSpec
 </a>
 </em>
 </td>
@@ -6606,6 +6618,18 @@ PoolSpec
 </tr>
 <tr>
 <td>
+<code>preservePoolNames</code><br/>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Preserve pool names as specified</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>preservePoolsOnDelete</code><br/>
 <em>
 bool
@@ -11076,7 +11100,7 @@ This list allows defining additional StorageClasses on top of default STANDARD s
 <h3 id="ceph.rook.io/v1.PoolSpec">PoolSpec
 </h3>
 <p>
-(<em>Appears on:</em><a href="#ceph.rook.io/v1.FilesystemSpec">FilesystemSpec</a>, <a href="#ceph.rook.io/v1.NamedBlockPoolSpec">NamedBlockPoolSpec</a>, <a href="#ceph.rook.io/v1.NamedPoolSpec">NamedPoolSpec</a>, <a href="#ceph.rook.io/v1.ObjectStoreSpec">ObjectStoreSpec</a>, <a href="#ceph.rook.io/v1.ObjectZoneSpec">ObjectZoneSpec</a>)
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.NamedBlockPoolSpec">NamedBlockPoolSpec</a>, <a href="#ceph.rook.io/v1.NamedPoolSpec">NamedPoolSpec</a>, <a href="#ceph.rook.io/v1.ObjectStoreSpec">ObjectStoreSpec</a>, <a href="#ceph.rook.io/v1.ObjectZoneSpec">ObjectZoneSpec</a>)
 </p>
 <div>
 <p>PoolSpec represents the spec of ceph pool</p>
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index aaa8893614d1..ecca30883c72 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -7204,6 +7204,9 @@ spec:
                             type: object
                           type: array
                       type: object
+                    name:
+                      description: Name of the pool
+                      type: string
                     parameters:
                       additionalProperties:
                         type: string
@@ -8236,6 +8239,9 @@ spec:
                 preserveFilesystemOnDelete:
                   description: Preserve the fs in the cluster on CephFilesystem CR deletion. Setting this to true automatically implies PreservePoolsOnDelete is true.
                   type: boolean
+                preservePoolNames:
+                  description: Preserve pool names as specified
+                  type: boolean
                 preservePoolsOnDelete:
                   description: Preserve pools on filesystem deletion
                   type: boolean
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 721ce8af19a1..a211a9c34f93 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -7199,6 +7199,9 @@ spec:
                             type: object
                           type: array
                       type: object
+                    name:
+                      description: Name of the pool
+                      type: string
                     parameters:
                       additionalProperties:
                         type: string
@@ -8231,6 +8234,9 @@ spec:
                 preserveFilesystemOnDelete:
                   description: Preserve the fs in the cluster on CephFilesystem CR deletion. Setting this to true automatically implies PreservePoolsOnDelete is true.
                   type: boolean
+                preservePoolNames:
+                  description: Preserve pool names as specified
+                  type: boolean
                 preservePoolsOnDelete:
                   description: Preserve pools on filesystem deletion
                   type: boolean
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 81035abb8589..7779128a3bfb 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -1168,12 +1168,16 @@ type CephFilesystemList struct {
 type FilesystemSpec struct {
 	// The metadata pool settings
 	// +nullable
-	MetadataPool PoolSpec `json:"metadataPool"`
+	MetadataPool NamedPoolSpec `json:"metadataPool"`
 
 	// The data pool settings, with optional predefined pool name.
 	// +nullable
 	DataPools []NamedPoolSpec `json:"dataPools"`
 
+	// Preserve pool names as specified
+	// +optional
+	PreservePoolNames bool `json:"preservePoolNames,omitempty"`
+
 	// Preserve pools on filesystem deletion
 	// +optional
 	PreservePoolsOnDelete bool `json:"preservePoolsOnDelete,omitempty"`
diff --git a/pkg/operator/ceph/disruption/clusterdisruption/pools.go b/pkg/operator/ceph/disruption/clusterdisruption/pools.go
index ee0240a94f62..fe25ad9c371a 100644
--- a/pkg/operator/ceph/disruption/clusterdisruption/pools.go
+++ b/pkg/operator/ceph/disruption/clusterdisruption/pools.go
@@ -52,7 +52,7 @@ func (r *ReconcileClusterDisruption) processPools(request reconcile.Request) (*c
 	}
 	poolCount += len(cephFilesystemList.Items)
 	for _, cephFilesystem := range cephFilesystemList.Items {
-		poolSpecs = append(poolSpecs, cephFilesystem.Spec.MetadataPool)
+		poolSpecs = append(poolSpecs, cephFilesystem.Spec.MetadataPool.PoolSpec)
 		for _, pool := range cephFilesystem.Spec.DataPools {
 			poolSpecs = append(poolSpecs, pool.PoolSpec)
 		}
diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go
index b8e6b36cf303..d7f3cb20a807 100644
--- a/pkg/operator/ceph/file/filesystem.go
+++ b/pkg/operator/ceph/file/filesystem.go
@@ -152,12 +152,13 @@ func validateFilesystem(context *clusterd.Context, clusterInfo *cephclient.Clust
 		}
 	}
 
-	if err := cephpool.ValidatePoolSpec(context, clusterInfo, clusterSpec, &f.Spec.MetadataPool); err != nil {
+	localMetadataPoolSpec := f.Spec.MetadataPool.PoolSpec
+	if err := cephpool.ValidatePoolSpec(context, clusterInfo, clusterSpec, &localMetadataPoolSpec); err != nil {
 		return errors.Wrap(err, "invalid metadata pool")
 	}
 	for _, p := range f.Spec.DataPools {
-		localpoolSpec := p.PoolSpec
-		if err := cephpool.ValidatePoolSpec(context, clusterInfo, clusterSpec, &localpoolSpec); err != nil {
+		localPoolSpec := p.PoolSpec
+		if err := cephpool.ValidatePoolSpec(context, clusterInfo, clusterSpec, &localPoolSpec); err != nil {
 			return errors.Wrap(err, "Invalid data pool")
 		}
 	}
@@ -190,12 +191,10 @@ func newFS(name, namespace string) *Filesystem {
 
 // createOrUpdatePools function sets the sizes for MetadataPool and dataPool
 func createOrUpdatePools(f *Filesystem, context *clusterd.Context, clusterInfo *cephclient.ClusterInfo, clusterSpec *cephv1.ClusterSpec, spec cephv1.FilesystemSpec) error {
-	// generating the metadata pool's name
-	metadataPool := cephv1.NamedPoolSpec{
-		Name:     GenerateMetaDataPoolName(f.Name),
-		PoolSpec: spec.MetadataPool,
-	}
+	metadataPool := spec.MetadataPool
 	metadataPool.Application = cephfsApplication
+	metadataPool.Name = generateMetaDataPoolName(f.Name, &spec)
+
 	err := cephclient.CreatePool(context, clusterInfo, clusterSpec, &metadataPool)
 	if err != nil {
 		return errors.Wrapf(err, "failed to update metadata pool %q", metadataPool.Name)
@@ -264,11 +263,10 @@ func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, clusterInfo *
 		reversedPoolMap[value] = key
 	}
 
-	spec.MetadataPool.Application = cephfsApplication
-	metadataPool := cephv1.NamedPoolSpec{
-		Name:     GenerateMetaDataPoolName(f.Name),
-		PoolSpec: spec.MetadataPool,
-	}
+	metadataPool := spec.MetadataPool
+	metadataPool.Application = cephfsApplication
+	metadataPool.Name = generateMetaDataPoolName(f.Name, &spec)
+
 	if _, poolFound := reversedPoolMap[metadataPool.Name]; !poolFound {
 		err = cephclient.CreatePool(context, clusterInfo, clusterSpec, &metadataPool)
 		if err != nil {
@@ -320,19 +318,40 @@ func downFilesystem(context *clusterd.Context, clusterInfo *cephclient.ClusterIn
 // or get predefined name from spec
 func generateDataPoolNames(f *Filesystem, spec cephv1.FilesystemSpec) []string {
 	var dataPoolNames []string
+
 	for i, pool := range spec.DataPools {
 		poolName := ""
+
 		if pool.Name == "" {
 			poolName = fmt.Sprintf("%s-%s%d", f.Name, dataPoolSuffix, i)
+		} else if spec.PreservePoolNames {
+			poolName = pool.Name
 		} else {
 			poolName = fmt.Sprintf("%s-%s", f.Name, pool.Name)
 		}
+
 		dataPoolNames = append(dataPoolNames, poolName)
 	}
+
 	return dataPoolNames
 }
 
 // GenerateMetaDataPoolName generates MetaDataPool name by prefixing the filesystem name to the constant metaDataPoolSuffix
 func GenerateMetaDataPoolName(fsName string) string {
-	return fmt.Sprintf("%s-%s", fsName, metaDataPoolSuffix)
+	return generateMetaDataPoolName(fsName, nil)
+}
+
+// GenerateMetaDataPoolName generates MetaDataPool name as specified in FilesystemSpec
+func generateMetaDataPoolName(fsName string, spec *cephv1.FilesystemSpec) string {
+	poolName := ""
+
+	if nil == spec || spec.MetadataPool.Name == "" {
+		poolName = fmt.Sprintf("%s-%s", fsName, metaDataPoolSuffix)
+	} else if spec.PreservePoolNames {
+		poolName = spec.MetadataPool.Name
+	} else {
+		poolName = fmt.Sprintf("%s-%s", fsName, spec.MetadataPool.Name)
+	}
+
+	return poolName
 }
diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go
index 75f33ade9f32..8f1e110b47b1 100644
--- a/pkg/operator/ceph/file/filesystem_test.go
+++ b/pkg/operator/ceph/file/filesystem_test.go
@@ -63,7 +63,7 @@ func TestValidateSpec(t *testing.T) {
 
 	// missing metadata pool
 	assert.NotNil(t, validateFilesystem(context, clusterInfo, clusterSpec, fs))
-	fs.Spec.MetadataPool = p
+	fs.Spec.MetadataPool.PoolSpec = p
 
 	// missing mds count
 	assert.NotNil(t, validateFilesystem(context, clusterInfo, clusterSpec, fs))
@@ -112,6 +112,26 @@ func TestGenerateDataPoolNames(t *testing.T) {
 	assert.Equal(t, expectedNames, names)
 }
 
+func TestPreservePoolNames(t *testing.T) {
+	fs := &Filesystem{Name: "fake", Namespace: "fake"}
+	fsSpec := cephv1.FilesystemSpec{
+		DataPools: []cephv1.NamedPoolSpec{
+			{
+				PoolSpec: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}},
+			},
+			{
+				Name:     "somename",
+				PoolSpec: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}},
+			},
+		},
+		PreservePoolNames: true,
+	}
+
+	expectedNames := []string{"fake-data0", "somename"}
+	names := generateDataPoolNames(fs, fsSpec)
+	assert.Equal(t, expectedNames, names)
+}
+
 func isBasePoolOperation(fsName, command string, args []string) bool {
 	if reflect.DeepEqual(args[0:7], []string{"osd", "pool", "create", fsName + "-metadata", "0", "replicated", fsName + "-metadata"}) {
 		return true
@@ -325,7 +345,9 @@ func fsTest(fsName string) cephv1.CephFilesystem {
 	return cephv1.CephFilesystem{
 		ObjectMeta: metav1.ObjectMeta{Name: fsName, Namespace: "ns"},
 		Spec: cephv1.FilesystemSpec{
-			MetadataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}},
+			MetadataPool: cephv1.NamedPoolSpec{
+				PoolSpec: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}},
+			},
 			DataPools: []cephv1.NamedPoolSpec{
 				{
 					PoolSpec: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}},

From 0649604355f9c9a13420ce0fa1bd90a83ee990ec Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Dec 2024 12:06:50 +0000
Subject: [PATCH 05/24] build(deps): bump golang.org/x/sync from 0.9.0 to
 0.10.0

Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.9.0 to 0.10.0.
- [Commits](https://github.com/golang/sync/compare/v0.9.0...v0.10.0)

---
updated-dependencies:
- dependency-name: golang.org/x/sync
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 399c08ba171b..c150365ddd6f 100644
--- a/go.mod
+++ b/go.mod
@@ -42,7 +42,7 @@ require (
 	go.uber.org/automaxprocs v1.6.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
-	golang.org/x/sync v0.9.0
+	golang.org/x/sync v0.10.0
 	gopkg.in/ini.v1 v1.67.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.31.2
diff --git a/go.sum b/go.sum
index 26776756e53f..cc6b53124cfa 100644
--- a/go.sum
+++ b/go.sum
@@ -1131,8 +1131,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
-golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

From b2623b0b21a2b9763fcd6da95b75077234af4011 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Dec 2024 12:24:22 +0000
Subject: [PATCH 06/24] build(deps): bump reviewdog/action-misspell from 1.23.0
 to 1.26.1

Bumps [reviewdog/action-misspell](https://github.com/reviewdog/action-misspell) from 1.23.0 to 1.26.1.
- [Release notes](https://github.com/reviewdog/action-misspell/releases)
- [Commits](https://github.com/reviewdog/action-misspell/compare/ef8b22c1cca06c8d306fc6be302c3dab0f6ca12f...18ffb61effb93b47e332f185216be7e49592e7e1)

---
updated-dependencies:
- dependency-name: reviewdog/action-misspell
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/codespell.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/codespell.yaml b/.github/workflows/codespell.yaml
index e5aaebd88294..21ca5047efaa 100644
--- a/.github/workflows/codespell.yaml
+++ b/.github/workflows/codespell.yaml
@@ -57,4 +57,4 @@ jobs:
         with:
           fetch-depth: 0
       - name: misspell
-        uses: reviewdog/action-misspell@ef8b22c1cca06c8d306fc6be302c3dab0f6ca12f # v1.23.0
+        uses: reviewdog/action-misspell@18ffb61effb93b47e332f185216be7e49592e7e1 # v1.26.1

From 355ae3d29d9a8b9957d9153002a541cc4c90dad4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 9 Dec 2024 12:24:28 +0000
Subject: [PATCH 07/24] build(deps): bump github/codeql-action from 3.27.5 to
 3.27.6

Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.27.5 to 3.27.6.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/f09c1c0a94de965c15400f5634aa42fac8fb8f88...aa578102511db1f4524ed59b8cc2bae4f6e88195)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/scorecards.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml
index 3b31ef3e5748..adfabda8f3a0 100644
--- a/.github/workflows/scorecards.yml
+++ b/.github/workflows/scorecards.yml
@@ -64,6 +64,6 @@ jobs:
       # Upload the results to GitHub's code scanning dashboard (optional).
       # Commenting out will disable upload of results to your repo's Code Scanning dashboard
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6
         with:
           sarif_file: results.sarif

From 9bf562c25381c914ae4672b9fe4a8e9df6b3cbcf Mon Sep 17 00:00:00 2001
From: Blaine Gardner <blaine.gardner@ibm.com>
Date: Mon, 9 Dec 2024 11:57:08 -0700
Subject: [PATCH 08/24] docs: update guides and examples for v1.16 release

Update upgrade guides, docs, and exmaple manifests for the Rook v1.16
release.

Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>
---
 Documentation/Upgrade/ceph-upgrade.md | 10 +++---
 Documentation/Upgrade/rook-upgrade.md | 49 +++++++++++++++------------
 2 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/Documentation/Upgrade/ceph-upgrade.md b/Documentation/Upgrade/ceph-upgrade.md
index 5205eac2b7cd..c4adae8be0ad 100644
--- a/Documentation/Upgrade/ceph-upgrade.md
+++ b/Documentation/Upgrade/ceph-upgrade.md
@@ -39,7 +39,7 @@ Official Ceph container images can be found on [Quay](https://quay.io/repository
 
 These images are tagged in a few ways:
 
-* The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v18.2.4-20240724`).
+* The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v19.2.0-20240927`).
     These tags are recommended for production clusters, as there is no possibility for the cluster to
     be heterogeneous with respect to the version of Ceph running in containers.
 * Ceph major version tags (e.g., `v18`) are useful for development and test clusters so that the
@@ -56,7 +56,7 @@ CephCluster CRD (`spec.cephVersion.image`).
 
 ```console
 ROOK_CLUSTER_NAMESPACE=rook-ceph
-NEW_CEPH_IMAGE='quay.io/ceph/ceph:v18.2.4-20240724'
+NEW_CEPH_IMAGE='quay.io/ceph/ceph:v19.2.0-20240927'
 kubectl -n $ROOK_CLUSTER_NAMESPACE patch CephCluster $ROOK_CLUSTER_NAMESPACE --type=merge -p "{\"spec\": {\"cephVersion\": {\"image\": \"$NEW_CEPH_IMAGE\"}}}"
 ```
 
@@ -68,7 +68,7 @@ employed by the new Rook operator release. Employing an outdated Ceph version wi
 in unexpected behaviour.
 
 ```console
-kubectl -n rook-ceph set image deploy/rook-ceph-tools rook-ceph-tools=quay.io/ceph/ceph:v18.2.4-20240724
+kubectl -n rook-ceph set image deploy/rook-ceph-tools rook-ceph-tools=quay.io/ceph/ceph:v19.2.0-20240927
 ```
 
 #### **3. Wait for the pod updates**
@@ -85,10 +85,10 @@ Confirm the upgrade is completed when the versions are all on the desired Ceph v
 ```console
 kubectl -n $ROOK_CLUSTER_NAMESPACE get deployment -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{"ceph-version="}{.metadata.labels.ceph-version}{"\n"}{end}' | sort | uniq
 This cluster is not yet finished:
-    ceph-version=v17.2.7-0
     ceph-version=v18.2.4-0
+    ceph-version=v19.2.0-0
 This cluster is finished:
-    ceph-version=v18.2.4-0
+    ceph-version=v19.2.0-0
 ```
 
 #### **4. Verify cluster health**
diff --git a/Documentation/Upgrade/rook-upgrade.md b/Documentation/Upgrade/rook-upgrade.md
index 9d6bc6e6bdad..f59cd1df13b9 100644
--- a/Documentation/Upgrade/rook-upgrade.md
+++ b/Documentation/Upgrade/rook-upgrade.md
@@ -14,7 +14,7 @@ We welcome feedback and opening issues!
 
 ## Supported Versions
 
-This guide is for upgrading from **Rook v1.14.x to Rook v1.15.x**.
+This guide is for upgrading from **Rook v1.15.x to Rook v1.16.x**.
 
 Please refer to the upgrade guides from previous releases for supported upgrade paths.
 Rook upgrades are only supported between official releases.
@@ -22,6 +22,7 @@ Rook upgrades are only supported between official releases.
 For a guide to upgrade previous versions of Rook, please refer to the version of documentation for
 those releases.
 
+* [Upgrade 1.14 to 1.15](https://rook.io/docs/rook/v1.15/Upgrade/rook-upgrade/)
 * [Upgrade 1.13 to 1.14](https://rook.io/docs/rook/v1.14/Upgrade/rook-upgrade/)
 * [Upgrade 1.12 to 1.13](https://rook.io/docs/rook/v1.13/Upgrade/rook-upgrade/)
 * [Upgrade 1.11 to 1.12](https://rook.io/docs/rook/v1.12/Upgrade/rook-upgrade/)
@@ -51,6 +52,12 @@ those releases.
 
 ## Breaking changes in v1.16
 
+* The minimum supported Kubernetes version is v1.27.
+
+* Rook no longer supports Ceph v17 (Quincy) as it is end of life. Rook v1.15 clusters should
+    [upgrade Ceph](./ceph-upgrade.md) to at least Ceph v18 (Reef) before upgrading Rook to v1.16.
+    Be aware of known [Ceph v18.2.4 ARM issues](https://github.com/rook/rook/issues/14502).
+
 * Rook fully deprecated CSI "holder" pods in Rook v1.16. Any Rook v1.15 CephCluster with
     `csi-*plugin-holder-*` pods present in the Rook operator namespace must follow holder pod
     removal migration steps outlined in
@@ -70,11 +77,11 @@ With this upgrade guide, there are a few notes to consider:
 
 Unless otherwise noted due to extenuating requirements, upgrades from one patch release of Rook to
 another are as simple as updating the common resources and the image of the Rook operator. For
-example, when Rook v1.15.1 is released, the process of updating from v1.15.0 is as simple as running
+example, when Rook v1.16.1 is released, the process of updating from v1.16.0 is as simple as running
 the following:
 
 ```console
-git clone --single-branch --depth=1 --branch v1.15.1 https://github.com/rook/rook.git
+git clone --single-branch --depth=1 --branch v1.16.1 https://github.com/rook/rook.git
 cd rook/deploy/examples
 ```
 
@@ -82,11 +89,11 @@ If the Rook Operator or CephCluster are deployed into a different namespace than
 `rook-ceph`, see the [Update common resources and CRDs](#1-update-common-resources-and-crds)
 section for instructions on how to change the default namespaces in `common.yaml`.
 
-Then, apply the latest changes from v1.15, and update the Rook Operator image.
+Then, apply the latest changes from v1.16, and update the Rook Operator image.
 
 ```console
 kubectl apply -f common.yaml -f crds.yaml
-kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.15.1
+kubectl -n rook-ceph set image deploy/rook-ceph-operator rook-ceph-operator=rook/ceph:v1.16.1
 ```
 
 As exemplified above, it is a good practice to update Rook common resources from the example
@@ -120,9 +127,9 @@ In order to successfully upgrade a Rook cluster, the following prerequisites mus
 
 ## Rook Operator Upgrade
 
-The examples given in this guide upgrade a live Rook cluster running `v1.14.9` to
-the version `v1.15.0`. This upgrade should work from any official patch release of Rook v1.14 to any
-official patch release of v1.15.
+The examples given in this guide upgrade a live Rook cluster running `v1.15.6` to
+the version `v1.16.0`. This upgrade should work from any official patch release of Rook v1.15 to any
+official patch release of v1.16.
 
 Let's get started!
 
@@ -183,7 +190,7 @@ kubectl apply -f deploy/examples/monitoring/rbac.yaml
 !!! hint
     The operator is automatically updated when using Helm charts.
 
-The largest portion of the upgrade is triggered when the operator's image is updated to `v1.15.x`.
+The largest portion of the upgrade is triggered when the operator's image is updated to `v1.16.x`.
 When the operator is updated, it will proceed to update all of the Ceph daemons.
 
 ```console
@@ -217,18 +224,18 @@ watch --exec kubectl -n $ROOK_CLUSTER_NAMESPACE get deployments -l rook_cluster=
 ```
 
 As an example, this cluster is midway through updating the OSDs. When all deployments report `1/1/1`
-availability and `rook-version=v1.15.0`, the Ceph cluster's core components are fully updated.
+availability and `rook-version=v1.16.0`, the Ceph cluster's core components are fully updated.
 
 ```console
 Every 2.0s: kubectl -n rook-ceph get deployment -o j...
 
-rook-ceph-mgr-a         req/upd/avl: 1/1/1      rook-version=v1.15.0
-rook-ceph-mon-a         req/upd/avl: 1/1/1      rook-version=v1.15.0
-rook-ceph-mon-b         req/upd/avl: 1/1/1      rook-version=v1.15.0
-rook-ceph-mon-c         req/upd/avl: 1/1/1      rook-version=v1.15.0
-rook-ceph-osd-0         req/upd/avl: 1//        rook-version=v1.15.0
-rook-ceph-osd-1         req/upd/avl: 1/1/1      rook-version=v1.14.9
-rook-ceph-osd-2         req/upd/avl: 1/1/1      rook-version=v1.14.9
+rook-ceph-mgr-a         req/upd/avl: 1/1/1      rook-version=v1.16.0
+rook-ceph-mon-a         req/upd/avl: 1/1/1      rook-version=v1.16.0
+rook-ceph-mon-b         req/upd/avl: 1/1/1      rook-version=v1.16.0
+rook-ceph-mon-c         req/upd/avl: 1/1/1      rook-version=v1.16.0
+rook-ceph-osd-0         req/upd/avl: 1//        rook-version=v1.16.0
+rook-ceph-osd-1         req/upd/avl: 1/1/1      rook-version=v1.15.6
+rook-ceph-osd-2         req/upd/avl: 1/1/1      rook-version=v1.15.6
 ```
 
 An easy check to see if the upgrade is totally finished is to check that there is only one
@@ -237,14 +244,14 @@ An easy check to see if the upgrade is totally finished is to check that there i
 ```console
 # kubectl -n $ROOK_CLUSTER_NAMESPACE get deployment -l rook_cluster=$ROOK_CLUSTER_NAMESPACE -o jsonpath='{range .items[*]}{"rook-version="}{.metadata.labels.rook-version}{"\n"}{end}' | sort | uniq
 This cluster is not yet finished:
-  rook-version=v1.14.9
-  rook-version=v1.15.0
+  rook-version=v1.15.6
+  rook-version=v1.16.0
 This cluster is finished:
-  rook-version=v1.15.0
+  rook-version=v1.16.0
 ```
 
 ### **5. Verify the updated cluster**
 
-At this point, the Rook operator should be running version `rook/ceph:v1.15.0`.
+At this point, the Rook operator should be running version `rook/ceph:v1.16.0`.
 
 Verify the CephCluster health using the [health verification doc](health-verification.md).

From 79e767e0e77fdd9d2e0e823ecd174fb6a2aa9e23 Mon Sep 17 00:00:00 2001
From: Blaine Gardner <blaine.gardner@ibm.com>
Date: Mon, 9 Dec 2024 11:59:20 -0700
Subject: [PATCH 09/24] docs: remove deprecated toplogyKey beta labels

Remove long-deprecated topologyKey beta label from examples:
failure-domain.beta.kubernetes.io/zone

Signed-off-by: Blaine Gardner <blaine.gardner@ibm.com>
---
 deploy/examples/filesystem-ec.yaml    | 2 --
 deploy/examples/filesystem.yaml       | 2 --
 deploy/examples/object-a.yaml         | 2 --
 deploy/examples/object-b.yaml         | 2 --
 deploy/examples/object-openshift.yaml | 2 --
 deploy/examples/object.yaml           | 2 --
 6 files changed, 12 deletions(-)

diff --git a/deploy/examples/filesystem-ec.yaml b/deploy/examples/filesystem-ec.yaml
index c65c1f7931a7..b14c59bbabe3 100644
--- a/deploy/examples/filesystem-ec.yaml
+++ b/deploy/examples/filesystem-ec.yaml
@@ -73,8 +73,6 @@ spec:
                     values:
                       - rook-ceph-mds
               # topologyKey: */zone can be used to spread MDS across different AZ
-              # Use <topologyKey: failure-domain.beta.kubernetes.io/zone> in k8s cluster if your cluster is v1.16 or lower
-              # Use <topologyKey: topology.kubernetes.io/zone>  in k8s cluster is v1.17 or upper
               topologyKey: topology.kubernetes.io/zone
     # A key/value list of annotations
     annotations:
diff --git a/deploy/examples/filesystem.yaml b/deploy/examples/filesystem.yaml
index 4cc72be01f71..740fcaf1acaa 100644
--- a/deploy/examples/filesystem.yaml
+++ b/deploy/examples/filesystem.yaml
@@ -91,8 +91,6 @@ spec:
                     values:
                       - rook-ceph-mds
               # topologyKey: */zone can be used to spread MDS across different AZ
-              # Use <topologyKey: failure-domain.beta.kubernetes.io/zone> in k8s cluster if your cluster is v1.16 or lower
-              # Use <topologyKey: topology.kubernetes.io/zone>  in k8s cluster is v1.17 or upper
               topologyKey: topology.kubernetes.io/zone
     # A key/value list of annotations
     # annotations:
diff --git a/deploy/examples/object-a.yaml b/deploy/examples/object-a.yaml
index aea27bec8192..faa93e0e4ccd 100644
--- a/deploy/examples/object-a.yaml
+++ b/deploy/examples/object-a.yaml
@@ -44,8 +44,6 @@ spec:
                     values:
                       - rook-ceph-rgw
               # topologyKey: */zone can be used to spread RGW across different AZ
-              # Use <topologyKey: failure-domain.beta.kubernetes.io/zone> in k8s cluster if your cluster is v1.16 or lower
-              # Use <topologyKey: topology.kubernetes.io/zone>  in k8s cluster is v1.17 or upper
               topologyKey: kubernetes.io/hostname
     # A key/value list of annotations
     #  nodeAffinity:
diff --git a/deploy/examples/object-b.yaml b/deploy/examples/object-b.yaml
index be0d83a14bca..4ef5658fe6ed 100644
--- a/deploy/examples/object-b.yaml
+++ b/deploy/examples/object-b.yaml
@@ -44,8 +44,6 @@ spec:
                     values:
                       - rook-ceph-rgw
               # topologyKey: */zone can be used to spread RGW across different AZ
-              # Use <topologyKey: failure-domain.beta.kubernetes.io/zone> in k8s cluster if your cluster is v1.16 or lower
-              # Use <topologyKey: topology.kubernetes.io/zone>  in k8s cluster is v1.17 or upper
               topologyKey: kubernetes.io/hostname
     # A key/value list of annotations
     #  nodeAffinity:
diff --git a/deploy/examples/object-openshift.yaml b/deploy/examples/object-openshift.yaml
index 228067390780..85fc4f5ec82b 100644
--- a/deploy/examples/object-openshift.yaml
+++ b/deploy/examples/object-openshift.yaml
@@ -65,8 +65,6 @@ spec:
                     values:
                       - rook-ceph-rgw
               # topologyKey: */zone can be used to spread RGW across different AZ
-              # Use <topologyKey: failure-domain.beta.kubernetes.io/zone> in k8s cluster if your cluster is v1.16 or lower
-              # Use <topologyKey: topology.kubernetes.io/zone>  in k8s cluster is v1.17 or upper
               topologyKey: kubernetes.io/hostname
     #  topologySpreadConstraints:
     #  tolerations:
diff --git a/deploy/examples/object.yaml b/deploy/examples/object.yaml
index bee795abb9d0..414d7ff8ca53 100644
--- a/deploy/examples/object.yaml
+++ b/deploy/examples/object.yaml
@@ -67,8 +67,6 @@ spec:
                     values:
                       - rook-ceph-rgw
               # topologyKey: */zone can be used to spread RGW across different AZ
-              # Use <topologyKey: failure-domain.beta.kubernetes.io/zone> in k8s cluster if your cluster is v1.16 or lower
-              # Use <topologyKey: topology.kubernetes.io/zone>  in k8s cluster is v1.17 or upper
               topologyKey: kubernetes.io/hostname
     # A key/value list of annotations
     #  nodeAffinity:

From 279e5f7cac0f3e1eb598ff2c5cb0df12eba0195e Mon Sep 17 00:00:00 2001
From: Madhu Rajanna <madhupr007@gmail.com>
Date: Mon, 9 Dec 2024 17:20:51 +0100
Subject: [PATCH 10/24] csi: use new flag to enable VGS feature

In the Alpha Version of the VolumeGroupSnapshot
Feature the flag to enable the feature was
`--enable-volume-group-snapshots=true` and in the beta
API the feature is diasbled by default to enable it
we need to set `--feature-gate=CSIVolumeGroupSnapshot=true`

There is a change in the flag between the API version
This PR add a code where we choose the required
flag based on the API version.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
---
 pkg/operator/ceph/csi/csi.go                  | 28 ++++++++++++++++---
 pkg/operator/ceph/csi/operator_driver.go      |  2 +-
 pkg/operator/ceph/csi/spec.go                 |  2 +-
 .../csi-cephfsplugin-provisioner-dep.yaml     |  2 +-
 .../rbd/csi-rbdplugin-provisioner-dep.yaml    |  2 +-
 5 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go
index b1ab03618a26..1e64c2b095b3 100644
--- a/pkg/operator/ceph/csi/csi.go
+++ b/pkg/operator/ceph/csi/csi.go
@@ -303,15 +303,35 @@ func (r *ReconcileCSI) setParams() error {
 
 	CSIParam.DriverNamePrefix = k8sutil.GetValue(r.opConfig.Parameters, "CSI_DRIVER_NAME_PREFIX", r.opConfig.OperatorNamespace)
 
-	_, err = r.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), "volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io", metav1.GetOptions{})
+	crd, err := r.context.ApiExtensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), "volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io", metav1.GetOptions{})
 	if err != nil && !kerrors.IsNotFound(err) {
 		return errors.Wrapf(err, "failed to get volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io CRD")
 	}
 	CSIParam.VolumeGroupSnapshotSupported = (err == nil)
 
-	CSIParam.EnableVolumeGroupSnapshot = true
-	if strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_VOLUME_GROUP_SNAPSHOT", "true"), "false") {
-		CSIParam.EnableVolumeGroupSnapshot = false
+	if err == nil && len(crd.Spec.Versions) > 0 {
+		ver := crd.Spec.Versions[0]
+		// Determine if VolumeGroupSnapshot feature should be disabled
+		disableVGS := strings.EqualFold(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_VOLUME_GROUP_SNAPSHOT", "true"), "false")
+		const (
+			enableVolumeGroupSnapshotFlag = "--enable-volume-group-snapshots="
+			featureGateFlag               = "--feature-gate=CSIVolumeGroupSnapshot="
+		)
+		// Check for "v1alpha1" version to determine the appropriate CLI flag
+		// In the "v1alpha1" version, we use the '--enable-volume-group-snapshots' flag.
+		// In later versions (e.g., "v1beta1"), we use the '--feature-gate=CSIVolumeGroupSnapshot' flag.
+		if ver.Name == "v1alpha1" {
+			CSIParam.VolumeGroupSnapshotCLIFlag = enableVolumeGroupSnapshotFlag + "true"
+		} else {
+			CSIParam.VolumeGroupSnapshotCLIFlag = featureGateFlag + "true"
+		}
+		if disableVGS {
+			if ver.Name == "v1alpha1" {
+				CSIParam.VolumeGroupSnapshotCLIFlag = enableVolumeGroupSnapshotFlag + "false"
+			} else {
+				CSIParam.VolumeGroupSnapshotCLIFlag = featureGateFlag + "false"
+			}
+		}
 	}
 
 	kubeApiBurst := k8sutil.GetValue(r.opConfig.Parameters, "CSI_KUBE_API_BURST", "")
diff --git a/pkg/operator/ceph/csi/operator_driver.go b/pkg/operator/ceph/csi/operator_driver.go
index 165107da3e20..0a3fa8107e98 100644
--- a/pkg/operator/ceph/csi/operator_driver.go
+++ b/pkg/operator/ceph/csi/operator_driver.go
@@ -139,7 +139,7 @@ func (r *ReconcileCSI) createOrUpdateCephFSDriverResource(cluster cephv1.CephClu
 	}
 
 	cephFsDriver.Spec.SnapshotPolicy = csiopv1a1.NoneSnapshotPolicy
-	if CSIParam.EnableVolumeGroupSnapshot {
+	if CSIParam.VolumeGroupSnapshotCLIFlag != "" {
 		cephFsDriver.Spec.SnapshotPolicy = csiopv1a1.VolumeGroupSnapshotPolicy
 	}
 
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
index 761494b90792..1e645ba13eb1 100644
--- a/pkg/operator/ceph/csi/spec.go
+++ b/pkg/operator/ceph/csi/spec.go
@@ -77,8 +77,8 @@ type Param struct {
 	CephFSAttachRequired                     bool
 	RBDAttachRequired                        bool
 	NFSAttachRequired                        bool
+	VolumeGroupSnapshotCLIFlag               string
 	VolumeGroupSnapshotSupported             bool
-	EnableVolumeGroupSnapshot                bool
 	LogLevel                                 uint8
 	SidecarLogLevel                          uint8
 	CephFSLivenessMetricsPort                uint16
diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
index 35fa8d0c179d..5ed29f7040cd 100644
--- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
+++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
@@ -63,7 +63,7 @@ spec:
             - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
             - "--extra-create-metadata=true"
           {{ if .VolumeGroupSnapshotSupported }}
-            - "--enable-volume-group-snapshots={{ .EnableVolumeGroupSnapshot }}"
+            - "{{ .VolumeGroupSnapshotCLIFlag }}"
           {{ end }}
           {{ if .KubeApiBurst }}
             - "--kube-api-burst={{ .KubeApiBurst }}"
diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
index 23a019bc035b..135338322cfa 100644
--- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
+++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
@@ -120,7 +120,7 @@ spec:
             - "--leader-election-retry-period={{ .LeaderElectionRetryPeriod }}"
             - "--extra-create-metadata=true"
             {{ if .VolumeGroupSnapshotSupported }}
-            - "--enable-volume-group-snapshots={{ .EnableVolumeGroupSnapshot }}"
+            - "{{ .VolumeGroupSnapshotCLIFlag }}"
             {{ end }}
             {{ if .KubeApiBurst }}
             - "--kube-api-burst={{ .KubeApiBurst }}"

From 83c8ec81609a02e4aacad38af6d72131ba4fb0a6 Mon Sep 17 00:00:00 2001
From: Artem Torubarov <artem.torubarov@clyso.com>
Date: Tue, 10 Dec 2024 11:49:40 +0100
Subject: [PATCH 11/24] rgw: add rgw_enable_apis config option

Signed-off-by: Artem Torubarov <artem.torubarov@clyso.com>
---
 .../Object-Storage/ceph-object-store-crd.md   |  17 +-
 Documentation/CRDs/specification.md           |  27 +-
 .../Object-Storage-RGW/object-storage.md      | 114 +++++++
 PendingReleaseNotes.md                        |   1 +
 .../charts/rook-ceph/templates/resources.yaml |  23 +-
 deploy/examples/crds.yaml                     |  23 +-
 .../examples/object-multi-instance-test.yaml  | 123 ++++++++
 pkg/apis/ceph.rook.io/v1/types.go             |  12 +
 .../ceph.rook.io/v1/zz_generated.deepcopy.go  |   5 +
 pkg/operator/ceph/object/config.go            |  31 --
 pkg/operator/ceph/object/rgw-probe.sh         |   3 +-
 pkg/operator/ceph/object/spec.go              | 105 ++++++-
 pkg/operator/ceph/object/spec_test.go         | 288 ++++++++++++++++++
 13 files changed, 730 insertions(+), 42 deletions(-)
 create mode 100644 deploy/examples/object-multi-instance-test.yaml

diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
index 9c928a372983..d3075e2a51f2 100644
--- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
+++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md
@@ -138,11 +138,26 @@ The following options can be configured in the `keystone`-section:
 * `tokenCacheSize`: specifies the maximum number of entries in each Keystone token cache.
 * `url`: The url of the Keystone API endpoint to use.
 
-The protocols section is divided into two parts:
+### Protocols Settings
 
+The protocols section is divided into three parts:
+
+- `enableAPIs` - list of APIs to be enabled in RGW instance. If no values set, all APIs will be enabled. Possible values: `s3, s3website, swift, swift_auth, admin, sts, iam, notifications`. Represents RGW [rgw_enable_apis](https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis) config parameter.
 - a section to configure S3
 - a section to configure swift
 
+```yaml
+spec:
+  [...]
+  protocols:
+    enableAPIs: []
+    swift:
+      # a section to configure swift
+    s3:
+      # a section to configure s3
+  [...]
+```
+
 #### protocols/S3 settings
 
 In the `s3` section of the `protocols` section the following options can be configured:
diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md
index 45a2c7f0f0fd..9530f3501d55 100644
--- a/Documentation/CRDs/specification.md
+++ b/Documentation/CRDs/specification.md
@@ -9854,6 +9854,13 @@ If spec.sharedPools are also empty, then RGW pools (spec.dataPool and spec.metad
 </tr>
 </tbody>
 </table>
+<h3 id="ceph.rook.io/v1.ObjectStoreAPI">ObjectStoreAPI
+(<code>string</code> alias)</h3>
+<p>
+(<em>Appears on:</em><a href="#ceph.rook.io/v1.ProtocolSpec">ProtocolSpec</a>)
+</p>
+<div>
+</div>
 <h3 id="ceph.rook.io/v1.ObjectStoreHostingSpec">ObjectStoreHostingSpec
 </h3>
 <p>
@@ -11422,6 +11429,23 @@ alive or ready to receive traffic.</p>
 <tbody>
 <tr>
 <td>
+<code>enableAPIs</code><br/>
+<em>
+<a href="#ceph.rook.io/v1.ObjectStoreAPI">
+[]ObjectStoreAPI
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Represents RGW &lsquo;rgw_enable_apis&rsquo; config option. See: <a href="https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis">https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis</a>
+If no value provided then all APIs will be enabled: s3, s3website, swift, swift_auth, admin, sts, iam, notifications
+If enabled APIs are set, all remaining APIs will be disabled.
+This option overrides S3.Enabled value.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>s3</code><br/>
 <em>
 <a href="#ceph.rook.io/v1.S3Spec">
@@ -11925,7 +11949,8 @@ bool
 </td>
 <td>
 <em>(Optional)</em>
-<p>Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.</p>
+<p>Deprecated: use protocol.enableAPIs instead.
+Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.</p>
 </td>
 </tr>
 <tr>
diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md
index 10123bef9077..484e2424c3f5 100644
--- a/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md
+++ b/Documentation/Storage-Configuration/Object-Storage-RGW/object-storage.md
@@ -17,6 +17,7 @@ Rook can configure the Ceph Object Store for several different scenarios. See ea
 3. Create [one or more object stores with pool placement targets and storage classes](#create-local-object-stores-with-pool-placements). This configuration allows Rook to provide different object placement options to object store clients.
 4. Connect to an [RGW service in an external Ceph cluster](#connect-to-an-external-object-store), rather than create a local object store.
 5. Configure [RGW Multisite](#object-multisite) to synchronize buckets between object stores in different clusters.
+6. Create a [multi-instance RGW setup](#object-multi-instance). This option allows to have multiple `CephObjectStore` with different configurations backed by the same storage pools. For example, serving S3, Swift, or Admin-ops API by separate RGW instances.
 
 !!! note
     Updating the configuration of an object store between these types is not supported.
@@ -669,6 +670,119 @@ Multisite also allows object stores to be independent and isolated from other ob
 
 For more information on multisite please read the [ceph multisite overview](ceph-object-multisite.md) for how to run it.
 
+## Object Multi-instance
+
+This section describes how to configure multiple `CephObjectStore` backed by the same storage pools.
+The setup allows using different configuration parameters for each `CephObjecStore`, like:
+
+- `hosting` and `gateway` configs to host object store APIs on different ports and domains. For example, having a independently-scalable deployments for internal and external traffic.
+- `protocols` to host `S3`, `Swift`, and `Admin-ops` APIs on a separate `CephObjectStores`.
+- having different resource limits, affinity or other configurations per `CephObjecStore` instance for other possible use-cases.
+
+Multi-instance setup can be described in two steps. The first step is to create `CephObjectRealm`, `CephObjectZoneGroup`, and `CephObjectZone`, where
+`CephObjectZone` contains storage pools configuration. This configuration will be shared across all `CephObjectStore` instances:
+
+```yaml
+apiVersion: ceph.rook.io/v1
+kind: CephObjectRealm
+metadata:
+  name: multi-instance-store
+  namespace: rook-ceph # namespace:cluster
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZoneGroup
+metadata:
+  name: multi-instance-store
+  namespace: rook-ceph # namespace:cluster
+spec:
+  realm: multi-instance-store
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZone
+metadata:
+  name: multi-instance-store
+  namespace: rook-ceph # namespace:cluster
+spec:
+  zoneGroup: multi-instance-store
+  metadataPool:
+    failureDomain: host
+    replicated:
+      size: 1
+      requireSafeReplicaSize: false
+  dataPool:
+    failureDomain: host
+    replicated:
+      size: 1
+      requireSafeReplicaSize: false
+```
+
+The second step defines multiple `CephObjectStore` with different configurations. All of the stores should refer to the same `zone`.
+
+```yaml
+# RGW instance to host admin-ops API only
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: store-admin
+  namespace: rook-ceph # namespace:cluster
+spec:
+  gateway:
+    port: 80
+    instances: 1
+  zone:
+    name: multi-instance-store
+  protocols:
+    enableAPIs: ["admin"]
+---
+# RGW instance to host S3 API only
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: store-s3
+  namespace: rook-ceph # namespace:cluster
+spec:
+  gateway:
+    port: 80
+    instances: 1
+  zone:
+    name: multi-instance-store
+  protocols:
+    enableAPIs:
+      - s3
+      - s3website
+      - sts
+      - iam
+      - notifications
+---
+# RGW instance to host SWIFT API only
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: store-swift
+  namespace: rook-ceph # namespace:cluster
+spec:
+  gateway:
+    port: 80
+    instances: 1
+  zone:
+    name: multi-instance-store
+  protocols:
+    enableAPIs:
+      - swift
+      - swift_auth
+    swift:
+      # if S3 API is disabled, then SWIFT can be hosted on root path without prefix
+      urlPrefix: "/"
+```
+
+!!! note
+    Child resources should refer to the appropriate RGW instance.
+    For example, a `CephObjectStoreUser` requires the Admin Ops API,
+    so it should refer to an instance where this API is enabled.
+    After the user is created, it can be used for all instances.
+
+See the [example configuration](https://github.com/rook/rook/blob/master/deploy/examples/object-multi-instance-test.yaml) for more details.
+
 ## Using Swift and Keystone
 
 It is possible to access an object store using the [Swift API](https://developer.openstack.org/api-ref/object-store/index.html).
diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index ef92c6650e33..a43b0c75c30a 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -9,3 +9,4 @@
 - Enable mirroring for CephBlockPoolRadosNamespaces (see [#14701](https://github.com/rook/rook/pull/14701)).
 - Enable periodic monitoring for CephBlockPoolRadosNamespaces mirroring (see [#14896](https://github.com/rook/rook/pull/14896)).
 - Allow migration of PVC based OSDs to enable or disable encryption (see [#14776](https://github.com/rook/rook/pull/14776)).
+- Support `rgw_enable_apis` option for CephObjectStore (see [#15064](https://github.com/rook/rook/pull/15064)).
diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml
index cc9cd8917b69..ea44da105f25 100644
--- a/deploy/charts/rook-ceph/templates/resources.yaml
+++ b/deploy/charts/rook-ceph/templates/resources.yaml
@@ -12428,6 +12428,25 @@ spec:
                 protocols:
                   description: The protocol specification
                   properties:
+                    enableAPIs:
+                      description: |-
+                        Represents RGW 'rgw_enable_apis' config option. See: https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis
+                        If no value provided then all APIs will be enabled: s3, s3website, swift, swift_auth, admin, sts, iam, notifications
+                        If enabled APIs are set, all remaining APIs will be disabled.
+                        This option overrides S3.Enabled value.
+                      items:
+                        enum:
+                          - s3
+                          - s3website
+                          - swift
+                          - swift_auth
+                          - admin
+                          - sts
+                          - iam
+                          - notifications
+                        type: string
+                      nullable: true
+                      type: array
                     s3:
                       description: The spec for S3
                       nullable: true
@@ -12437,7 +12456,9 @@ spec:
                           nullable: true
                           type: boolean
                         enabled:
-                          description: Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.
+                          description: |-
+                            Deprecated: use protocol.enableAPIs instead.
+                            Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.
                           nullable: true
                           type: boolean
                       type: object
diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml
index 1c7c0ccecf31..1ec217174b75 100644
--- a/deploy/examples/crds.yaml
+++ b/deploy/examples/crds.yaml
@@ -12419,6 +12419,25 @@ spec:
                 protocols:
                   description: The protocol specification
                   properties:
+                    enableAPIs:
+                      description: |-
+                        Represents RGW 'rgw_enable_apis' config option. See: https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis
+                        If no value provided then all APIs will be enabled: s3, s3website, swift, swift_auth, admin, sts, iam, notifications
+                        If enabled APIs are set, all remaining APIs will be disabled.
+                        This option overrides S3.Enabled value.
+                      items:
+                        enum:
+                          - s3
+                          - s3website
+                          - swift
+                          - swift_auth
+                          - admin
+                          - sts
+                          - iam
+                          - notifications
+                        type: string
+                      nullable: true
+                      type: array
                     s3:
                       description: The spec for S3
                       nullable: true
@@ -12428,7 +12447,9 @@ spec:
                           nullable: true
                           type: boolean
                         enabled:
-                          description: Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.
+                          description: |-
+                            Deprecated: use protocol.enableAPIs instead.
+                            Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.
                           nullable: true
                           type: boolean
                       type: object
diff --git a/deploy/examples/object-multi-instance-test.yaml b/deploy/examples/object-multi-instance-test.yaml
new file mode 100644
index 000000000000..740a51edac86
--- /dev/null
+++ b/deploy/examples/object-multi-instance-test.yaml
@@ -0,0 +1,123 @@
+#################################################################################################################
+# Create an object store with settings for a test environment. Only a single OSD is required in this example.
+#  kubectl create -f object-multi-instance-test.yaml
+#
+# The example below provides minimalistic test configuration for multi-instance RGW deployment.
+# It defines CephObjectRealm, CephObjectZoneGroup, and CephObjectZone CRs responsible for storage pools configuration.
+# Then, three separate CephObjectStore resources are created.
+# These stores refer to the same CephObjectZone, and therefore, to the same storage pools, representing the same ObjectStorage data.
+# The CephObjectStore resources are configured to host different RGW APIs: S3, SWIFT, and Admin-ops.
+# The configuration of the CephObjectStore can be further extended to include any number of RGW deployments backed by the same data.
+# Deployments can be customized to host different APIs, allocate different resources, use different domains,
+# and apply any other options available in the CephObjectStore CRD.
+# Finally, CephObjectStoreUser is created. Child resources like user should refer to RGW instance hosting admin-ops API.
+# In this case - store-admin.
+#################################################################################################################
+apiVersion: ceph.rook.io/v1
+kind: CephObjectRealm
+metadata:
+  name: multi-instance-store
+  namespace: rook-ceph # namespace:cluster
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZoneGroup
+metadata:
+  # Names for realm/zonegroup/zone can be arbitrary.
+  # For non-multisite setup, reusing the same name across realm/zonegroup/zone is advised for simplicity.
+  # If multiple sharedPools.poolPlacements are configured for zone, then zonegroup name will be used as a prefix
+  # in S3 bucket location: <zonegroup-name>:<placement-name>. See pool placement documentation:
+  # https://docs.ceph.com/en/latest/radosgw/placement/#s3-bucket-placement - Ceph RGW doc
+  name: multi-instance-store
+  namespace: rook-ceph # namespace:cluster
+spec:
+  realm: multi-instance-store
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZone
+metadata:
+  name: multi-instance-store
+  namespace: rook-ceph # namespace:cluster
+spec:
+  zoneGroup: multi-instance-store
+  metadataPool:
+    failureDomain: host
+    replicated:
+      size: 1
+      requireSafeReplicaSize: false
+  dataPool:
+    failureDomain: host
+    replicated:
+      size: 1
+      requireSafeReplicaSize: false
+  # Alternatively, configure pool placements with pre-existing pools here.
+  # More details about Pool placements and storage classes:
+  # https://rook.io/docs/rook/latest-release/Storage-Configuration/Object-Storage-RGW/object-storage/#create-local-object-stores-with-pool-placements
+  # sharedPools:
+  #   poolPlacements:
+  #   ...
+---
+# RGW instance to host admin-ops API only
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: store-admin
+  namespace: rook-ceph # namespace:cluster
+spec:
+  gateway:
+    port: 80
+    instances: 1
+  zone:
+    name: multi-instance-store
+  protocols:
+    enableAPIs: ["admin"]
+---
+# RGW instance to host S3 API only
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: store-s3
+  namespace: rook-ceph # namespace:cluster
+spec:
+  gateway:
+    port: 80
+    instances: 1
+  zone:
+    name: multi-instance-store
+  protocols:
+    enableAPIs:
+      - s3
+      - s3website
+      - sts
+      - iam
+      - notifications
+---
+# RGW instance to host SWIFT API only
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: store-swift
+  namespace: rook-ceph # namespace:cluster
+spec:
+  gateway:
+    port: 80
+    instances: 1
+  zone:
+    name: multi-instance-store
+  protocols:
+    enableAPIs:
+      - swift
+      - swift_auth
+    swift:
+      # if S3 API is disabled, then SWIFT can be hosted on root path without prefix
+      urlPrefix: "/"
+---
+# ObjectStore user should refer to ObjectStore instance hosting admin API.
+# Created used can be used for all ObjectStore instances.
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: multi-instance-user
+  namespace: rook-ceph # namespace:cluster
+spec:
+  store: store-admin
+  displayName: "my display name"
diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go
index 1e25fcebb512..2ef5e28582de 100755
--- a/pkg/apis/ceph.rook.io/v1/types.go
+++ b/pkg/apis/ceph.rook.io/v1/types.go
@@ -1751,6 +1751,14 @@ type EndpointAddress struct {
 
 // ProtocolSpec represents a Ceph Object Store protocol specification
 type ProtocolSpec struct {
+	// Represents RGW 'rgw_enable_apis' config option. See: https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis
+	// If no value provided then all APIs will be enabled: s3, s3website, swift, swift_auth, admin, sts, iam, notifications
+	// If enabled APIs are set, all remaining APIs will be disabled.
+	// This option overrides S3.Enabled value.
+	// +optional
+	// +nullable
+	EnableAPIs []ObjectStoreAPI `json:"enableAPIs,omitempty"`
+
 	// The spec for S3
 	// +optional
 	// +nullable
@@ -1762,8 +1770,12 @@ type ProtocolSpec struct {
 	Swift *SwiftSpec `json:"swift"`
 }
 
+// +kubebuilder:validation:Enum=s3;s3website;swift;swift_auth;admin;sts;iam;notifications
+type ObjectStoreAPI string
+
 // S3Spec represents Ceph Object Store specification for the S3 API
 type S3Spec struct {
+	// Deprecated: use protocol.enableAPIs instead.
 	// Whether to enable S3. This defaults to true (even if protocols.s3 is not present in the CRD). This maintains backwards compatibility – by default S3 is enabled.
 	// +nullable
 	// +optional
diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
index c5dff621e123..b63b72756509 100644
--- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
+++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go
@@ -4261,6 +4261,11 @@ func (in *ProbeSpec) DeepCopy() *ProbeSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ProtocolSpec) DeepCopyInto(out *ProtocolSpec) {
 	*out = *in
+	if in.EnableAPIs != nil {
+		in, out := &in.EnableAPIs, &out.EnableAPIs
+		*out = make([]ObjectStoreAPI, len(*in))
+		copy(*out, *in)
+	}
 	if in.S3 != nil {
 		in, out := &in.S3, &out.S3
 		*out = new(S3Spec)
diff --git a/pkg/operator/ceph/object/config.go b/pkg/operator/ceph/object/config.go
index cbfc3ce99491..983589d0dcad 100644
--- a/pkg/operator/ceph/object/config.go
+++ b/pkg/operator/ceph/object/config.go
@@ -174,53 +174,22 @@ func (c *clusterConfig) setFlagsMonConfigStore(rgwConfig *rgwConfig) error {
 		return err
 	}
 
-	s3disabled := false
 	if s3 := rgwConfig.Protocols.S3; s3 != nil {
-		if s3.Enabled != nil && !*s3.Enabled {
-			s3disabled = true
-		}
-
 		if s3.AuthUseKeystone != nil {
 			configOptions["rgw_s3_auth_use_keystone"] = fmt.Sprintf("%t", *s3.AuthUseKeystone)
 		}
-
 	}
 
 	if swift := rgwConfig.Protocols.Swift; swift != nil {
-
 		if swift.AccountInUrl != nil {
 			configOptions["rgw_swift_account_in_url"] = fmt.Sprintf("%t", *swift.AccountInUrl)
 		}
-
 		if swift.UrlPrefix != nil {
 			configOptions["rgw_swift_url_prefix"] = *swift.UrlPrefix
-
-			if configOptions["rgw_swift_url_prefix"] == "/" {
-				logger.Warning("Forcefully disabled S3 as the swift prefix is given as a slash /. Ignoring any S3 options (including Enabled=true)!")
-				// this will later on disable the s3 api using the rgw_enable_apis setting
-				s3disabled = true
-			}
-
 		}
 		if swift.VersioningEnabled != nil {
 			configOptions["rgw_swift_versioning_enabled"] = fmt.Sprintf("%t", *swift.VersioningEnabled)
 		}
-
-	}
-
-	if s3disabled {
-		// XXX: how to handle enabled APIs? We only configure s3 and
-		// swift in the resource, `admin` is required for the operator to
-		// work, `swift_auth` is required to access swift without keystone
-		// – not sure about the additional APIs
-		// https://docs.ceph.com/en/latest/radosgw/config-ref/#confval-rgw_enable_apis
-		// see also https://docs.ceph.com/en/octopus/radosgw/config-ref/#swift-settings on disabling s3
-		// when using '/' as prefix
-
-		// Swift was enabled so far already by default, so perhaps better
-		// not change that if someone relies on it.
-
-		configOptions["rgw_enable_apis"] = "s3website, swift, swift_auth, admin, sts, iam, notifications"
 	}
 
 	for flag, val := range configOptions {
diff --git a/pkg/operator/ceph/object/rgw-probe.sh b/pkg/operator/ceph/object/rgw-probe.sh
index 853b430bc56b..0b7de897928b 100644
--- a/pkg/operator/ceph/object/rgw-probe.sh
+++ b/pkg/operator/ceph/object/rgw-probe.sh
@@ -3,6 +3,7 @@
 PROBE_TYPE="{{ .ProbeType }}"
 PROBE_PORT="{{ .Port }}"
 PROBE_PROTOCOL="{{ .Protocol }}"
+PROBE_PATH="{{ .Path }}"
 
 # standard bash codes start at 126 and progress upward. pick error codes from 125 downward for
 # script as to allow curl to output new error codes and still return a distinctive number.
@@ -13,7 +14,7 @@ PROBE_ERR_CODE=124
 STARTUP_TYPE='startup'
 READINESS_TYPE='readiness'
 
-RGW_URL="$PROBE_PROTOCOL://0.0.0.0:$PROBE_PORT"
+RGW_URL="$PROBE_PROTOCOL://0.0.0.0:$PROBE_PORT$PROBE_PATH"
 
 function check() {
   local URL="$1"
diff --git a/pkg/operator/ceph/object/spec.go b/pkg/operator/ceph/object/spec.go
index 1b6e5b056707..b84b44b17c15 100644
--- a/pkg/operator/ceph/object/spec.go
+++ b/pkg/operator/ceph/object/spec.go
@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net/url"
 	"path"
+	"slices"
 	"strings"
 	"text/template"
 
@@ -71,6 +72,8 @@ chown --recursive --verbose ceph:ceph $VAULT_TOKEN_NEW_PATH
 var (
 	//go:embed rgw-probe.sh
 	rgwProbeScriptTemplate string
+
+	rgwAPIwithoutS3 = []string{"s3website", "swift", "swift_auth", "admin", "sts", "iam", "notifications"}
 )
 
 type ProbeType string
@@ -89,6 +92,7 @@ type rgwProbeConfig struct {
 
 	Protocol ProtocolType
 	Port     string
+	Path     string
 }
 
 func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment, error) {
@@ -401,23 +405,27 @@ func (c *clusterConfig) makeDaemonContainer(rgwConfig *rgwConfig) (v1.Container,
 		}
 	}
 
-	s3Enabled, err := c.CheckRGWSSES3Enabled()
+	if flags := buildRGWConfigFlags(c.store); len(flags) != 0 {
+		container.Args = append(container.Args, flags...)
+	}
+
+	s3EncryptionEnabled, err := c.CheckRGWSSES3Enabled()
 	if err != nil {
 		return v1.Container{}, err
 	}
-	if s3Enabled {
+	if s3EncryptionEnabled {
 		logger.Debugf("enabliing SSE-S3. %v", c.store.Spec.Security.ServerSideEncryptionS3)
 
-		container.Args = append(container.Args, c.sseS3DefaultOptions(s3Enabled)...)
+		container.Args = append(container.Args, c.sseS3DefaultOptions(s3EncryptionEnabled)...)
 		if c.store.Spec.Security.ServerSideEncryptionS3.IsTokenAuthEnabled() {
-			container.Args = append(container.Args, c.sseS3VaultTokenOptions(s3Enabled)...)
+			container.Args = append(container.Args, c.sseS3VaultTokenOptions(s3EncryptionEnabled)...)
 		}
 		if c.store.Spec.Security.ServerSideEncryptionS3.IsTLSEnabled() {
-			container.Args = append(container.Args, c.sseS3VaultTLSOptions(s3Enabled)...)
+			container.Args = append(container.Args, c.sseS3VaultTLSOptions(s3EncryptionEnabled)...)
 		}
 	}
 
-	if s3Enabled || kmsEnabled {
+	if s3EncryptionEnabled || kmsEnabled {
 		vaultVolMount := v1.VolumeMount{Name: rgwVaultVolumeName, MountPath: rgwVaultDirName}
 		container.VolumeMounts = append(container.VolumeMounts, vaultVolMount)
 	}
@@ -459,11 +467,17 @@ func noLivenessProbe() *v1.Probe {
 }
 
 func (c *clusterConfig) defaultReadinessProbe() (*v1.Probe, error) {
+	probePath, disableProbe := getRGWProbePath(c.store.Spec.Protocols)
+	if disableProbe {
+		logger.Infof("disabling startup probe for %q store", c.store.Name)
+		return nil, nil
+	}
 	proto, port := c.endpointInfo()
 	cfg := rgwProbeConfig{
 		ProbeType: ReadinessProbeType,
 		Protocol:  proto,
 		Port:      port.String(),
+		Path:      probePath,
 	}
 	script, err := renderProbe(cfg)
 	if err != nil {
@@ -489,13 +503,52 @@ func (c *clusterConfig) defaultReadinessProbe() (*v1.Probe, error) {
 	return probe, nil
 }
 
+// getRGWProbePath - returns custom path for RGW probe and returns true if probe should be disabled.
+func getRGWProbePath(protocolSpec cephv1.ProtocolSpec) (path string, disable bool) {
+	enabledAPIs := buildRGWEnableAPIsConfigVal(protocolSpec)
+	if len(enabledAPIs) == 0 {
+		// all apis including s3 are enabled
+		// using default s3 Probe
+		return "", false
+	}
+	if slices.Contains(enabledAPIs, "s3") {
+		// using default s3 Probe
+		return "", false
+	}
+	if slices.Contains(enabledAPIs, "swift") {
+		// using swift api for probe
+		// calculate path for swift probe
+		prefix := "/swift/"
+		if protocolSpec.Swift != nil && protocolSpec.Swift.UrlPrefix != nil && *protocolSpec.Swift.UrlPrefix != "" {
+			prefix = *protocolSpec.Swift.UrlPrefix
+			if !strings.HasPrefix(prefix, "/") {
+				prefix = "/" + prefix
+			}
+			if !strings.HasSuffix(prefix, "/") {
+				prefix += "/"
+			}
+		}
+		prefix += "info"
+		return prefix, false
+	}
+	// both swift and s3 are disabled - disable probe.
+	return "", true
+}
+
 func (c *clusterConfig) defaultStartupProbe() (*v1.Probe, error) {
+	probePath, disableProbe := getRGWProbePath(c.store.Spec.Protocols)
+	if disableProbe {
+		logger.Infof("disabling startup probe for %q store", c.store.Name)
+		return nil, nil
+	}
 	proto, port := c.endpointInfo()
 	cfg := rgwProbeConfig{
 		ProbeType: StartupProbeType,
 		Protocol:  proto,
 		Port:      port.String(),
+		Path:      probePath,
 	}
+
 	script, err := renderProbe(cfg)
 	if err != nil {
 		return nil, err
@@ -904,6 +957,46 @@ func (c *clusterConfig) sseS3VaultTLSOptions(setOptions bool) []string {
 	return rgwOptions
 }
 
+// Builds list of rgw config parameters which should be passed as CLI flags.
+// Consider set config option as flag if BOTH criteria fulfilled:
+//  1. config value is not secret
+//  2. config change requires RGW daemon restart
+//
+// Otherwise set rgw config parameter to mon database in ./config.go -> setFlagsMonConfigStore()
+// CLI flags override values from mon db: see ceph config docs: https://docs.ceph.com/en/latest/rados/configuration/ceph-conf/#config-sources
+func buildRGWConfigFlags(objectStore *cephv1.CephObjectStore) []string {
+	var res []string
+	// todo: move all flags here
+	if enableAPIs := buildRGWEnableAPIsConfigVal(objectStore.Spec.Protocols); len(enableAPIs) != 0 {
+		res = append(res, cephconfig.NewFlag("rgw_enable_apis", strings.Join(enableAPIs, ",")))
+		logger.Debugf("Enabling APIs for RGW instance %q: %s", objectStore.Name, enableAPIs)
+	}
+	return res
+}
+
+func buildRGWEnableAPIsConfigVal(protocolSpec cephv1.ProtocolSpec) []string {
+	if len(protocolSpec.EnableAPIs) != 0 {
+		// handle explicit enabledAPIS spec
+		enabledAPIs := make([]string, len(protocolSpec.EnableAPIs))
+		for i, v := range protocolSpec.EnableAPIs {
+			enabledAPIs[i] = strings.TrimSpace(string(v))
+		}
+		return enabledAPIs
+	}
+
+	// if enabledAPIs not set, check if S3 should be disabled
+	if protocolSpec.S3 != nil && protocolSpec.S3.Enabled != nil && !*protocolSpec.S3.Enabled { //nolint // disable deprecation check
+		return rgwAPIwithoutS3
+	}
+	// see also https://docs.ceph.com/en/octopus/radosgw/config-ref/#swift-settings on disabling s3
+	// when using '/' as prefix
+	if protocolSpec.Swift != nil && protocolSpec.Swift.UrlPrefix != nil && *protocolSpec.Swift.UrlPrefix == "/" {
+		logger.Warning("Forcefully disabled S3 as the swift prefix is given as a slash /. Ignoring any S3 options (including Enabled=true)!")
+		return rgwAPIwithoutS3
+	}
+	return nil
+}
+
 func renderProbe(cfg rgwProbeConfig) (string, error) {
 	var writer bytes.Buffer
 	name := string(cfg.ProbeType) + "-probe"
diff --git a/pkg/operator/ceph/object/spec_test.go b/pkg/operator/ceph/object/spec_test.go
index f920f5f5a1cb..32f65f618b0e 100644
--- a/pkg/operator/ceph/object/spec_test.go
+++ b/pkg/operator/ceph/object/spec_test.go
@@ -18,7 +18,9 @@ package object
 
 import (
 	"context"
+	_ "embed"
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -1151,3 +1153,289 @@ func TestGetHostnameFromEndpoint(t *testing.T) {
 		})
 	}
 }
+
+func strPtr(in string) *string {
+	return &in
+}
+
+func Test_buildRGWEnableAPIsConfigVal(t *testing.T) {
+	type args struct {
+		protocolSpec cephv1.ProtocolSpec
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "nothing set",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{},
+			},
+			want: nil,
+		},
+		{
+			name: "only admin enabled",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{
+						"admin",
+					},
+				},
+			},
+			want: []string{"admin"},
+		},
+		{
+			name: "admin and s3 enabled",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{
+						"admin",
+						"s3",
+					},
+				},
+			},
+			want: []string{"admin", "s3"},
+		},
+		{
+			name: "whitespaces trimmed",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{
+						" s3 ",
+						" swift ",
+					},
+				},
+			},
+			want: []string{"s3", "swift"},
+		},
+		{
+			name: "s3 disabled",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					S3: &cephv1.S3Spec{
+						Enabled: new(bool),
+					},
+				},
+			},
+			want: rgwAPIwithoutS3,
+		},
+		{
+			name: "s3 disabled when swift prefix is '/'",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					Swift: &cephv1.SwiftSpec{
+						UrlPrefix: strPtr("/"),
+					},
+				},
+			},
+			want: rgwAPIwithoutS3,
+		},
+		{
+			name: "s3 is not disabled when swift prefix is not '/'",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					Swift: &cephv1.SwiftSpec{
+						UrlPrefix: strPtr("object/"),
+					},
+				},
+			},
+			want: nil,
+		},
+		{
+			name: "enableAPIs overrides s3 option",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{
+						" s3 ",
+						" swift ",
+					},
+					S3: &cephv1.S3Spec{
+						Enabled: new(bool),
+					},
+				},
+			},
+			want: []string{"s3", "swift"},
+		},
+		{
+			name: "enableAPIs overrides swift prefix option",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{
+						" s3 ",
+						" swift ",
+					},
+					Swift: &cephv1.SwiftSpec{
+						UrlPrefix: strPtr("/"),
+					},
+				},
+			},
+			want: []string{"s3", "swift"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := buildRGWEnableAPIsConfigVal(tt.args.protocolSpec)
+			assert.ElementsMatch(t, got, tt.want)
+		})
+	}
+}
+
+func Test_buildRGWConfigFlags(t *testing.T) {
+	type args struct {
+		objectStore *cephv1.CephObjectStore
+	}
+	tests := []struct {
+		name string
+		args args
+		want []string
+	}{
+		{
+			name: "nothing set",
+			args: args{
+				objectStore: &cephv1.CephObjectStore{},
+			},
+			want: nil,
+		},
+		{
+			name: "enabled APIs set",
+			args: args{
+				objectStore: &cephv1.CephObjectStore{
+					Spec: cephv1.ObjectStoreSpec{
+						Protocols: cephv1.ProtocolSpec{
+							EnableAPIs: []cephv1.ObjectStoreAPI{
+								"swift",
+								"admin",
+							},
+						},
+					},
+				},
+			},
+			want: []string{
+				"--rgw-enable-apis=swift,admin",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := buildRGWConfigFlags(tt.args.objectStore); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("buildRGWConfigFlags() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_getRGWProbePathAndCode(t *testing.T) {
+	type args struct {
+		protocolSpec cephv1.ProtocolSpec
+	}
+	tests := []struct {
+		name        string
+		args        args
+		wantPath    string
+		wantDisable bool
+	}{
+		{
+			name: "all apis enabled - return s3 probe",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{},
+					S3:         &cephv1.S3Spec{},
+					Swift:      &cephv1.SwiftSpec{},
+				},
+			},
+			wantPath:    "",
+			wantDisable: false,
+		},
+		{
+			name: "s3 disabled - return default swift probe",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{},
+					S3: &cephv1.S3Spec{
+						Enabled: new(bool),
+					},
+					Swift: &cephv1.SwiftSpec{},
+				},
+			},
+			wantPath:    "/swift/info",
+			wantDisable: false,
+		},
+		{
+			name: "s3 disabled in api list - return default swift probe",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{
+						"swift",
+						"admin",
+					},
+				},
+			},
+			wantPath:    "/swift/info",
+			wantDisable: false,
+		},
+		{
+			name: "s3 disabled - return swift with custom prefix probe",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{
+						"swift",
+						"admin",
+					},
+					Swift: &cephv1.SwiftSpec{
+						UrlPrefix: strPtr("some-path"),
+					},
+				},
+			},
+			wantPath:    "/some-path/info",
+			wantDisable: false,
+		},
+		{
+			name: "s3 disabled - return swift with root prefix probe",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					Swift: &cephv1.SwiftSpec{
+						UrlPrefix: strPtr("/"),
+					},
+				},
+			},
+			wantPath:    "/info",
+			wantDisable: false,
+		},
+		{
+			name: "s3 and swift disabled - disable probe",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{
+						"admin",
+					},
+				},
+			},
+			wantPath:    "",
+			wantDisable: true,
+		},
+		{
+			name: "no suitable api enabled - disable probe",
+			args: args{
+				protocolSpec: cephv1.ProtocolSpec{
+					EnableAPIs: []cephv1.ObjectStoreAPI{
+						"sts",
+					},
+				},
+			},
+			wantPath:    "",
+			wantDisable: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotPath, gotDisable := getRGWProbePath(tt.args.protocolSpec)
+			if gotPath != tt.wantPath {
+				t.Errorf("getRGWProbePath() gotPath = %v, want %v", gotPath, tt.wantPath)
+			}
+			if gotDisable != tt.wantDisable {
+				t.Errorf("getRGWProbePath() gotDisable = %v, want %v", gotDisable, tt.wantDisable)
+			}
+		})
+	}
+}

From e5a2d1be1f57a28c4756288c93f40638c44f8643 Mon Sep 17 00:00:00 2001
From: Travis Nielsen <tnielsen@redhat.com>
Date: Tue, 10 Dec 2024 16:39:58 -0700
Subject: [PATCH 12/24] ci: remove obsolete test check from mergify

The test for disabling host network was removed with the multus
work so there is no longer a need to check for that test
in the mergify list of tests

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
---
 .mergify.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.mergify.yml b/.mergify.yml
index 45938a112319..9a895e21742f 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -227,7 +227,6 @@ pull_request_rules:
       - "check-success=canary-tests / rgw-multisite-testing (quay.io/ceph/ceph:v18)"
       - "check-success=canary-tests / encryption-pvc-kms-ibm-kp (quay.io/ceph/ceph:v18)"
       - "check-success=canary-tests / multus-public-and-cluster (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / csi-hostnetwork-disabled (quay.io/ceph/ceph:v18)"
       - "check-success=TestCephSmokeSuite (v1.27.16)"
       - "check-success=TestCephSmokeSuite (v1.31.0)"
       - "check-success=TestCephHelmSuite (v1.27.16)"

From 56ac6132309c7431e127887a005c0307e8017a03 Mon Sep 17 00:00:00 2001
From: Artem Torubarov <artem.torubarov@clyso.com>
Date: Wed, 11 Dec 2024 16:47:56 +0100
Subject: [PATCH 13/24] ci: add arttor to reviewers list

Signed-off-by: Artem Torubarov <artem.torubarov@clyso.com>
---
 CODE-OWNERS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CODE-OWNERS b/CODE-OWNERS
index 80f23271b6e0..20e2cd0a2728 100644
--- a/CODE-OWNERS
+++ b/CODE-OWNERS
@@ -14,3 +14,4 @@ approvers:
 reviewers:
 - Madhu-1
 - parth-gr
+- arttor

From 23f3db12615d2aaa565cabfcbe826801210e368e Mon Sep 17 00:00:00 2001
From: Travis Nielsen <tnielsen@redhat.com>
Date: Wed, 11 Dec 2024 09:42:59 -0700
Subject: [PATCH 14/24] ci: suppress the error from creating the dashboard
 admin user

The dashboard admin rgw user has timing isssues in the CI.
For now, just suppress the CI failure and log the error
until we can spend more time to get a reliable wait for the
user creation.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
---
 tests/integration/ceph_base_object_test.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go
index 79521d1ec25c..1730a03fa331 100644
--- a/tests/integration/ceph_base_object_test.go
+++ b/tests/integration/ceph_base_object_test.go
@@ -179,7 +179,10 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils
 		for _, objectStore := range objectStores.Items {
 			err, output := installer.Execute("radosgw-admin", []string{"user", "info", "--uid=dashboard-admin", fmt.Sprintf("--rgw-realm=%s", objectStore.GetName())}, namespace)
 			logger.Infof("output: %s", output)
-			assert.NoError(t, err)
+			if err != nil {
+				// Just log the error until we get a more reliable way to wait for the user to be created
+				logger.Errorf("failed to get dashboard-admin from object store %s. %+v", objectStore.GetName(), err)
+			}
 		}
 	})
 }

From 85375bf0735284b76a2fdc589e8bc4c86fff0676 Mon Sep 17 00:00:00 2001
From: Travis Nielsen <tnielsen@redhat.com>
Date: Wed, 11 Dec 2024 11:21:21 -0700
Subject: [PATCH 15/24] ci: use ceph-ci instead of daemon-base images

Ceph has changed the image build process to only require a
dockerfile and stop using the ceph-container repo. The
daily images are pushed to the quay.io/ceph-ci/ceph repo,
so the Rook CI will now start using those images.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
---
 .github/workflows/daily-nightly-jobs.yml    | 2 +-
 tests/framework/installer/ceph_installer.go | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml
index d211e03a1e1d..3e9ac4acae62 100644
--- a/.github/workflows/daily-nightly-jobs.yml
+++ b/.github/workflows/daily-nightly-jobs.yml
@@ -356,5 +356,5 @@ jobs:
     if: github.repository == 'rook/rook'
     uses: ./.github/workflows/canary-integration-test.yml
     with:
-      ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/daemon-base:latest-main-devel", "quay.io/ceph/daemon-base:latest-reef-devel", "quay.io/ceph/daemon-base:latest-squid-devel"]'
+      ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph-ci/ceph:main", "quay.io/ceph-ci/ceph:reef", "quay.io/ceph-ci/ceph:squid"]'
     secrets: inherit
diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go
index f1f89b5d9a0f..2ef5e737e9e1 100644
--- a/tests/framework/installer/ceph_installer.go
+++ b/tests/framework/installer/ceph_installer.go
@@ -46,10 +46,10 @@ const (
 	reefTestImage  = "quay.io/ceph/ceph:v18"
 	squidTestImage = "quay.io/ceph/ceph:v19"
 	// test with the current development versions
-	reefDevelTestImage  = "quay.io/ceph/daemon-base:latest-reef-devel"
-	squidDevelTestImage = "quay.io/ceph/daemon-base:latest-squid-devel"
+	reefDevelTestImage  = "quay.io/ceph-ci/ceph:reef"
+	squidDevelTestImage = "quay.io/ceph-ci/ceph:squid"
 	// test with the latest Ceph main image
-	mainTestImage      = "quay.io/ceph/daemon-base:latest-main-devel"
+	mainTestImage      = "quay.io/ceph-ci/ceph:main"
 	cephOperatorLabel  = "app=rook-ceph-operator"
 	defaultclusterName = "test-cluster"
 

From 07f5700a87d31f564e013c39ad921b47381ed9b0 Mon Sep 17 00:00:00 2001
From: Madhu Rajanna <madhupr007@gmail.com>
Date: Wed, 11 Dec 2024 10:34:06 +0100
Subject: [PATCH 16/24] csi: update cephcsi to latest release

cephcsi 3.13.0 is released today and
update the Rook to use the latest image

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
---
 Documentation/Helm-Charts/operator-chart.md                 | 2 +-
 .../Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md      | 6 +++---
 .../Storage-Configuration/Ceph-CSI/custom-images.md         | 2 +-
 deploy/charts/rook-ceph/values.yaml                         | 2 +-
 deploy/examples/images.txt                                  | 2 +-
 deploy/examples/operator-openshift.yaml                     | 2 +-
 deploy/examples/operator.yaml                               | 2 +-
 pkg/operator/ceph/csi/spec.go                               | 2 +-
 pkg/operator/ceph/csi/util_test.go                          | 2 +-
 9 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md
index 1b1e0b2b3a37..c1a587c479bf 100644
--- a/Documentation/Helm-Charts/operator-chart.md
+++ b/Documentation/Helm-Charts/operator-chart.md
@@ -60,7 +60,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` |
 | `csi.cephFSPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. | `1` |
 | `csi.cephcsi.repository` | Ceph CSI image repository | `"quay.io/cephcsi/cephcsi"` |
-| `csi.cephcsi.tag` | Ceph CSI image tag | `"v3.12.3"` |
+| `csi.cephcsi.tag` | Ceph CSI image tag | `"v3.13.0"` |
 | `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port | `9081` |
 | `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Deployments and DaemonSets Pods | `nil` |
 | `csi.clusterName` | Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster | `nil` |
diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md
index d0264755e582..17ca0d0ac3d8 100644
--- a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md
+++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md
@@ -217,10 +217,10 @@ CSI-Addons supports the following operations:
 
 Ceph-CSI supports encrypting PersistentVolumeClaims (PVCs) for both RBD and CephFS.
 This can be achieved using LUKS for RBD and fscrypt for CephFS. More details on encrypting RBD PVCs can be found
-[here](https://github.com/ceph/ceph-csi/blob/v3.12.3/docs/deploy-rbd.md#encryption-for-rbd-volumes),
+[here](https://github.com/ceph/ceph-csi/blob/v3.13.0/docs/deploy-rbd.md#encryption-for-rbd-volumes),
 which includes a full list of supported encryption configurations.
-More details on encrypting CephFS PVCs can be found [here](https://github.com/ceph/ceph-csi/blob/v3.12.3/docs/deploy-cephfs.md#cephfs-volume-encryption).
-A sample KMS configmap can be found [here](https://github.com/ceph/ceph-csi/blob/v3.12.3/examples/kms/vault/kms-config.yaml).
+More details on encrypting CephFS PVCs can be found [here](https://github.com/ceph/ceph-csi/blob/v3.13.0/docs/deploy-cephfs.md#cephfs-volume-encryption).
+A sample KMS configmap can be found [here](https://github.com/ceph/ceph-csi/blob/v3.13.0/examples/kms/vault/kms-config.yaml).
 
 !!! note
     Not all KMS are compatible with fscrypt. Generally, KMS that either store secrets to use directly (like Vault)
diff --git a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md
index c2e32882597d..cb26bd3462e2 100644
--- a/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md
+++ b/Documentation/Storage-Configuration/Ceph-CSI/custom-images.md
@@ -18,7 +18,7 @@ kubectl -n $ROOK_OPERATOR_NAMESPACE edit configmap rook-ceph-operator-config
 The default upstream images are included below, which you can change to your desired images.
 
 ```yaml
-ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.12.3"
+ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.13.0"
 ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1"
 ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v5.0.1"
 ROOK_CSI_ATTACHER_IMAGE: "registry.k8s.io/sig-storage/csi-attacher:v4.6.1"
diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml
index b16d8234ddf5..ca6dc6a4210d 100644
--- a/deploy/charts/rook-ceph/values.yaml
+++ b/deploy/charts/rook-ceph/values.yaml
@@ -480,7 +480,7 @@ csi:
     # -- Ceph CSI image repository
     repository: quay.io/cephcsi/cephcsi
     # -- Ceph CSI image tag
-    tag: v3.12.3
+    tag: v3.13.0
 
   registrar:
     # -- Kubernetes CSI registrar image repository
diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt
index 8052af0b6822..3846c6835a29 100644
--- a/deploy/examples/images.txt
+++ b/deploy/examples/images.txt
@@ -2,7 +2,7 @@
  gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:v20240513-v0.1.0-35-gefb3255
  quay.io/ceph/ceph:v18.2.4
  quay.io/ceph/cosi:v0.1.2
- quay.io/cephcsi/cephcsi:v3.12.3
+ quay.io/cephcsi/cephcsi:v3.13.0
  quay.io/csiaddons/k8s-sidecar:v0.11.0
  registry.k8s.io/sig-storage/csi-attacher:v4.6.1
  registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1
diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml
index ca427af04309..763de626c8c8 100644
--- a/deploy/examples/operator-openshift.yaml
+++ b/deploy/examples/operator-openshift.yaml
@@ -189,7 +189,7 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.12.3"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.13.0"
   # ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1"
   # ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v1.11.1"
   # ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v5.0.1"
diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml
index c0ccbc693951..5e67ca6dad32 100644
--- a/deploy/examples/operator.yaml
+++ b/deploy/examples/operator.yaml
@@ -119,7 +119,7 @@ data:
   # The default version of CSI supported by Rook will be started. To change the version
   # of the CSI driver to something other than what is officially supported, change
   # these images to the desired release of the CSI driver.
-  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.12.3"
+  # ROOK_CSI_CEPH_IMAGE: "quay.io/cephcsi/cephcsi:v3.13.0"
   # ROOK_CSI_REGISTRAR_IMAGE: "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1"
   # ROOK_CSI_RESIZER_IMAGE: "registry.k8s.io/sig-storage/csi-resizer:v1.11.1"
   # ROOK_CSI_PROVISIONER_IMAGE: "registry.k8s.io/sig-storage/csi-provisioner:v5.0.1"
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
index 1e645ba13eb1..4e66838c39f1 100644
--- a/pkg/operator/ceph/csi/spec.go
+++ b/pkg/operator/ceph/csi/spec.go
@@ -131,7 +131,7 @@ var (
 // manually challenging.
 var (
 	// image names
-	DefaultCSIPluginImage   = "quay.io/cephcsi/cephcsi:v3.12.3"
+	DefaultCSIPluginImage   = "quay.io/cephcsi/cephcsi:v3.13.0"
 	DefaultRegistrarImage   = "registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.11.1"
 	DefaultProvisionerImage = "registry.k8s.io/sig-storage/csi-provisioner:v5.0.1"
 	DefaultAttacherImage    = "registry.k8s.io/sig-storage/csi-attacher:v4.6.1"
diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go
index 2e974a1d0f3d..29f5cbced138 100644
--- a/pkg/operator/ceph/csi/util_test.go
+++ b/pkg/operator/ceph/csi/util_test.go
@@ -284,7 +284,7 @@ func Test_getImage(t *testing.T) {
 			args: args{
 				data:         map[string]string{},
 				settingName:  "ROOK_CSI_CEPH_IMAGE",
-				defaultImage: "quay.io/cephcsi/cephcsi:v3.12.3",
+				defaultImage: "quay.io/cephcsi/cephcsi:v3.13.0",
 			},
 			want: DefaultCSIPluginImage,
 		},

From 07d8012d3678ab8e846afb043128fd03876e0985 Mon Sep 17 00:00:00 2001
From: Travis Nielsen <tnielsen@redhat.com>
Date: Fri, 22 Nov 2024 11:21:29 -0700
Subject: [PATCH 17/24] manifest: update default ceph version to v19

With the v19.2.0 release being out for some time now,
update the default version to be deployed with Rook
as v19.2.0.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
---
 .../workflows/canary-integration-suite.yml    |  2 +-
 .github/workflows/canary-integration-test.yml |  2 +-
 .github/workflows/daily-nightly-jobs.yml      |  2 +-
 .mergify.yml                                  | 40 +++++++++----------
 .../CRDs/Cluster/ceph-cluster-crd.md          |  8 ++--
 Documentation/CRDs/Cluster/host-cluster.md    |  6 +--
 Documentation/CRDs/Cluster/pvc-cluster.md     |  6 +--
 Documentation/CRDs/Cluster/stretch-cluster.md |  2 +-
 Documentation/Upgrade/ceph-upgrade.md         |  2 +-
 deploy/charts/rook-ceph-cluster/values.yaml   |  6 +--
 .../examples/cluster-external-management.yaml |  2 +-
 deploy/examples/cluster-multus-test.yaml      |  2 +-
 deploy/examples/cluster-on-local-pvc.yaml     |  2 +-
 deploy/examples/cluster-on-pvc-minikube.yaml  |  2 +-
 deploy/examples/cluster-on-pvc.yaml           |  2 +-
 deploy/examples/cluster-stretched-aws.yaml    |  2 +-
 deploy/examples/cluster-stretched.yaml        |  2 +-
 deploy/examples/cluster.yaml                  |  4 +-
 deploy/examples/images.txt                    |  2 +-
 deploy/examples/toolbox.yaml                  |  2 +-
 design/ceph/ceph-cluster-cleanup.md           |  2 +-
 .../ceph/cluster/osd/deviceset_test.go        |  4 +-
 .../test-cluster-on-pvc-encrypted.yaml        |  2 +-
 23 files changed, 53 insertions(+), 53 deletions(-)

diff --git a/.github/workflows/canary-integration-suite.yml b/.github/workflows/canary-integration-suite.yml
index 731d50fcdbbc..4b3204bd22bc 100644
--- a/.github/workflows/canary-integration-suite.yml
+++ b/.github/workflows/canary-integration-suite.yml
@@ -31,5 +31,5 @@ jobs:
   canary-tests:
     uses: ./.github/workflows/canary-integration-test.yml
     with:
-      ceph_images: '["quay.io/ceph/ceph:v18"]'
+      ceph_images: '["quay.io/ceph/ceph:v19"]'
     secrets: inherit
diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml
index 42a7d1207dab..5ff12d5f6d69 100644
--- a/.github/workflows/canary-integration-test.yml
+++ b/.github/workflows/canary-integration-test.yml
@@ -5,7 +5,7 @@ on:
     inputs:
       ceph_images:
         description: "JSON list of Ceph images for creating Ceph cluster"
-        default: '["quay.io/ceph/ceph:v18"]'
+        default: '["quay.io/ceph/ceph:v19"]'
         type: string
 
 defaults:
diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml
index 3e9ac4acae62..97656176b039 100644
--- a/.github/workflows/daily-nightly-jobs.yml
+++ b/.github/workflows/daily-nightly-jobs.yml
@@ -356,5 +356,5 @@ jobs:
     if: github.repository == 'rook/rook'
     uses: ./.github/workflows/canary-integration-test.yml
     with:
-      ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph-ci/ceph:main", "quay.io/ceph-ci/ceph:reef", "quay.io/ceph-ci/ceph:squid"]'
+      ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/ceph:v19", "quay.io/ceph-ci/ceph:main", "quay.io/ceph-ci/ceph:reef", "quay.io/ceph-ci/ceph:squid"]'
     secrets: inherit
diff --git a/.mergify.yml b/.mergify.yml
index 9a895e21742f..8d4894ff1a5a 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -207,26 +207,26 @@ pull_request_rules:
       - "check-success=crds-gen"
       - "check-success=docs-check"
       - "check-success=pylint"
-      - "check-success=canary-tests / canary (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / raw-disk-with-object (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / two-osds-in-device (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / osd-with-metadata-partition-device (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / osd-with-metadata-device (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / encryption (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / lvm (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / pvc (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / pvc-db (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / pvc-db-wal (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / encryption-pvc (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / encryption-pvc-db (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / encryption-pvc-db-wal (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / encryption-pvc-kms-vault-token-auth (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / encryption-pvc-kms-vault-k8s-auth (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / lvm-pvc (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / multi-cluster-mirroring (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / rgw-multisite-testing (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / encryption-pvc-kms-ibm-kp (quay.io/ceph/ceph:v18)"
-      - "check-success=canary-tests / multus-public-and-cluster (quay.io/ceph/ceph:v18)"
+      - "check-success=canary-tests / canary (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / raw-disk-with-object (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / two-osds-in-device (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / osd-with-metadata-partition-device (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / osd-with-metadata-device (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / encryption (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / lvm (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / pvc (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / pvc-db (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / pvc-db-wal (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / encryption-pvc (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / encryption-pvc-db (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / encryption-pvc-db-wal (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / encryption-pvc-kms-vault-token-auth (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / encryption-pvc-kms-vault-k8s-auth (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / lvm-pvc (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / multi-cluster-mirroring (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / rgw-multisite-testing (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / encryption-pvc-kms-ibm-kp (quay.io/ceph/ceph:v19)"
+      - "check-success=canary-tests / multus-public-and-cluster (quay.io/ceph/ceph:v19)"
       - "check-success=TestCephSmokeSuite (v1.27.16)"
       - "check-success=TestCephSmokeSuite (v1.31.0)"
       - "check-success=TestCephHelmSuite (v1.27.16)"
diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
index c8212008bcac..3cfd60d2dd0e 100755
--- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md
+++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md
@@ -26,7 +26,7 @@ Settings can be specified at the global level to apply to the cluster as a whole
 * `external`:
     * `enable`: if `true`, the cluster will not be managed by Rook but via an external entity. This mode is intended to connect to an existing cluster. In this case, Rook will only consume the external cluster. However, Rook will be able to deploy various daemons in Kubernetes such as object gateways, mds and nfs if an image is provided and will refuse otherwise. If this setting is enabled **all** the other options will be ignored except `cephVersion.image` and `dataDirHostPath`. See [external cluster configuration](external-cluster/external-cluster.md). If `cephVersion.image` is left blank, Rook will refuse the creation of extra CRs like object, file and nfs.
 * `cephVersion`: The version information for launching the ceph daemons.
-    * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v18.2.4`. For more details read the [container images section](#ceph-container-images).
+    * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v19.2.0`. For more details read the [container images section](#ceph-container-images).
         For the latest ceph images, see the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags/).
         To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version.
         Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v19` will be updated each time a new Squid build is released.
@@ -431,7 +431,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -538,7 +538,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -668,7 +668,7 @@ kubectl -n rook-ceph get CephCluster -o yaml
       deviceClasses:
       - name: hdd
     version:
-      image: quay.io/ceph/ceph:v18.2.4
+      image: quay.io/ceph/ceph:v19.2.0
       version: 16.2.6-0
     conditions:
     - lastHeartbeatTime: "2021-03-02T21:22:11Z"
diff --git a/Documentation/CRDs/Cluster/host-cluster.md b/Documentation/CRDs/Cluster/host-cluster.md
index 9e2bd97e685d..9d26088632bf 100644
--- a/Documentation/CRDs/Cluster/host-cluster.md
+++ b/Documentation/CRDs/Cluster/host-cluster.md
@@ -22,7 +22,7 @@ metadata:
 spec:
   cephVersion:
     # see the "Cluster Settings" section below for more details on which image of ceph to run
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -49,7 +49,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -101,7 +101,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
diff --git a/Documentation/CRDs/Cluster/pvc-cluster.md b/Documentation/CRDs/Cluster/pvc-cluster.md
index fc3efe4322e5..651bd17ecfa4 100644
--- a/Documentation/CRDs/Cluster/pvc-cluster.md
+++ b/Documentation/CRDs/Cluster/pvc-cluster.md
@@ -18,7 +18,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
@@ -72,7 +72,7 @@ spec:
           requests:
             storage: 10Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
     allowUnsupported: false
   dashboard:
     enabled: true
@@ -128,7 +128,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
diff --git a/Documentation/CRDs/Cluster/stretch-cluster.md b/Documentation/CRDs/Cluster/stretch-cluster.md
index 00e0adf95c99..d35fa7cc7117 100644
--- a/Documentation/CRDs/Cluster/stretch-cluster.md
+++ b/Documentation/CRDs/Cluster/stretch-cluster.md
@@ -34,7 +34,7 @@ spec:
       - name: b
       - name: c
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
     allowUnsupported: true
   # Either storageClassDeviceSets or the storage section can be specified for creating OSDs.
   # This example uses all devices for simplicity.
diff --git a/Documentation/Upgrade/ceph-upgrade.md b/Documentation/Upgrade/ceph-upgrade.md
index c4adae8be0ad..6ec671f33d73 100644
--- a/Documentation/Upgrade/ceph-upgrade.md
+++ b/Documentation/Upgrade/ceph-upgrade.md
@@ -42,7 +42,7 @@ These images are tagged in a few ways:
 * The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v19.2.0-20240927`).
     These tags are recommended for production clusters, as there is no possibility for the cluster to
     be heterogeneous with respect to the version of Ceph running in containers.
-* Ceph major version tags (e.g., `v18`) are useful for development and test clusters so that the
+* Ceph major version tags (e.g., `v19`) are useful for development and test clusters so that the
     latest version of Ceph is always available.
 
 **Ceph containers other than the official images from the registry above will not be supported.**
diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml
index 47297069d216..7576f05bdfd4 100644
--- a/deploy/charts/rook-ceph-cluster/values.yaml
+++ b/deploy/charts/rook-ceph-cluster/values.yaml
@@ -25,7 +25,7 @@ toolbox:
   # -- Enable Ceph debugging pod deployment. See [toolbox](../Troubleshooting/ceph-toolbox.md)
   enabled: false
   # -- Toolbox image, defaults to the image used by the Ceph cluster
-  image: #quay.io/ceph/ceph:v18.2.4
+  image: #quay.io/ceph/ceph:v19.2.0
   # -- Toolbox tolerations
   tolerations: []
   # -- Toolbox affinity
@@ -92,9 +92,9 @@ cephClusterSpec:
     # v18 is Reef, v19 is Squid
     # RECOMMENDATION: In production, use a specific version tag instead of the general v18 flag, which pulls the latest release and could result in different
     # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
-    # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v18.2.4-20240724
+    # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v19.2.0-20240927
     # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
     # Whether to allow unsupported versions of Ceph. Currently Reef and Squid are supported.
     # Future versions such as Tentacle (v20) would require this to be set to `true`.
     # Do not set to true in production.
diff --git a/deploy/examples/cluster-external-management.yaml b/deploy/examples/cluster-external-management.yaml
index f1ef32560b3c..777e6419bf45 100644
--- a/deploy/examples/cluster-external-management.yaml
+++ b/deploy/examples/cluster-external-management.yaml
@@ -20,4 +20,4 @@ spec:
   dataDirHostPath: /var/lib/rook
   # providing an image is required, if you want to create other CRs (rgw, mds, nfs)
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4 # Should match external cluster version
+    image: quay.io/ceph/ceph:v19.2.0 # Should match external cluster version
diff --git a/deploy/examples/cluster-multus-test.yaml b/deploy/examples/cluster-multus-test.yaml
index 21cb6dda2cb4..5342c762d2ae 100644
--- a/deploy/examples/cluster-multus-test.yaml
+++ b/deploy/examples/cluster-multus-test.yaml
@@ -15,7 +15,7 @@ metadata:
 spec:
   dataDirHostPath: /var/lib/rook
   cephVersion:
-    image: quay.io/ceph/ceph:v18
+    image: quay.io/ceph/ceph:v19
     allowUnsupported: true
   mon:
     count: 1
diff --git a/deploy/examples/cluster-on-local-pvc.yaml b/deploy/examples/cluster-on-local-pvc.yaml
index c93b52af8225..f38f1f21e8ad 100644
--- a/deploy/examples/cluster-on-local-pvc.yaml
+++ b/deploy/examples/cluster-on-local-pvc.yaml
@@ -174,7 +174,7 @@ spec:
           requests:
             storage: 10Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
     allowUnsupported: false
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-on-pvc-minikube.yaml b/deploy/examples/cluster-on-pvc-minikube.yaml
index 00f56a72607e..993d938ae92d 100644
--- a/deploy/examples/cluster-on-pvc-minikube.yaml
+++ b/deploy/examples/cluster-on-pvc-minikube.yaml
@@ -150,7 +150,7 @@ spec:
   crashCollector:
     disable: false
   cephVersion:
-    image: quay.io/ceph/ceph:v18
+    image: quay.io/ceph/ceph:v19
     allowUnsupported: false
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-on-pvc.yaml b/deploy/examples/cluster-on-pvc.yaml
index 398de65cf28a..522277ab402b 100644
--- a/deploy/examples/cluster-on-pvc.yaml
+++ b/deploy/examples/cluster-on-pvc.yaml
@@ -34,7 +34,7 @@ spec:
           requests:
             storage: 10Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
     allowUnsupported: false
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-stretched-aws.yaml b/deploy/examples/cluster-stretched-aws.yaml
index 48796dfe26c4..c2ac50725a5c 100644
--- a/deploy/examples/cluster-stretched-aws.yaml
+++ b/deploy/examples/cluster-stretched-aws.yaml
@@ -45,7 +45,7 @@ spec:
   mgr:
     count: 2
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
     allowUnsupported: true
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster-stretched.yaml b/deploy/examples/cluster-stretched.yaml
index 5d595f3ca4cd..d45fbac1c1ea 100644
--- a/deploy/examples/cluster-stretched.yaml
+++ b/deploy/examples/cluster-stretched.yaml
@@ -39,7 +39,7 @@ spec:
   mgr:
     count: 2
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
     allowUnsupported: true
   skipUpgradeChecks: false
   continueUpgradeAfterChecksEvenIfNotHealthy: false
diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml
index 70b4068757b9..b82bfd8fba4a 100644
--- a/deploy/examples/cluster.yaml
+++ b/deploy/examples/cluster.yaml
@@ -19,9 +19,9 @@ spec:
     # v18 is Reef, v19 is Squid
     # RECOMMENDATION: In production, use a specific version tag instead of the general v19 flag, which pulls the latest release and could result in different
     # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
-    # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v18.2.4-20240724
+    # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v19.2.0-20240927
     # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
     # Whether to allow unsupported versions of Ceph. Currently Reef and Squid are supported.
     # Future versions such as Tentacle (v20) would require this to be set to `true`.
     # Do not set to true in production.
diff --git a/deploy/examples/images.txt b/deploy/examples/images.txt
index 8052af0b6822..e9bc6e2b31d7 100644
--- a/deploy/examples/images.txt
+++ b/deploy/examples/images.txt
@@ -1,6 +1,6 @@
  docker.io/rook/ceph:master
  gcr.io/k8s-staging-sig-storage/objectstorage-sidecar:v20240513-v0.1.0-35-gefb3255
- quay.io/ceph/ceph:v18.2.4
+ quay.io/ceph/ceph:v19.2.0
  quay.io/ceph/cosi:v0.1.2
  quay.io/cephcsi/cephcsi:v3.12.3
  quay.io/csiaddons/k8s-sidecar:v0.11.0
diff --git a/deploy/examples/toolbox.yaml b/deploy/examples/toolbox.yaml
index a060b91b51b6..5559e064a0b9 100644
--- a/deploy/examples/toolbox.yaml
+++ b/deploy/examples/toolbox.yaml
@@ -19,7 +19,7 @@ spec:
       serviceAccountName: rook-ceph-default
       containers:
         - name: rook-ceph-tools
-          image: quay.io/ceph/ceph:v18.2.4
+          image: quay.io/ceph/ceph:v19
           command:
             - /bin/bash
             - -c
diff --git a/design/ceph/ceph-cluster-cleanup.md b/design/ceph/ceph-cluster-cleanup.md
index 1af2fda984ec..ba534bbcf079 100644
--- a/design/ceph/ceph-cluster-cleanup.md
+++ b/design/ceph/ceph-cluster-cleanup.md
@@ -34,7 +34,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
   dataDirHostPath: /var/lib/rook
   mon:
     count: 3
diff --git a/pkg/operator/ceph/cluster/osd/deviceset_test.go b/pkg/operator/ceph/cluster/osd/deviceset_test.go
index 281b6f18fc14..0a2da91018a2 100644
--- a/pkg/operator/ceph/cluster/osd/deviceset_test.go
+++ b/pkg/operator/ceph/cluster/osd/deviceset_test.go
@@ -294,8 +294,8 @@ func TestPVCName(t *testing.T) {
 }
 
 func TestCreateValidImageVersionLabel(t *testing.T) {
-	image := "ceph/ceph:v18.2.4"
-	assert.Equal(t, "ceph_ceph_v18.2.4", createValidImageVersionLabel(image))
+	image := "ceph/ceph:v19.2.0"
+	assert.Equal(t, "ceph_ceph_v19.2.0", createValidImageVersionLabel(image))
 	image = "rook/ceph:master"
 	assert.Equal(t, "rook_ceph_master", createValidImageVersionLabel(image))
 	image = ".invalid_label"
diff --git a/tests/manifests/test-cluster-on-pvc-encrypted.yaml b/tests/manifests/test-cluster-on-pvc-encrypted.yaml
index 7e363abae53b..2ebdeb6d1925 100644
--- a/tests/manifests/test-cluster-on-pvc-encrypted.yaml
+++ b/tests/manifests/test-cluster-on-pvc-encrypted.yaml
@@ -14,7 +14,7 @@ spec:
           requests:
             storage: 5Gi
   cephVersion:
-    image: quay.io/ceph/ceph:v18.2.4
+    image: quay.io/ceph/ceph:v19.2.0
   dashboard:
     enabled: false
   network:

From 6db75b76b8b256bff90ed0e20519816b1f908724 Mon Sep 17 00:00:00 2001
From: Travis Nielsen <tnielsen@redhat.com>
Date: Wed, 11 Dec 2024 15:02:25 -0700
Subject: [PATCH 18/24] ci: use correct ceph-ci repo

The repo should be quay.ceph.io, instead of quay.io

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
---
 .github/workflows/daily-nightly-jobs.yml    | 2 +-
 tests/framework/installer/ceph_installer.go | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml
index 97656176b039..41fb53c081e1 100644
--- a/.github/workflows/daily-nightly-jobs.yml
+++ b/.github/workflows/daily-nightly-jobs.yml
@@ -356,5 +356,5 @@ jobs:
     if: github.repository == 'rook/rook'
     uses: ./.github/workflows/canary-integration-test.yml
     with:
-      ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/ceph:v19", "quay.io/ceph-ci/ceph:main", "quay.io/ceph-ci/ceph:reef", "quay.io/ceph-ci/ceph:squid"]'
+      ceph_images: '["quay.io/ceph/ceph:v18", "quay.io/ceph/ceph:v19", "quay.ceph.io/ceph-ci/ceph:main", "quay.ceph.io/ceph-ci/ceph:reef", "quay.ceph.io/ceph-ci/ceph:squid"]'
     secrets: inherit
diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go
index 2ef5e737e9e1..e7fed2c1575a 100644
--- a/tests/framework/installer/ceph_installer.go
+++ b/tests/framework/installer/ceph_installer.go
@@ -46,10 +46,10 @@ const (
 	reefTestImage  = "quay.io/ceph/ceph:v18"
 	squidTestImage = "quay.io/ceph/ceph:v19"
 	// test with the current development versions
-	reefDevelTestImage  = "quay.io/ceph-ci/ceph:reef"
-	squidDevelTestImage = "quay.io/ceph-ci/ceph:squid"
+	reefDevelTestImage  = "quay.ceph.io/ceph-ci/ceph:reef"
+	squidDevelTestImage = "quay.ceph.io/ceph-ci/ceph:squid"
 	// test with the latest Ceph main image
-	mainTestImage      = "quay.io/ceph-ci/ceph:main"
+	mainTestImage      = "quay.ceph.io/ceph-ci/ceph:main"
 	cephOperatorLabel  = "app=rook-ceph-operator"
 	defaultclusterName = "test-cluster"
 

From 7c2f41e72e57b3800db9ffde360bf18588968f07 Mon Sep 17 00:00:00 2001
From: Travis Nielsen <tnielsen@redhat.com>
Date: Thu, 12 Dec 2024 09:48:42 -0700
Subject: [PATCH 19/24] build: add support for k8s 1.32

With the release of K8s 1.32, we update the CI and docs
to support this new release, to maintain the most recent
six releases of K8s.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
---
 .github/workflows/canary-test-config/action.yaml     |  2 +-
 .github/workflows/integration-test-helm-suite.yaml   |  2 +-
 .../integration-test-keystone-auth-suite.yaml        |  2 +-
 .github/workflows/integration-test-mgr-suite.yaml    |  2 +-
 .../integration-test-multi-cluster-suite.yaml        |  2 +-
 .github/workflows/integration-test-object-suite.yaml |  2 +-
 .github/workflows/integration-test-smoke-suite.yaml  |  2 +-
 .../workflows/integration-test-upgrade-suite.yaml    |  4 ++--
 .github/workflows/integration-tests-on-release.yaml  | 12 ++++++------
 .mergify.yml                                         | 12 ++++++------
 .../Getting-Started/Prerequisites/prerequisites.md   |  2 +-
 Documentation/Getting-Started/quickstart.md          |  2 +-
 PendingReleaseNotes.md                               |  2 ++
 tests/scripts/github-action-helper.sh                |  2 +-
 14 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/canary-test-config/action.yaml b/.github/workflows/canary-test-config/action.yaml
index 4945390a5985..f87fa2db7eeb 100644
--- a/.github/workflows/canary-test-config/action.yaml
+++ b/.github/workflows/canary-test-config/action.yaml
@@ -12,7 +12,7 @@ runs:
     - name: Setup Minikube
       shell: bash --noprofile --norc -eo pipefail -x {0}
       run: |
-        tests/scripts/github-action-helper.sh install_minikube_with_none_driver v1.31.0
+        tests/scripts/github-action-helper.sh install_minikube_with_none_driver v1.32.0
 
     - name: install deps
       shell: bash --noprofile --norc -eo pipefail -x {0}
diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml
index f0fcafa30be8..fa9b2e965d94 100644
--- a/.github/workflows/integration-test-helm-suite.yaml
+++ b/.github/workflows/integration-test-helm-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-keystone-auth-suite.yaml b/.github/workflows/integration-test-keystone-auth-suite.yaml
index 85d9bacc22eb..e77e3bab7422 100644
--- a/.github/workflows/integration-test-keystone-auth-suite.yaml
+++ b/.github/workflows/integration-test-keystone-auth-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml
index 68a56d486703..b6ad6820c168 100644
--- a/.github/workflows/integration-test-mgr-suite.yaml
+++ b/.github/workflows/integration-test-mgr-suite.yaml
@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.31.0"]
+        kubernetes-versions: ["v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml
index 05233cf08a41..d740f1ff4a9f 100644
--- a/.github/workflows/integration-test-multi-cluster-suite.yaml
+++ b/.github/workflows/integration-test-multi-cluster-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.31.0"]
+        kubernetes-versions: ["v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml
index 77eecc343420..1f2aee71a27e 100644
--- a/.github/workflows/integration-test-object-suite.yaml
+++ b/.github/workflows/integration-test-object-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml
index e16ff3b88881..49ef417448dd 100644
--- a/.github/workflows/integration-test-smoke-suite.yaml
+++ b/.github/workflows/integration-test-smoke-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml
index d8eb22d5f671..fe160392b867 100644
--- a/.github/workflows/integration-test-upgrade-suite.yaml
+++ b/.github/workflows/integration-test-upgrade-suite.yaml
@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -73,7 +73,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.github/workflows/integration-tests-on-release.yaml b/.github/workflows/integration-tests-on-release.yaml
index 0512356df3d2..d25644c84774 100644
--- a/.github/workflows/integration-tests-on-release.yaml
+++ b/.github/workflows/integration-tests-on-release.yaml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.28.12", "v1.29.7", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -61,7 +61,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.28.12", "v1.29.7", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -102,7 +102,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.28.12", "v1.29.7", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -140,7 +140,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.28.12", "v1.29.7", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -178,7 +178,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.28.12", "v1.29.7", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.28.15", "v1.30.8", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -219,7 +219,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.27.16", "v1.31.0"]
+        kubernetes-versions: ["v1.27.16", "v1.32.0"]
     steps:
       - name: checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
diff --git a/.mergify.yml b/.mergify.yml
index 8d4894ff1a5a..b6818c4e0db7 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -228,16 +228,16 @@ pull_request_rules:
       - "check-success=canary-tests / encryption-pvc-kms-ibm-kp (quay.io/ceph/ceph:v19)"
       - "check-success=canary-tests / multus-public-and-cluster (quay.io/ceph/ceph:v19)"
       - "check-success=TestCephSmokeSuite (v1.27.16)"
-      - "check-success=TestCephSmokeSuite (v1.31.0)"
+      - "check-success=TestCephSmokeSuite (v1.32.0)"
       - "check-success=TestCephHelmSuite (v1.27.16)"
-      - "check-success=TestCephHelmSuite (v1.31.0)"
-      - "check-success=TestCephMultiClusterDeploySuite (v1.31.0)"
+      - "check-success=TestCephHelmSuite (v1.32.0)"
+      - "check-success=TestCephMultiClusterDeploySuite (v1.32.0)"
       - "check-success=TestCephObjectSuite (v1.27.16)"
-      - "check-success=TestCephObjectSuite (v1.31.0)"
+      - "check-success=TestCephObjectSuite (v1.32.0)"
       - "check-success=TestCephUpgradeSuite (v1.27.16)"
-      - "check-success=TestCephUpgradeSuite (v1.31.0)"
+      - "check-success=TestCephUpgradeSuite (v1.32.0)"
       - "check-success=TestHelmUpgradeSuite (v1.27.16)"
-      - "check-success=TestHelmUpgradeSuite (v1.31.0)"
+      - "check-success=TestHelmUpgradeSuite (v1.32.0)"
     actions:
       merge:
         method: merge
diff --git a/Documentation/Getting-Started/Prerequisites/prerequisites.md b/Documentation/Getting-Started/Prerequisites/prerequisites.md
index 914736435460..182f6e5f59cc 100644
--- a/Documentation/Getting-Started/Prerequisites/prerequisites.md
+++ b/Documentation/Getting-Started/Prerequisites/prerequisites.md
@@ -7,7 +7,7 @@ and Rook is granted the required privileges (see below for more information).
 
 ## Kubernetes Version
 
-Kubernetes versions **v1.27** through **v1.31** are supported.
+Kubernetes versions **v1.27** through **v1.32** are supported.
 
 ## CPU Architecture
 
diff --git a/Documentation/Getting-Started/quickstart.md b/Documentation/Getting-Started/quickstart.md
index b4672df545e9..913868a71971 100644
--- a/Documentation/Getting-Started/quickstart.md
+++ b/Documentation/Getting-Started/quickstart.md
@@ -12,7 +12,7 @@ This guide will walk through the basic setup of a Ceph cluster and enable K8s ap
 
 ## Kubernetes Version
 
-Kubernetes versions **v1.27** through **v1.31** are supported.
+Kubernetes versions **v1.27** through **v1.32** are supported.
 
 ## CPU Architecture
 
diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index a43b0c75c30a..c850f99df4f6 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -3,9 +3,11 @@
 ## Breaking Changes
 
 - Removed support for Ceph Quincy (v17) since it has reached end of life
+- Minimum K8s version updated to v1.27
 
 ## Features
 
+- Added supported for K8s version v1.32
 - Enable mirroring for CephBlockPoolRadosNamespaces (see [#14701](https://github.com/rook/rook/pull/14701)).
 - Enable periodic monitoring for CephBlockPoolRadosNamespaces mirroring (see [#14896](https://github.com/rook/rook/pull/14896)).
 - Allow migration of PVC based OSDs to enable or disable encryption (see [#14776](https://github.com/rook/rook/pull/14776)).
diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh
index 558c315a5459..927a560814ac 100755
--- a/tests/scripts/github-action-helper.sh
+++ b/tests/scripts/github-action-helper.sh
@@ -692,7 +692,7 @@ function test_csi_nfs_workload {
 }
 
 function install_minikube_with_none_driver() {
-  CRICTL_VERSION="v1.31.1"
+  CRICTL_VERSION="v1.32.0"
   MINIKUBE_VERSION="v1.34.0"
 
   sudo apt update

From e5f79dba7f9bed5f004ddff7ca514f63dc94f84e Mon Sep 17 00:00:00 2001
From: Joshua Hoblitt <josh@hoblitt.com>
Date: Mon, 21 Oct 2024 16:48:21 -0700
Subject: [PATCH 20/24] object: factor out bucket.setS3Agent()

Add a s3Agent field to bucket.Provisioner struct as a step towards
allowing the s3Agent / s3 client to be mocked in unit tests.

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
---
 .../ceph/object/bucket/provisioner.go         | 63 +++++++++----------
 1 file changed, 28 insertions(+), 35 deletions(-)

diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go
index 074905625506..4e3dec79fedc 100644
--- a/pkg/operator/ceph/object/bucket/provisioner.go
+++ b/pkg/operator/ceph/object/bucket/provisioner.go
@@ -54,6 +54,7 @@ type Provisioner struct {
 	tlsCert              []byte
 	insecureTLS          bool
 	adminOpsClient       *admin.API
+	s3Agent              *object.S3Agent
 }
 
 type additionalConfigSpec struct {
@@ -95,12 +96,7 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
 		return nil, errors.Wrap(err, "Provision: can't create ceph user")
 	}
 
-	var s3svc *object.S3Agent
-	if p.insecureTLS {
-		s3svc, err = object.NewInsecureS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG))
-	} else {
-		s3svc, err = object.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG), p.tlsCert)
-	}
+	err = p.setS3Agent()
 	if err != nil {
 		return nil, err
 	}
@@ -116,7 +112,7 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
 		// if bucket already exists, this returns error: TooManyBuckets because we set the quota
 		// below. If it already exists, assume we are good to go
 		logger.Debugf("creating bucket %q", p.bucketName)
-		err = s3svc.CreateBucket(p.bucketName)
+		err = p.s3Agent.CreateBucket(p.bucketName)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error creating bucket %q", p.bucketName)
 		}
@@ -172,29 +168,13 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
 		return nil, err
 	}
 
-	// get the bucket's owner via the bucket metadata
-	stats, err := p.adminOpsClient.GetBucketInfo(p.clusterInfo.Context, admin.Bucket{Bucket: p.bucketName})
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to get bucket %q stats", p.bucketName)
-	}
-
-	objectUser, err := p.adminOpsClient.GetUser(p.clusterInfo.Context, admin.User{ID: stats.Owner})
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to get user %q", stats.Owner)
-	}
-
-	var s3svc *object.S3Agent
-	if p.insecureTLS {
-		s3svc, err = object.NewInsecureS3Agent(objectUser.Keys[0].AccessKey, objectUser.Keys[0].SecretKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG))
-	} else {
-		s3svc, err = object.NewS3Agent(objectUser.Keys[0].AccessKey, objectUser.Keys[0].SecretKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG), p.tlsCert)
-	}
+	err = p.setS3Agent()
 	if err != nil {
 		return nil, err
 	}
 
 	// if the policy does not exist, we'll create a new and append the statement to it
-	policy, err := s3svc.GetBucketPolicy(p.bucketName)
+	policy, err := p.s3Agent.GetBucketPolicy(p.bucketName)
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
 			if aerr.Code() != "NoSuchBucketPolicy" {
@@ -215,7 +195,7 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
 	} else {
 		policy = policy.ModifyBucketPolicy(*statement)
 	}
-	out, err := s3svc.PutBucketPolicy(p.bucketName, *policy)
+	out, err := p.s3Agent.PutBucketPolicy(p.bucketName, *policy)
 
 	logger.Infof("PutBucketPolicy output: %v", out)
 	if err != nil {
@@ -280,19 +260,17 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error {
 			return err
 		}
 
-		var s3svc *object.S3Agent
-		if p.insecureTLS {
-			s3svc, err = object.NewInsecureS3Agent(user.Keys[0].AccessKey, user.Keys[0].SecretKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG))
-		} else {
-			s3svc, err = object.NewS3Agent(user.Keys[0].AccessKey, user.Keys[0].SecretKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG), p.tlsCert)
-		}
+		p.accessKeyID = user.Keys[0].AccessKey
+		p.secretAccessKey = user.Keys[0].SecretKey
+
+		err = p.setS3Agent()
 		if err != nil {
 			return err
 		}
 
 		// Ignore cases where there is no bucket policy. This may have occurred if an error ended a Grant()
 		// call before the policy was attached to the bucket
-		policy, err := s3svc.GetBucketPolicy(p.bucketName)
+		policy, err := p.s3Agent.GetBucketPolicy(p.bucketName)
 		if err != nil {
 			if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NoSuchBucketPolicy" {
 				policy = nil
@@ -317,7 +295,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error {
 			} else {
 				policy = policy.ModifyBucketPolicy(*statement)
 			}
-			out, err := s3svc.PutBucketPolicy(p.bucketName, *policy)
+			out, err := p.s3Agent.PutBucketPolicy(p.bucketName, *policy)
 			logger.Infof("PutBucketPolicy output: %v", out)
 			if err != nil {
 				return errors.Wrap(err, "failed to update policy")
@@ -329,7 +307,7 @@ func (p Provisioner) Revoke(ob *bktv1alpha1.ObjectBucket) error {
 		// drop policy if present
 		if policy != nil {
 			policy = policy.DropPolicyStatements(p.cephUserName)
-			_, err := s3svc.PutBucketPolicy(p.bucketName, *policy)
+			_, err := p.s3Agent.PutBucketPolicy(p.bucketName, *policy)
 			if err != nil {
 				return err
 			}
@@ -754,3 +732,18 @@ func (p *Provisioner) setAdminOpsAPIClient() error {
 
 	return nil
 }
+
+func (p *Provisioner) setS3Agent() error {
+	var err error
+
+	if p.insecureTLS {
+		p.s3Agent, err = object.NewInsecureS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG))
+	} else {
+		p.s3Agent, err = object.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG), p.tlsCert)
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

From 57b7eeec800bc008507b7e2ff5b47d50d4503ece Mon Sep 17 00:00:00 2001
From: Joshua Hoblitt <josh@hoblitt.com>
Date: Tue, 22 Oct 2024 15:09:04 -0700
Subject: [PATCH 21/24] object: add httpClient param to object.NewS3Agent()

To allow the caller to pass in their own transport when testing.

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
---
 .../ceph/object/bucket/provisioner.go         | 13 ++------
 .../ceph/object/notification/provisioner.go   |  5 +--
 pkg/operator/ceph/object/s3-handlers.go       | 25 +++++++--------
 pkg/operator/ceph/object/s3-handlers_test.go  | 32 ++++++++++++++++---
 tests/framework/clients/bucket.go             |  6 +---
 .../ceph_bucket_notification_test.go          | 14 +++-----
 tests/integration/ceph_object_test.go         | 14 +++-----
 7 files changed, 50 insertions(+), 59 deletions(-)

diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go
index 4e3dec79fedc..eba55118602b 100644
--- a/pkg/operator/ceph/object/bucket/provisioner.go
+++ b/pkg/operator/ceph/object/bucket/provisioner.go
@@ -735,15 +735,6 @@ func (p *Provisioner) setAdminOpsAPIClient() error {
 
 func (p *Provisioner) setS3Agent() error {
 	var err error
-
-	if p.insecureTLS {
-		p.s3Agent, err = object.NewInsecureS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG))
-	} else {
-		p.s3Agent, err = object.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG), p.tlsCert)
-	}
-	if err != nil {
-		return err
-	}
-
-	return nil
+	p.s3Agent, err = object.NewS3Agent(p.accessKeyID, p.secretAccessKey, p.getObjectStoreEndpoint(), logger.LevelAt(capnslog.DEBUG), p.tlsCert, p.insecureTLS, nil)
+	return err
 }
diff --git a/pkg/operator/ceph/object/notification/provisioner.go b/pkg/operator/ceph/object/notification/provisioner.go
index dc1579fa0ed8..47d0d37e50e2 100644
--- a/pkg/operator/ceph/object/notification/provisioner.go
+++ b/pkg/operator/ceph/object/notification/provisioner.go
@@ -91,10 +91,7 @@ func newS3Agent(p provisioner) (*object.S3Agent, error) {
 		}
 	}
 
-	if insecureTLS {
-		return object.NewInsecureS3Agent(accessKey, secretKey, objContext.Endpoint, logger.LevelAt(capnslog.DEBUG))
-	}
-	return object.NewS3Agent(accessKey, secretKey, objContext.Endpoint, logger.LevelAt(capnslog.DEBUG), tlsCert)
+	return object.NewS3Agent(accessKey, secretKey, objContext.Endpoint, logger.LevelAt(capnslog.DEBUG), tlsCert, insecureTLS, nil)
 }
 
 // TODO: convert all rules without restrictions once the AWS SDK supports that
diff --git a/pkg/operator/ceph/object/s3-handlers.go b/pkg/operator/ceph/object/s3-handlers.go
index 13eccb8261d1..6bc510ef0fbf 100644
--- a/pkg/operator/ceph/object/s3-handlers.go
+++ b/pkg/operator/ceph/object/s3-handlers.go
@@ -39,27 +39,24 @@ type S3Agent struct {
 	Client *s3.S3
 }
 
-func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte) (*S3Agent, error) {
-	return newS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, false)
-}
-
-func NewInsecureS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) {
-	return newS3Agent(accessKey, secretKey, endpoint, debug, nil, true)
-}
-
-func newS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte, insecure bool) (*S3Agent, error) {
+func NewS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byte, insecure bool, httpClient *http.Client) (*S3Agent, error) {
 	logLevel := aws.LogOff
 	if debug {
 		logLevel = aws.LogDebug
 	}
-	client := http.Client{
-		Timeout: HttpTimeOut,
-	}
 	tlsEnabled := false
 	if len(tlsCert) > 0 || insecure {
 		tlsEnabled = true
-		client.Transport = BuildTransportTLS(tlsCert, insecure)
 	}
+	if httpClient == nil {
+		httpClient = &http.Client{
+			Timeout: HttpTimeOut,
+		}
+		if tlsEnabled {
+			httpClient.Transport = BuildTransportTLS(tlsCert, insecure)
+		}
+	}
+
 	session, err := awssession.NewSession(
 		aws.NewConfig().
 			WithRegion(CephRegion).
@@ -68,7 +65,7 @@ func newS3Agent(accessKey, secretKey, endpoint string, debug bool, tlsCert []byt
 			WithS3ForcePathStyle(true).
 			WithMaxRetries(5).
 			WithDisableSSL(!tlsEnabled).
-			WithHTTPClient(&client).
+			WithHTTPClient(httpClient).
 			WithLogLevel(logLevel),
 	)
 	if err != nil {
diff --git a/pkg/operator/ceph/object/s3-handlers_test.go b/pkg/operator/ceph/object/s3-handlers_test.go
index f7f20fe0ccaf..628dea1e512a 100644
--- a/pkg/operator/ceph/object/s3-handlers_test.go
+++ b/pkg/operator/ceph/object/s3-handlers_test.go
@@ -32,7 +32,7 @@ func TestNewS3Agent(t *testing.T) {
 	t.Run("test without tls/debug", func(t *testing.T) {
 		debug := false
 		insecure := false
-		s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure)
+		s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure, nil)
 		assert.NoError(t, err)
 		assert.NotEqual(t, aws.LogDebug, s3Agent.Client.Config.LogLevel)
 		assert.Equal(t, nil, s3Agent.Client.Config.HTTPClient.Transport)
@@ -42,7 +42,7 @@ func TestNewS3Agent(t *testing.T) {
 		debug := true
 		logLevel := aws.LogDebug
 		insecure := false
-		s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure)
+		s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure, nil)
 		assert.NoError(t, err)
 		assert.Equal(t, &logLevel, s3Agent.Client.Config.LogLevel)
 		assert.Nil(t, s3Agent.Client.Config.HTTPClient.Transport)
@@ -51,7 +51,7 @@ func TestNewS3Agent(t *testing.T) {
 	t.Run("test without tls client cert but insecure tls", func(t *testing.T) {
 		debug := true
 		insecure := true
-		s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure)
+		s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure, nil)
 		assert.NoError(t, err)
 		assert.NotNil(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.RootCAs) // still includes sys certs
 		assert.True(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify)
@@ -61,7 +61,7 @@ func TestNewS3Agent(t *testing.T) {
 		debug := true
 		insecure := false
 		tlsCert := []byte("tlsCert")
-		s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, insecure)
+		s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, insecure, nil)
 		assert.NoError(t, err)
 		assert.NotNil(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.RootCAs)
 		assert.False(t, *s3Agent.Client.Config.DisableSSL)
@@ -70,10 +70,32 @@ func TestNewS3Agent(t *testing.T) {
 		debug := true
 		insecure := true
 		tlsCert := []byte("tlsCert")
-		s3Agent, err := newS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, insecure)
+		s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, tlsCert, insecure, nil)
 		assert.NoError(t, err)
 		assert.NotNil(t, s3Agent.Client.Config.HTTPClient.Transport)
 		assert.True(t, s3Agent.Client.Config.HTTPClient.Transport.(*http.Transport).TLSClientConfig.InsecureSkipVerify)
 		assert.False(t, *s3Agent.Client.Config.DisableSSL)
 	})
+	t.Run("test with custom http.Client", func(t *testing.T) {
+		debug := true
+		logLevel := aws.LogDebug
+		insecure := false
+		httpClient := &http.Client{
+			Transport: &http.Transport{
+				// set some values to check if they are passed through
+				MaxIdleConns:        7,
+				MaxIdleConnsPerHost: 13,
+				MaxConnsPerHost:     17,
+			},
+		}
+		s3Agent, err := NewS3Agent(accessKey, secretKey, endpoint, debug, nil, insecure, httpClient)
+		assert.NoError(t, err)
+		assert.Equal(t, &logLevel, s3Agent.Client.Config.LogLevel)
+		assert.NotNil(t, s3Agent.Client.Config.HTTPClient.Transport)
+		assert.True(t, *s3Agent.Client.Config.DisableSSL)
+		transport := s3Agent.Client.Config.HTTPClient.Transport
+		assert.Equal(t, 7, transport.(*http.Transport).MaxIdleConns)
+		assert.Equal(t, 13, transport.(*http.Transport).MaxIdleConnsPerHost)
+		assert.Equal(t, 17, transport.(*http.Transport).MaxConnsPerHost)
+	})
 }
diff --git a/tests/framework/clients/bucket.go b/tests/framework/clients/bucket.go
index 6b86aca379cd..fd0216d7e811 100644
--- a/tests/framework/clients/bucket.go
+++ b/tests/framework/clients/bucket.go
@@ -157,11 +157,7 @@ func (b *BucketOperation) CheckBucketNotificationSetonRGW(namespace, storeName,
 	s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
 	s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
 	s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
-	if tlsEnabled {
-		s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
-	} else {
-		s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
-	}
+	s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, tlsEnabled, nil)
 	if err != nil {
 		logger.Infof("failed to s3client due to %v", err)
 		return false
diff --git a/tests/integration/ceph_bucket_notification_test.go b/tests/integration/ceph_bucket_notification_test.go
index d8330ec99166..8e0bdfdfac77 100644
--- a/tests/integration/ceph_bucket_notification_test.go
+++ b/tests/integration/ceph_bucket_notification_test.go
@@ -123,11 +123,8 @@ func testBucketNotifications(s *suite.Suite, helper *clients.TestClient, k8sh *u
 		s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
 		s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
 		s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
-		if objectStore.Spec.IsTLSEnabled() {
-			s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
-		} else {
-			s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
-		}
+		insecure := objectStore.Spec.IsTLSEnabled()
+		s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
 
 		assert.Nil(t, err)
 		logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
@@ -227,11 +224,8 @@ func testBucketNotifications(s *suite.Suite, helper *clients.TestClient, k8sh *u
 		s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
 		s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
 		s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
-		if objectStore.Spec.IsTLSEnabled() {
-			s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
-		} else {
-			s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
-		}
+		insecure := objectStore.Spec.IsTLSEnabled()
+		s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
 
 		assert.Nil(t, err)
 		logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go
index 3763a3c1e20a..8ef593ca4e22 100644
--- a/tests/integration/ceph_object_test.go
+++ b/tests/integration/ceph_object_test.go
@@ -218,11 +218,8 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh
 		s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
 		s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
 		s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
-		if objectStore.Spec.IsTLSEnabled() {
-			s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
-		} else {
-			s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
-		}
+		insecure := objectStore.Spec.IsTLSEnabled()
+		s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
 
 		assert.Nil(t, err)
 		logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
@@ -334,11 +331,8 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh
 			s3AccessKey := string(secret.Data["AWS_ACCESS_KEY_ID"])
 			s3SecretKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"])
 
-			if objectStore.Spec.IsTLSEnabled() {
-				s3client, err = rgw.NewInsecureS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true)
-			} else {
-				s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil)
-			}
+			insecure := objectStore.Spec.IsTLSEnabled()
+			s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
 			assert.Nil(t, err)
 			logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
 		})

From 97b904c717f69cf6f918bc2334407589cb1537ce Mon Sep 17 00:00:00 2001
From: Joshua Hoblitt <josh@hoblitt.com>
Date: Mon, 21 Oct 2024 17:21:27 -0700
Subject: [PATCH 22/24] object: add obc bucketPolicy

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
---
 PendingReleaseNotes.md                        |   1 +
 .../ceph/object/bucket/provisioner.go         |  94 ++++++-
 .../ceph/object/bucket/provisioner_test.go    | 240 +++++++++---------
 pkg/operator/ceph/object/bucket/util.go       |   5 +
 4 files changed, 205 insertions(+), 135 deletions(-)

diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md
index a43b0c75c30a..b399d84cd37a 100644
--- a/PendingReleaseNotes.md
+++ b/PendingReleaseNotes.md
@@ -10,3 +10,4 @@
 - Enable periodic monitoring for CephBlockPoolRadosNamespaces mirroring (see [#14896](https://github.com/rook/rook/pull/14896)).
 - Allow migration of PVC based OSDs to enable or disable encryption (see [#14776](https://github.com/rook/rook/pull/14776)).
 - Support `rgw_enable_apis` option for CephObjectStore (see [#15064](https://github.com/rook/rook/pull/15064)).
+- ObjectBucketClaim management of s3 bucket policy via the `bucketPolicy` field (see [#15138](https://github.com/rook/rook/pull/15138)).
diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go
index eba55118602b..d2d7cb9cc627 100644
--- a/pkg/operator/ceph/object/bucket/provisioner.go
+++ b/pkg/operator/ceph/object/bucket/provisioner.go
@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/ceph/go-ceph/rgw/admin"
 	"github.com/coreos/pkg/capnslog"
 	"github.com/google/go-cmp/cmp"
@@ -62,6 +63,7 @@ type additionalConfigSpec struct {
 	maxSize          *int64
 	bucketMaxObjects *int64
 	bucketMaxSize    *int64
+	bucketPolicy     *string
 }
 
 var _ apibkt.Provisioner = &Provisioner{}
@@ -129,7 +131,12 @@ func (p Provisioner) Provision(options *apibkt.BucketOptions) (*bktv1alpha1.Obje
 	}
 	logger.Infof("set user %q bucket max to %d", p.cephUserName, singleBucketQuota)
 
-	err = p.setAdditionalSettings(options)
+	additionalConfig, err := additionalConfigSpecFromMap(options.ObjectBucketClaim.Spec.AdditionalConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to process additionalConfig")
+	}
+
+	err = p.setAdditionalSettings(additionalConfig)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to set additional settings for OBC %q in NS %q associated with CephObjectStore %q in NS %q", options.ObjectBucketClaim.Name, options.ObjectBucketClaim.Namespace, p.objectStoreName, p.clusterInfo.Namespace)
 	}
@@ -173,6 +180,24 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
 		return nil, err
 	}
 
+	additionalConfig, err := additionalConfigSpecFromMap(options.ObjectBucketClaim.Spec.AdditionalConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to process additionalConfig")
+	}
+
+	// setting quota limit if it is enabled
+	err = p.setAdditionalSettings(additionalConfig)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to set additional settings for OBC %q in NS %q associated with CephObjectStore %q in NS %q", options.ObjectBucketClaim.Name, options.ObjectBucketClaim.Namespace, p.objectStoreName, p.clusterInfo.Namespace)
+	}
+
+	if additionalConfig.bucketPolicy != nil {
+		// if the user is managing the bucket policy, there's nothing else to do
+		return p.composeObjectBucket(), nil
+	}
+
+	// generate the bucket policy if it isn't managed by the user
+
 	// if the policy does not exist, we'll create a new and append the statement to it
 	policy, err := p.s3Agent.GetBucketPolicy(p.bucketName)
 	if err != nil {
@@ -202,12 +227,6 @@ func (p Provisioner) Grant(options *apibkt.BucketOptions) (*bktv1alpha1.ObjectBu
 		return nil, err
 	}
 
-	// setting quota limit if it is enabled
-	err = p.setAdditionalSettings(options)
-	if err != nil {
-		return nil, errors.Wrapf(err, "failed to set additional settings for OBC %q in NS %q associated with CephObjectStore %q in NS %q", options.ObjectBucketClaim.Name, options.ObjectBucketClaim.Namespace, p.objectStoreName, p.clusterInfo.Namespace)
-	}
-
 	// returned ob with connection info
 	return p.composeObjectBucket(), nil
 }
@@ -552,13 +571,8 @@ func (p *Provisioner) populateDomainAndPort(sc *storagev1.StorageClass) error {
 }
 
 // Check for additional options mentioned in OBC and set them accordingly
-func (p *Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error {
-	additionalConfig, err := additionalConfigSpecFromMap(options.ObjectBucketClaim.Spec.AdditionalConfig)
-	if err != nil {
-		return errors.Wrap(err, "failed to process additionalConfig")
-	}
-
-	err = p.setUserQuota(additionalConfig)
+func (p *Provisioner) setAdditionalSettings(additionalConfig *additionalConfigSpec) error {
+	err := p.setUserQuota(additionalConfig)
 	if err != nil {
 		return errors.Wrap(err, "failed to set user quota")
 	}
@@ -568,6 +582,11 @@ func (p *Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error
 		return errors.Wrap(err, "failed to set bucket quota")
 	}
 
+	err = p.setBucketPolicy(additionalConfig)
+	if err != nil {
+		return errors.Wrap(err, "failed to set bucket policy")
+	}
+
 	return nil
 }
 
@@ -675,6 +694,53 @@ func (p *Provisioner) setBucketQuota(additionalConfig *additionalConfigSpec) err
 	return nil
 }
 
+func (p *Provisioner) setBucketPolicy(additionalConfig *additionalConfigSpec) error {
+	svc := p.s3Agent.Client
+	var livePolicy *string
+
+	policyResp, err := svc.GetBucketPolicy(&s3.GetBucketPolicyInput{
+		Bucket: &p.bucketName,
+	})
+	if err != nil {
+		// when no policy is set, an err with code "NoSuchBucketPolicy" is returned
+		if aerr, ok := err.(awserr.Error); ok {
+			if aerr.Code() != "NoSuchBucketPolicy" {
+				return errors.Wrapf(err, "failed to fetch policy for bucket %q", p.bucketName)
+			}
+		}
+	} else {
+		livePolicy = policyResp.Policy
+	}
+
+	diff := cmp.Diff(&livePolicy, additionalConfig.bucketPolicy)
+	if diff == "" {
+		// policy is in sync
+		return nil
+	}
+
+	logger.Debugf("Policy for bucket %q has changed. diff:%s", p.bucketName, diff)
+	if additionalConfig.bucketPolicy == nil {
+		// if policy is out of sync and the new policy is nil, we should delete the live policy
+		_, err = svc.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{
+			Bucket: &p.bucketName,
+		})
+		if err != nil {
+			return errors.Wrapf(err, "failed to delete policy for bucket %q", p.bucketName)
+		}
+	} else {
+		// set the new policy
+		_, err = svc.PutBucketPolicy(&s3.PutBucketPolicyInput{
+			Bucket: &p.bucketName,
+			Policy: additionalConfig.bucketPolicy,
+		})
+		if err != nil {
+			return errors.Wrapf(err, "failed to set policy for bucket %q", p.bucketName)
+		}
+	}
+
+	return nil
+}
+
 func (p *Provisioner) setTlsCaCert() error {
 	objStore, err := p.getObjectStore()
 	if err != nil {
diff --git a/pkg/operator/ceph/object/bucket/provisioner_test.go b/pkg/operator/ceph/object/bucket/provisioner_test.go
index 467b0a54fa0a..e5a6ea478eb3 100644
--- a/pkg/operator/ceph/object/bucket/provisioner_test.go
+++ b/pkg/operator/ceph/object/bucket/provisioner_test.go
@@ -25,9 +25,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/ceph/go-ceph/rgw/admin"
-	"github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
-	apibkt "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner/api"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	rookclient "github.com/rook/rook/pkg/client/clientset/versioned/fake"
 	"github.com/rook/rook/pkg/clusterd"
@@ -146,7 +145,7 @@ func TestQuanityToInt64(t *testing.T) {
 	}
 }
 
-func TestProvisioner_setAdditionalSettings(t *testing.T) {
+func TestProvisioner_setUserQuota(t *testing.T) {
 	newProvisioner := func(t *testing.T, getResult *map[string]string, getSeen, putSeen *map[string][]string) *Provisioner {
 		mockClient := &object.MockClient{
 			MockDo: func(req *http.Request) (*http.Response, error) {
@@ -196,10 +195,9 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		return p
 	}
 
-	t.Run("user and bucket quota should remain disabled", func(t *testing.T) {
+	t.Run("user quota should remain disabled", func(t *testing.T) {
 		getResult := map[string]string{
-			userPath:   `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
-			bucketPath: `{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}}`,
+			userPath: `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
 		}
 		getSeen := map[string][]string{}
 		putSeen := map[string][]string{}
@@ -207,28 +205,18 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
 		p.setBucketName("bob")
 
-		err := p.setAdditionalSettings(&apibkt.BucketOptions{
-			ObjectBucketClaim: &v1alpha1.ObjectBucketClaim{
-				Spec: v1alpha1.ObjectBucketClaimSpec{
-					AdditionalConfig: map[string]string{},
-				},
-			},
-		})
+		err := p.setUserQuota(&additionalConfigSpec{})
 		assert.NoError(t, err)
 
 		assert.Len(t, getSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("uid=bob", getSeen[userPath]), 1)
-		assert.Len(t, getSeen[bucketPath], 1)
-		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
 
-		assert.Len(t, putSeen[userPath], 0)   // user quota should not be touched
-		assert.Len(t, putSeen[bucketPath], 0) // bucket quota should not be touched
+		assert.Len(t, putSeen[userPath], 0) // user quota should not be touched
 	})
 
-	t.Run("user and bucket quota should be disabled", func(t *testing.T) {
+	t.Run("user quota should be disabled", func(t *testing.T) {
 		getResult := map[string]string{
-			userPath:   `{"enabled":true,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":2}`,
-			bucketPath: `{"owner": "bob","bucket_quota":{"enabled":true,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":3}}`,
+			userPath: `{"enabled":true,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":2}`,
 		}
 		getSeen := map[string][]string{}
 		putSeen := map[string][]string{}
@@ -236,30 +224,19 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
 		p.setBucketName("bob")
 
-		err := p.setAdditionalSettings(&apibkt.BucketOptions{
-			ObjectBucketClaim: &v1alpha1.ObjectBucketClaim{
-				Spec: v1alpha1.ObjectBucketClaimSpec{
-					AdditionalConfig: map[string]string{},
-				},
-			},
-		})
+		err := p.setUserQuota(&additionalConfigSpec{})
 		assert.NoError(t, err)
 
 		assert.Len(t, getSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("uid=bob", getSeen[userPath]), 1)
-		assert.Len(t, getSeen[bucketPath], 1)
-		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
 
 		assert.Len(t, putSeen[userPath], 1)
 		assert.Equal(t, 1, numberOfCallsWithValue("enabled=false", putSeen[userPath]))
-		assert.Len(t, putSeen[bucketPath], 1)
-		assert.Equal(t, 1, numberOfCallsWithValue("enabled=false", putSeen[bucketPath]))
 	})
 
 	t.Run("user maxSize quota should be enabled", func(t *testing.T) {
 		getResult := map[string]string{
-			userPath:   `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
-			bucketPath: `{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1}}`,
+			userPath: `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
 		}
 		getSeen := map[string][]string{}
 		putSeen := map[string][]string{}
@@ -267,32 +244,22 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
 		p.setBucketName("bob")
 
-		err := p.setAdditionalSettings(&apibkt.BucketOptions{
-			ObjectBucketClaim: &v1alpha1.ObjectBucketClaim{
-				Spec: v1alpha1.ObjectBucketClaimSpec{
-					AdditionalConfig: map[string]string{
-						"maxSize": "2",
-					},
-				},
-			},
+		err := p.setUserQuota(&additionalConfigSpec{
+			maxSize: aws.Int64(2),
 		})
 		assert.NoError(t, err)
 
 		assert.Len(t, getSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("uid=bob", getSeen[userPath]), 1)
-		assert.Len(t, getSeen[bucketPath], 1)
-		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
 
 		assert.Len(t, putSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("enabled=true", putSeen[userPath]), 1)
 		assert.Equal(t, numberOfCallsWithValue("max-size=2", putSeen[userPath]), 1)
-		assert.Len(t, putSeen[bucketPath], 0) // bucket quota should not be touched
 	})
 
 	t.Run("user maxObjects quota should be enabled", func(t *testing.T) {
 		getResult := map[string]string{
-			userPath:   `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
-			bucketPath: `{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1}}`,
+			userPath: `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
 		}
 		getSeen := map[string][]string{}
 		putSeen := map[string][]string{}
@@ -300,32 +267,22 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
 		p.setBucketName("bob")
 
-		err := p.setAdditionalSettings(&apibkt.BucketOptions{
-			ObjectBucketClaim: &v1alpha1.ObjectBucketClaim{
-				Spec: v1alpha1.ObjectBucketClaimSpec{
-					AdditionalConfig: map[string]string{
-						"maxObjects": "2",
-					},
-				},
-			},
+		err := p.setUserQuota(&additionalConfigSpec{
+			maxObjects: aws.Int64(2),
 		})
 		assert.NoError(t, err)
 
 		assert.Len(t, getSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("uid=bob", getSeen[userPath]), 1)
-		assert.Len(t, getSeen[bucketPath], 1)
-		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
 
 		assert.Len(t, putSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("enabled=true", putSeen[userPath]), 1)
 		assert.Equal(t, numberOfCallsWithValue("max-objects=2", putSeen[userPath]), 1)
-		assert.Len(t, putSeen[bucketPath], 0) // bucket quota should not be touched
 	})
 
 	t.Run("user maxObjects and maxSize quotas should be enabled", func(t *testing.T) {
 		getResult := map[string]string{
-			userPath:   `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
-			bucketPath: `{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1}}`,
+			userPath: `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
 		}
 		getSeen := map[string][]string{}
 		putSeen := map[string][]string{}
@@ -333,34 +290,24 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
 		p.setBucketName("bob")
 
-		err := p.setAdditionalSettings(&apibkt.BucketOptions{
-			ObjectBucketClaim: &v1alpha1.ObjectBucketClaim{
-				Spec: v1alpha1.ObjectBucketClaimSpec{
-					AdditionalConfig: map[string]string{
-						"maxObjects": "2",
-						"maxSize":    "3",
-					},
-				},
-			},
+		err := p.setUserQuota(&additionalConfigSpec{
+			maxObjects: aws.Int64(2),
+			maxSize:    aws.Int64(3),
 		})
 		assert.NoError(t, err)
 
 		assert.Len(t, getSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("uid=bob", getSeen[userPath]), 1)
-		assert.Len(t, getSeen[bucketPath], 1)
-		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
 
 		assert.Len(t, putSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("enabled=true", putSeen[userPath]), 1)
 		assert.Equal(t, numberOfCallsWithValue("max-objects=2", putSeen[userPath]), 1)
 		assert.Equal(t, numberOfCallsWithValue("max-size=3", putSeen[userPath]), 1)
-		assert.Len(t, putSeen[bucketPath], 0) // bucket quota should not be touched
 	})
 
 	t.Run("user quotas are enabled and need updated enabled", func(t *testing.T) {
 		getResult := map[string]string{
-			userPath:   `{"enabled":true,"check_on_raw":false,"max_size":1,"max_size_kb":0,"max_objects":1}`,
-			bucketPath: `{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1}}`,
+			userPath: `{"enabled":true,"check_on_raw":false,"max_size":1,"max_size_kb":0,"max_objects":1}`,
 		}
 		getSeen := map[string][]string{}
 		putSeen := map[string][]string{}
@@ -368,33 +315,113 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
 		p.setBucketName("bob")
 
-		err := p.setAdditionalSettings(&apibkt.BucketOptions{
-			ObjectBucketClaim: &v1alpha1.ObjectBucketClaim{
-				Spec: v1alpha1.ObjectBucketClaimSpec{
-					AdditionalConfig: map[string]string{
-						"maxObjects": "12",
-						"maxSize":    "13",
-					},
-				},
-			},
+		err := p.setUserQuota(&additionalConfigSpec{
+			maxObjects: aws.Int64(12),
+			maxSize:    aws.Int64(13),
 		})
 		assert.NoError(t, err)
 
 		assert.Len(t, getSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("uid=bob", getSeen[userPath]), 1)
-		assert.Len(t, getSeen[bucketPath], 1)
-		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
 
 		assert.Len(t, putSeen[userPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("enabled=true", putSeen[userPath]), 1)
 		assert.Equal(t, numberOfCallsWithValue("max-objects=12", putSeen[userPath]), 1)
 		assert.Equal(t, numberOfCallsWithValue("max-size=13", putSeen[userPath]), 1)
+	})
+}
+
+func TestProvisioner_setBucketQuota(t *testing.T) {
+	newProvisioner := func(t *testing.T, getResult *map[string]string, getSeen, putSeen *map[string][]string) *Provisioner {
+		mockClient := &object.MockClient{
+			MockDo: func(req *http.Request) (*http.Response, error) {
+				path := req.URL.Path
+
+				// t.Logf("HTTP req: %#v", req)
+				t.Logf("HTTP %s: %s %s", req.Method, path, req.URL.RawQuery)
+
+				statusCode := 200
+				if _, ok := (*getResult)[path]; !ok {
+					// path not configured
+					statusCode = 500
+				}
+				responseBody := []byte(`[]`)
+
+				switch method := req.Method; method {
+				case http.MethodGet:
+					(*getSeen)[path] = append((*getSeen)[path], req.URL.RawQuery)
+					responseBody = []byte((*getResult)[path])
+				case http.MethodPut:
+					if (*putSeen)[path] == nil {
+						(*putSeen)[path] = []string{}
+					}
+					(*putSeen)[path] = append((*putSeen)[path], req.URL.RawQuery)
+				default:
+					panic(fmt.Sprintf("unexpected request: %q. method %q. path %q", req.URL.RawQuery, req.Method, path))
+				}
+
+				return &http.Response{
+					StatusCode: statusCode,
+					Body:       io.NopCloser(bytes.NewReader(responseBody)),
+				}, nil
+			},
+		}
+
+		adminClient, err := admin.New("rgw.test", "accesskey", "secretkey", mockClient)
+		assert.NoError(t, err)
+
+		p := &Provisioner{
+			clusterInfo: &client.ClusterInfo{
+				Context: context.Background(),
+			},
+			cephUserName:   "bob",
+			adminOpsClient: adminClient,
+		}
+
+		return p
+	}
+
+	t.Run("bucket quota should remain disabled", func(t *testing.T) {
+		getResult := map[string]string{
+			bucketPath: `{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}}`,
+		}
+		getSeen := map[string][]string{}
+		putSeen := map[string][]string{}
+
+		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
+		p.setBucketName("bob")
+
+		err := p.setBucketQuota(&additionalConfigSpec{})
+		assert.NoError(t, err)
+
+		assert.Len(t, getSeen[bucketPath], 1)
+		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
+
 		assert.Len(t, putSeen[bucketPath], 0) // bucket quota should not be touched
 	})
 
+	t.Run("bucket quota should be disabled", func(t *testing.T) {
+		getResult := map[string]string{
+			bucketPath: `{"owner": "bob","bucket_quota":{"enabled":true,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":3}}`,
+		}
+		getSeen := map[string][]string{}
+		putSeen := map[string][]string{}
+
+		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
+		p.setBucketName("bob")
+
+		err := p.setBucketQuota(&additionalConfigSpec{})
+		assert.NoError(t, err)
+
+		assert.Len(t, getSeen[bucketPath], 1)
+		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
+
+		assert.Len(t, putSeen[bucketPath], 1)
+		assert.Equal(t, 1, numberOfCallsWithValue("enabled=false", putSeen[bucketPath]))
+	})
+
 	t.Run("bucket maxObjects quota should be enabled", func(t *testing.T) {
 		getResult := map[string]string{
-			userPath:   `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
 			bucketPath: `{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1,"max_size_kb":0,"max_objects":-1}}`,
 		}
 		getSeen := map[string][]string{}
@@ -403,23 +430,14 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
 		p.setBucketName("bob")
 
-		err := p.setAdditionalSettings(&apibkt.BucketOptions{
-			ObjectBucketClaim: &v1alpha1.ObjectBucketClaim{
-				Spec: v1alpha1.ObjectBucketClaimSpec{
-					AdditionalConfig: map[string]string{
-						"bucketMaxObjects": "4",
-					},
-				},
-			},
+		err := p.setBucketQuota(&additionalConfigSpec{
+			bucketMaxObjects: aws.Int64(4),
 		})
 		assert.NoError(t, err)
 
-		assert.Len(t, getSeen[userPath], 1)
-		assert.Equal(t, numberOfCallsWithValue("uid=bob", getSeen[userPath]), 1)
 		assert.Len(t, getSeen[bucketPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
 
-		assert.Len(t, putSeen[userPath], 0) // user quota should not be touched
 		assert.Len(t, putSeen[bucketPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("enabled=true", putSeen[bucketPath]), 1)
 		assert.Equal(t, numberOfCallsWithValue("max-objects=4", putSeen[bucketPath]), 1)
@@ -427,7 +445,6 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 
 	t.Run("bucket maxSize quota should be enabled", func(t *testing.T) {
 		getResult := map[string]string{
-			userPath:   `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
 			bucketPath: `{"bucket_quota":{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}}`,
 		}
 		getSeen := map[string][]string{}
@@ -436,23 +453,14 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
 		p.setBucketName("bob")
 
-		err := p.setAdditionalSettings(&apibkt.BucketOptions{
-			ObjectBucketClaim: &v1alpha1.ObjectBucketClaim{
-				Spec: v1alpha1.ObjectBucketClaimSpec{
-					AdditionalConfig: map[string]string{
-						"bucketMaxSize": "5",
-					},
-				},
-			},
+		err := p.setBucketQuota(&additionalConfigSpec{
+			bucketMaxSize: aws.Int64(5),
 		})
 		assert.NoError(t, err)
 
-		assert.Len(t, getSeen[userPath], 1)
-		assert.Equal(t, numberOfCallsWithValue("uid=bob", getSeen[userPath]), 1)
 		assert.Len(t, getSeen[bucketPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
 
-		assert.Len(t, putSeen[userPath], 0) // user quota should not be touched
 		assert.Len(t, putSeen[bucketPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("enabled=true", putSeen[bucketPath]), 1)
 		assert.Equal(t, numberOfCallsWithValue("max-size=5", putSeen[bucketPath]), 1)
@@ -460,7 +468,6 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 
 	t.Run("bucket quotas are enabled and need updated enabled", func(t *testing.T) {
 		getResult := map[string]string{
-			userPath:   `{"enabled":false,"check_on_raw":false,"max_size":-1024,"max_size_kb":0,"max_objects":-1}`,
 			bucketPath: `{"bucket_quota":{"enabled":true,"check_on_raw":false,"max_size":5,"max_size_kb":0,"max_objects":4}}`,
 		}
 		getSeen := map[string][]string{}
@@ -469,24 +476,15 @@ func TestProvisioner_setAdditionalSettings(t *testing.T) {
 		p := newProvisioner(t, &getResult, &getSeen, &putSeen)
 		p.setBucketName("bob")
 
-		err := p.setAdditionalSettings(&apibkt.BucketOptions{
-			ObjectBucketClaim: &v1alpha1.ObjectBucketClaim{
-				Spec: v1alpha1.ObjectBucketClaimSpec{
-					AdditionalConfig: map[string]string{
-						"bucketMaxObjects": "14",
-						"bucketMaxSize":    "15",
-					},
-				},
-			},
+		err := p.setBucketQuota(&additionalConfigSpec{
+			bucketMaxObjects: aws.Int64(14),
+			bucketMaxSize:    aws.Int64(15),
 		})
 		assert.NoError(t, err)
 
-		assert.Len(t, getSeen[userPath], 1)
-		assert.Equal(t, numberOfCallsWithValue("uid=bob", getSeen[userPath]), 1)
 		assert.Len(t, getSeen[bucketPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("bucket=bob", getSeen[bucketPath]), 1)
 
-		assert.Len(t, putSeen[userPath], 0) // user quota should not be touched
 		assert.Len(t, putSeen[bucketPath], 1)
 		assert.Equal(t, numberOfCallsWithValue("enabled=true", putSeen[bucketPath]), 1)
 		assert.Equal(t, numberOfCallsWithValue("max-objects=14", putSeen[bucketPath]), 1)
diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go
index de1936d2649b..22a0e04db14a 100644
--- a/pkg/operator/ceph/object/bucket/util.go
+++ b/pkg/operator/ceph/object/bucket/util.go
@@ -120,6 +120,11 @@ func additionalConfigSpecFromMap(config map[string]string) (*additionalConfigSpe
 		}
 	}
 
+	if _, ok := config["bucketPolicy"]; ok {
+		policy := config["bucketPolicy"]
+		spec.bucketPolicy = &policy
+	}
+
 	return &spec, nil
 }
 

From 8fd34e88bbaf105d4085dfa9c08213320e5d8896 Mon Sep 17 00:00:00 2001
From: Joshua Hoblitt <josh@hoblitt.com>
Date: Mon, 4 Nov 2024 15:41:05 -0700
Subject: [PATCH 23/24] test: add obc bucketPolicy integration test

Signed-off-by: Joshua Hoblitt <josh@hoblitt.com>
---
 tests/integration/ceph_object_test.go | 245 ++++++++++++++++++++++++++
 1 file changed, 245 insertions(+)

diff --git a/tests/integration/ceph_object_test.go b/tests/integration/ceph_object_test.go
index 8ef593ca4e22..d3faf041274c 100644
--- a/tests/integration/ceph_object_test.go
+++ b/tests/integration/ceph_object_test.go
@@ -23,6 +23,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
@@ -31,6 +33,7 @@ import (
 	"github.com/rook/rook/tests/framework/installer"
 	"github.com/rook/rook/tests/framework/utils"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -420,6 +423,248 @@ func testObjectStoreOperations(s *suite.Suite, helper *clients.TestClient, k8sh
 		})
 	})
 
+	t.Run("OBC bucket policy management", func(t *testing.T) {
+		bucketName := "bucket-policy-test"
+		var obName string
+		var s3client *rgw.S3Agent
+		bucketPolicy1 := `
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "arn:aws:iam:::user/foo"
+            },
+            "Action": [
+                "s3:GetObject",
+                "s3:PutObject",
+                "s3:DeleteObject",
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": [
+                "arn:aws:s3:::bucket-policy-test/*"
+            ]
+        }
+    ]
+}
+		`
+
+		bucketPolicy2 := `
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "arn:aws:iam:::user/bar"
+            },
+            "Action": [
+                "s3:GetObject",
+                "s3:ListBucket",
+                "s3:GetBucketLocation"
+            ],
+            "Resource": [
+                "arn:aws:s3:::bucket-policy-test/*"
+            ]
+        }
+    ]
+}
+		`
+
+		t.Run("create obc with bucketPolicy", func(t *testing.T) {
+			newObc := v1alpha1.ObjectBucketClaim{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      bucketName,
+					Namespace: namespace,
+				},
+				Spec: v1alpha1.ObjectBucketClaimSpec{
+					BucketName:       bucketName,
+					StorageClassName: bucketStorageClassName,
+					AdditionalConfig: map[string]string{
+						"bucketPolicy": bucketPolicy1,
+					},
+				},
+			}
+			_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Create(ctx, &newObc, metav1.CreateOptions{})
+			assert.Nil(t, err)
+			obcBound := utils.Retry(20, 2*time.Second, "OBC is Bound", func() bool {
+				liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+				if err != nil {
+					return false
+				}
+
+				return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound
+			})
+			assert.True(t, obcBound)
+
+			// wait until obc is Bound to lookup the ob name
+			obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+			assert.Nil(t, err)
+			obName = obc.Spec.ObjectBucketName
+
+			obBound := utils.Retry(20, 2*time.Second, "OB is Bound", func() bool {
+				liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
+				if err != nil {
+					return false
+				}
+
+				return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound
+			})
+			assert.True(t, obBound)
+
+			// verify that bucketPolicy is set
+			assert.Equal(t, bucketPolicy1, obc.Spec.AdditionalConfig["bucketPolicy"])
+
+			// as the tests are running external to k8s, the internal svc can't be used
+			labelSelector := "rgw=" + storeName
+			services, err := k8sh.Clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
+			assert.Nil(t, err)
+			assert.Equal(t, 1, len(services.Items))
+			s3endpoint := services.Items[0].Spec.ClusterIP + ":80"
+
+			secret, err := k8sh.Clientset.CoreV1().Secrets(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+			assert.Nil(t, err)
+			s3AccessKey := string(secret.Data["AWS_ACCESS_KEY_ID"])
+			s3SecretKey := string(secret.Data["AWS_SECRET_ACCESS_KEY"])
+
+			insecure := objectStore.Spec.IsTLSEnabled()
+			s3client, err = rgw.NewS3Agent(s3AccessKey, s3SecretKey, s3endpoint, true, nil, insecure, nil)
+			assert.Nil(t, err)
+			logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)
+		})
+
+		t.Run("policy was applied verbatim to bucket", func(t *testing.T) {
+			policyResp, err := s3client.Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
+				Bucket: &bucketName,
+			})
+			require.NoError(t, err)
+			assert.Equal(t, bucketPolicy1, *policyResp.Policy)
+		})
+
+		t.Run("update obc bucketPolicy", func(t *testing.T) {
+			obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+			assert.Nil(t, err)
+
+			obc.Spec.AdditionalConfig["bucketPolicy"] = bucketPolicy2
+
+			_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{})
+			assert.Nil(t, err)
+			obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool {
+				liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+				if err != nil {
+					return false
+				}
+
+				return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound
+			})
+			assert.True(t, obcBound)
+
+			// wait until obc is Bound to lookup the ob name
+			obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+			assert.Nil(t, err)
+			obName = obc.Spec.ObjectBucketName
+
+			obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool {
+				liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
+				if err != nil {
+					return false
+				}
+
+				return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound
+			})
+			assert.True(t, obBound)
+
+			// verify that updated bucketPolicy is set
+			assert.Equal(t, bucketPolicy2, obc.Spec.AdditionalConfig["bucketPolicy"])
+		})
+
+		t.Run("policy update applied verbatim to bucket", func(t *testing.T) {
+			var livePolicy string
+			utils.Retry(20, time.Second, "policy changed", func() bool {
+				policyResp, err := s3client.Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
+					Bucket: &bucketName,
+				})
+				if err != nil {
+					return false
+				}
+
+				livePolicy = *policyResp.Policy
+				return bucketPolicy2 == livePolicy
+			})
+			assert.Equal(t, bucketPolicy2, livePolicy)
+		})
+
+		t.Run("remove obc bucketPolicy", func(t *testing.T) {
+			obc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+			assert.Nil(t, err)
+
+			obc.Spec.AdditionalConfig = map[string]string{}
+
+			_, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Update(ctx, obc, metav1.UpdateOptions{})
+			assert.Nil(t, err)
+			obcBound := utils.Retry(20, time.Second, "OBC is Bound", func() bool {
+				liveObc, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+				if err != nil {
+					return false
+				}
+
+				return liveObc.Status.Phase == v1alpha1.ObjectBucketClaimStatusPhaseBound
+			})
+			assert.True(t, obcBound)
+
+			// wait until obc is Bound to lookup the ob name
+			obc, err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+			assert.Nil(t, err)
+			obName = obc.Spec.ObjectBucketName
+
+			obBound := utils.Retry(20, time.Second, "OB is Bound", func() bool {
+				liveOb, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
+				if err != nil {
+					return false
+				}
+
+				return liveOb.Status.Phase == v1alpha1.ObjectBucketStatusPhaseBound
+			})
+			assert.True(t, obBound)
+
+			// verify that bucketPolicy is unset
+			assert.NotContains(t, obc.Spec.AdditionalConfig, "bucketPolicy")
+		})
+
+		t.Run("policy was removed from bucket", func(t *testing.T) {
+			var err error
+			utils.Retry(20, time.Second, "policy is gone", func() bool {
+				_, err = s3client.Client.GetBucketPolicy(&s3.GetBucketPolicyInput{
+					Bucket: &bucketName,
+				})
+				return err != nil
+			})
+			require.Error(t, err)
+			require.Implements(t, (*awserr.Error)(nil), err)
+			aerr, _ := err.(awserr.Error)
+			assert.Equal(t, aerr.Code(), "NoSuchBucketPolicy")
+		})
+
+		t.Run("delete bucket", func(t *testing.T) {
+			err = k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Delete(ctx, bucketName, metav1.DeleteOptions{})
+			assert.Nil(t, err)
+
+			absent := utils.Retry(20, time.Second, "OBC is absent", func() bool {
+				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBucketClaims(namespace).Get(ctx, bucketName, metav1.GetOptions{})
+				return err != nil
+			})
+			assert.True(t, absent)
+
+			absent = utils.Retry(20, time.Second, "OB is absent", func() bool {
+				_, err := k8sh.BucketClientset.ObjectbucketV1alpha1().ObjectBuckets().Get(ctx, obName, metav1.GetOptions{})
+				return err != nil
+			})
+			assert.True(t, absent)
+		})
+	})
+
 	t.Run("Regression check: OBC does not revert to Pending phase", func(t *testing.T) {
 		// A bug exists in older versions of lib-bucket-provisioner that will revert a bucket and claim
 		// back to "Pending" phase after being created and initially "Bound" by looping infinitely in

From 02fee54b31701f98b49acb935199821f4644665f Mon Sep 17 00:00:00 2001
From: Ceph Jenkins <ceph-jenkins@redhat.com>
Date: Sun, 15 Dec 2024 03:06:17 -0500
Subject: [PATCH 24/24] csv: add additional csv changes that other commits
 bring

add generated csv changes

Signed-off-by: Ceph Jenkins <ceph-jenkins@redhat.com>
---
 build/csv/ceph/ceph.rook.io_cephfilesystems.yaml  |  4 ++++
 build/csv/ceph/ceph.rook.io_cephobjectstores.yaml | 14 ++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/build/csv/ceph/ceph.rook.io_cephfilesystems.yaml b/build/csv/ceph/ceph.rook.io_cephfilesystems.yaml
index fce67dc19fa6..9cc1990896a8 100644
--- a/build/csv/ceph/ceph.rook.io_cephfilesystems.yaml
+++ b/build/csv/ceph/ceph.rook.io_cephfilesystems.yaml
@@ -237,6 +237,8 @@ spec:
                           type: object
                         type: array
                     type: object
+                  name:
+                    type: string
                   parameters:
                     additionalProperties:
                       type: string
@@ -1081,6 +1083,8 @@ spec:
                 type: object
               preserveFilesystemOnDelete:
                 type: boolean
+              preservePoolNames:
+                type: boolean
               preservePoolsOnDelete:
                 type: boolean
               statusCheck:
diff --git a/build/csv/ceph/ceph.rook.io_cephobjectstores.yaml b/build/csv/ceph/ceph.rook.io_cephobjectstores.yaml
index d56100724710..26ca845b034e 100644
--- a/build/csv/ceph/ceph.rook.io_cephobjectstores.yaml
+++ b/build/csv/ceph/ceph.rook.io_cephobjectstores.yaml
@@ -1402,6 +1402,20 @@ spec:
                 type: boolean
               protocols:
                 properties:
+                  enableAPIs:
+                    items:
+                      enum:
+                      - s3
+                      - s3website
+                      - swift
+                      - swift_auth
+                      - admin
+                      - sts
+                      - iam
+                      - notifications
+                      type: string
+                    nullable: true
+                    type: array
                   s3:
                     nullable: true
                     properties: