From 6142bb0a67d6d69e22a7a35ac6afe0d04198a1d1 Mon Sep 17 00:00:00 2001 From: Rook Date: Mon, 24 Feb 2025 19:08:17 +0000 Subject: [PATCH] Deployed 57fb00c92 to v1.16 in docs/rook with MkDocs 1.6.1 and mike 2.1.3 --- .../ceph-block-pool-crd/index.html | 2 +- .../ceph-rbd-mirror-crd/index.html | 2 +- .../CRDs/Cluster/ceph-cluster-crd/index.html | 4 +- .../advance-external/index.html | 6 +- .../consumer-import/index.html | 6 +- .../provider-export/index.html | 2 +- .../topology-for-external-mode/index.html | 2 +- .../CRDs/Cluster/network-providers/index.html | 2 +- .../CRDs/Cluster/stretch-cluster/index.html | 2 +- .../ceph-filesystem-crd/index.html | 4 +- .../ceph-fs-mirror-crd/index.html | 2 +- .../v1.16/CRDs/ceph-client-crd/index.html | 2 +- docs/rook/v1.16/CRDs/ceph-nfs-crd/index.html | 2 +- .../Contributing/ci-configuration/index.html | 2 +- .../development-environment/index.html | 2 +- .../Contributing/development-flow/index.html | 4 +- .../Getting-Started/ceph-openshift/index.html | 2 +- .../example-configurations/index.html | 4 +- .../v1.16/Getting-Started/intro/index.html | 2 +- .../Getting-Started/quickstart/index.html | 6 +- .../Helm-Charts/ceph-cluster-chart/index.html | 2 +- .../v1.16/Helm-Charts/helm-charts/index.html | 2 +- .../Helm-Charts/operator-chart/index.html | 2 +- .../Advanced/ceph-configuration/index.html | 2 +- .../Advanced/ceph-osd-mgmt/index.html | 4 +- .../block-storage/index.html | 2 +- .../Ceph-CSI/ceph-csi-drivers/index.html | 4 +- .../Ceph-CSI/ceph-csi-snapshot/index.html | 12 +- .../Ceph-CSI/ceph-csi-volume-clone/index.html | 4 +- .../ceph-csi-volume-group-snapshot/index.html | 6 +- .../Ceph-CSI/custom-images/index.html | 2 +- .../Monitoring/ceph-monitoring/index.html | 2 +- .../NFS/nfs-advanced/index.html | 2 +- .../NFS/nfs-csi-driver/index.html | 10 +- .../Storage-Configuration/NFS/nfs/index.html | 4 +- .../ceph-object-multisite/index.html | 4 +- .../object-storage/index.html | 2 +- .../disaster-recovery/index.html | 2 +- .../openshift-common-issues/index.html | 2 +- docs/rook/v1.16/sitemap.xml | 166 +++++++++--------- docs/rook/v1.16/sitemap.xml.gz | Bin 1066 -> 1066 bytes 41 files changed, 148 insertions(+), 148 deletions(-) diff --git a/docs/rook/v1.16/CRDs/Block-Storage/ceph-block-pool-crd/index.html b/docs/rook/v1.16/CRDs/Block-Storage/ceph-block-pool-crd/index.html index b3f976c34..e7907f870 100644 --- a/docs/rook/v1.16/CRDs/Block-Storage/ceph-block-pool-crd/index.html +++ b/docs/rook/v1.16/CRDs/Block-Storage/ceph-block-pool-crd/index.html @@ -147,7 +147,7 @@ size: 4 replicasPerFailureDomain: 2 subFailureDomain: rack -

Pool Settings

Metadata

Spec

Add specific pool properties

With poolProperties you can set any pool property:

1
+

Pool Settings

Metadata

Spec

Add specific pool properties

With poolProperties you can set any pool property:

1
 2
 3
spec:
   parameters:
diff --git a/docs/rook/v1.16/CRDs/Block-Storage/ceph-rbd-mirror-crd/index.html b/docs/rook/v1.16/CRDs/Block-Storage/ceph-rbd-mirror-crd/index.html
index f21f750a5..7384a4208 100644
--- a/docs/rook/v1.16/CRDs/Block-Storage/ceph-rbd-mirror-crd/index.html
+++ b/docs/rook/v1.16/CRDs/Block-Storage/ceph-rbd-mirror-crd/index.html
@@ -11,4 +11,4 @@
   namespace: rook-ceph
 spec:
   count: 1
-

Prerequisites

This guide assumes you have created a Rook cluster as explained in the main Quickstart guide

Settings

If any setting is unspecified, a suitable default will be used automatically.

RBDMirror metadata

RBDMirror Settings

Configuring mirroring peers

\ No newline at end of file +

Prerequisites

This guide assumes you have created a Rook cluster as explained in the main Quickstart guide

Settings

If any setting is unspecified, a suitable default will be used automatically.

RBDMirror metadata

RBDMirror Settings

Configuring mirroring peers

\ No newline at end of file diff --git a/docs/rook/v1.16/CRDs/Cluster/ceph-cluster-crd/index.html b/docs/rook/v1.16/CRDs/Cluster/ceph-cluster-crd/index.html index 458f90cbd..01319a660 100644 --- a/docs/rook/v1.16/CRDs/Cluster/ceph-cluster-crd/index.html +++ b/docs/rook/v1.16/CRDs/Cluster/ceph-cluster-crd/index.html @@ -1,11 +1,11 @@ - CephCluster CRD - Rook Ceph Documentation
Skip to content

CephCluster CRD

Rook allows creation and customization of storage clusters through the custom resource definitions (CRDs). There are primarily four different modes in which to create your cluster.

  1. Host Storage Cluster: Consume storage from host paths and raw devices
  2. PVC Storage Cluster: Dynamically provision storage underneath Rook by specifying the storage class Rook should use to consume storage (via PVCs)
  3. Stretched Storage Cluster: Distribute Ceph mons across three zones, while storage (OSDs) is only configured in two zones
  4. External Ceph Cluster: Connect your K8s applications to an external Ceph cluster

See the separate topics for a description and examples of each of these scenarios.

Settings

Settings can be specified at the global level to apply to the cluster as a whole, while other settings can be specified at more fine-grained levels. If any setting is unspecified, a suitable default will be used automatically.

Cluster metadata

  • name: The name that will be used internally for the Ceph cluster. Most commonly the name is the same as the namespace since multiple clusters are not supported in the same namespace.
  • namespace: The Kubernetes namespace that will be created for the Rook cluster. The services, pods, and other resources created by the operator will be added to this namespace. The common scenario is to create a single Rook cluster. If multiple clusters are created, they must not have conflicting devices or host paths.

Cluster Settings

  • external:
    • enable: if true, the cluster will not be managed by Rook but via an external entity. This mode is intended to connect to an existing cluster. In this case, Rook will only consume the external cluster. However, Rook will be able to deploy various daemons in Kubernetes such as object gateways, mds and nfs if an image is provided and will refuse otherwise. If this setting is enabled all the other options will be ignored except cephVersion.image and dataDirHostPath. See external cluster configuration. If cephVersion.image is left blank, Rook will refuse the creation of extra CRs like object, file and nfs.
  • cephVersion: The version information for launching the ceph daemons.
    • image: The image used for running the ceph daemons. For example, quay.io/ceph/ceph:v19.2.1. For more details read the container images section. For the latest ceph images, see the Ceph DockerHub. To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version. Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag v19 will be updated each time a new Squid build is released. Using the general v19 tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster.
    • allowUnsupported: If true, allow an unsupported major version of the Ceph release. Currently Reef and Squid are supported. Future versions such as Tentacle (v20) would require this to be set to true. Should be set to false in production.
    • imagePullPolicy: The image pull policy for the ceph daemon pods. Possible values are Always, IfNotPresent, and Never. The default is IfNotPresent.
  • dataDirHostPath: The path on the host (hostPath) where config and data should be stored for each of the services. If there are multiple clusters, the directory must be unique for each cluster. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Following paths and any of their subpaths must not be used: /etc/ceph, /rook or /var/log/ceph.
    • WARNING: For test scenarios, if you delete a cluster and start a new cluster on the same hosts, the path used by dataDirHostPath must be deleted. Otherwise, stale keys and other config will remain from the previous cluster and the new mons will fail to start. If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes empty dir docs.
  • skipUpgradeChecks: if set to true Rook won't perform any upgrade checks on Ceph daemons during an upgrade. Use this at YOUR OWN RISK, only if you know what you're doing. To understand Rook's upgrade process of Ceph, read the upgrade doc.
  • continueUpgradeAfterChecksEvenIfNotHealthy: if set to true Rook will continue the OSD daemon upgrade process even if the PGs are not clean, or continue with the MDS upgrade even the file system is not healthy.
  • upgradeOSDRequiresHealthyPGs: if set to true OSD upgrade process won't start until PGs are healthy.
  • dashboard: Settings for the Ceph dashboard. To view the dashboard in your browser see the dashboard guide.
    • enabled: Whether to enable the dashboard to view cluster status
    • urlPrefix: Allows to serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy)
    • port: Allows to change the default port where the dashboard is served
    • ssl: Whether to serve the dashboard via SSL, ignored on Ceph versions older than 13.2.2
  • monitoring: Settings for monitoring Ceph using Prometheus. To enable monitoring on your cluster see the monitoring guide.
    • enabled: Whether to enable the prometheus service monitor for an internal cluster. For an external cluster, whether to create an endpoint port for the metrics. Default is false.
    • metricsDisabled: Whether to disable the metrics reported by Ceph. If false, the prometheus mgr module and Ceph exporter are enabled. If true, the prometheus mgr module and Ceph exporter are both disabled. Default is false.
    • externalMgrEndpoints: external cluster manager endpoints
    • externalMgrPrometheusPort: external prometheus manager module port. See external cluster configuration for more details.
    • port: The internal prometheus manager module port where the prometheus mgr module listens. The port may need to be configured when host networking is enabled.
    • interval: The interval for the prometheus module to to scrape targets.
    • exporter: Ceph exporter metrics config.
      • perfCountersPrioLimit: Specifies which performance counters are exported. Corresponds to --prio-limit Ceph exporter flag. 0 - all counters are exported, default is 5.
      • statsPeriodSeconds: Time to wait before sending requests again to exporter server (seconds). Corresponds to --stats-period Ceph exporter flag. Default is 5.
  • network: For the network settings for the cluster, refer to the network configuration settings
  • mon: contains mon related options mon settings For more details on the mons and when to choose a number other than 3, see the mon health doc.
  • mgr: manager top level section
    • count: set number of ceph managers between 1 to 2. The default value is 2. If there are two managers, it is important for all mgr services point to the active mgr and not the standby mgr. Rook automatically updates the label mgr_role on the mgr pods to be either active or standby. Therefore, services need just to add the label mgr_role=active to their selector to point to the active mgr. This applies to all services that rely on the ceph mgr such as the dashboard or the prometheus metrics collector.
    • modules: A list of Ceph manager modules to enable or disable. Note the "dashboard" and "monitoring" modules are already configured by other settings.
  • crashCollector: The settings for crash collector daemon(s).
    • disable: is set to true, the crash collector will not run on any node where a Ceph daemon runs
    • daysToRetain: specifies the number of days to keep crash entries in the Ceph cluster. By default the entries are kept indefinitely.
  • logCollector: The settings for log collector daemon.
    • enabled: if set to true, the log collector will run as a side-car next to each Ceph daemon. The Ceph configuration option log_to_file will be turned on, meaning Ceph daemons will log on files in addition to still logging to container's stdout. These logs will be rotated. In case a daemon terminates with a segfault, the coredump files will be commonly be generated in /var/lib/systemd/coredump directory on the host, depending on the underlying OS location. (default: true)
    • periodicity: how often to rotate daemon's log. (default: 24h). Specified with a time suffix which may be h for hours or d for days. Rotating too often will slightly impact the daemon's performance since the signal briefly interrupts the program.
  • annotations: annotations configuration settings
  • labels: labels configuration settings
  • placement: placement configuration settings
  • resources: resources configuration settings
  • priorityClassNames: priority class names configuration settings
  • storage: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes.
    • useAllNodes: true or false, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values. If individual nodes are specified under the nodes field, then useAllNodes must be set to false.
    • nodes: Names of individual nodes in the cluster that should have their storage included in accordance with either the cluster level configuration specified above or any node specific overrides described in the next section below. useAllNodes must be set to false to use specific nodes and their config. See node settings below.
    • config: Config settings applied to all OSDs on the node unless overridden by devices. See the config settings below.
    • allowDeviceClassUpdate: Whether to allow changing the device class of an OSD after it is created. The default is false to prevent unintentional data movement or CRUSH changes if the device class is changed accidentally.
    • allowOsdCrushWeightUpdate: Whether Rook will resize the OSD CRUSH weight when the OSD PVC size is increased. This allows cluster data to be rebalanced to make most effective use of new OSD space. The default is false since data rebalancing can cause temporary cluster slowdown.
    • storage selection settings
    • Storage Class Device Sets
    • onlyApplyOSDPlacement: Whether the placement specific for OSDs is merged with the all placement. If false, the OSD placement will be merged with the all placement. If true, the OSD placement will be applied and the all placement will be ignored. The placement for OSDs is computed from several different places depending on the type of OSD:
      • For non-PVCs: placement.all and placement.osd
      • For PVCs: placement.all and inside the storageClassDeviceSets from the placement or preparePlacement
    • flappingRestartIntervalHours: Defines the time for which an OSD pod will sleep before restarting, if it stopped due to flapping. Flapping occurs where OSDs are marked down by Ceph more than 5 times in 600 seconds. The OSDs will stay down when flapping since they likely have a bad disk or other issue that needs investigation. If the issue with the OSD is fixed manually, the OSD pod can be manually restarted. The sleep is disabled if this interval is set to 0.
    • scheduleAlways: Whether to always schedule OSD pods on nodes declared explicitly in the "nodes" section, even if they are temporarily not schedulable. If set to true, consider adding placement tolerations for unschedulable nodes.
    • fullRatio: The ratio at which Ceph should block IO if the OSDs are too full. The default is 0.95.
    • backfillFullRatio: The ratio at which Ceph should stop backfilling data if the OSDs are too full. The default is 0.90.
    • nearFullRatio: The ratio at which Ceph should raise a health warning if the cluster is almost full. The default is 0.85.
  • disruptionManagement: The section for configuring management of daemon disruptions
    • managePodBudgets: if true, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically via the strategy outlined in the design. The operator will block eviction of OSDs by default and unblock them safely when drains are detected.
    • osdMaintenanceTimeout: is a duration in minutes that determines how long an entire failureDomain like region/zone/host will be held in noout (in addition to the default DOWN/OUT interval) when it is draining. The default value is 30 minutes.
    • pgHealthCheckTimeout: A duration in minutes that the operator will wait for the placement groups to become healthy (see pgHealthyRegex) after a drain was completed and OSDs came back up. Operator will continue with the next drain if the timeout exceeds. No values or 0 means that the operator will wait until the placement groups are healthy before unblocking the next drain.
    • pgHealthyRegex: The regular expression that is used to determine which PG states should be considered healthy. The default is ^(active\+clean|active\+clean\+scrubbing|active\+clean\+scrubbing\+deep)$.
  • removeOSDsIfOutAndSafeToRemove: If true the operator will remove the OSDs that are down and whose data has been restored to other OSDs. In Ceph terms, the OSDs are out and safe-to-destroy when they are removed.
  • cleanupPolicy: cleanup policy settings
  • security: security page for key management configuration
  • cephConfig: Set Ceph config options using the Ceph Mon config store
  • csi: Set CSI Driver options

Ceph container images

Official releases of Ceph Container images are available from Docker Hub.

These are general purpose Ceph container with all necessary daemons and dependencies installed.

TAG MEANING
vRELNUM Latest release in this series (e.g., v19 = Squid)
vRELNUM.Y Latest stable release in this stable series (e.g., v19.2)
vRELNUM.Y.Z A specific release (e.g., v19.2.1)
vRELNUM.Y.Z-YYYYMMDD A specific build (e.g., v19.2.1-20250202)

A specific will contain a specific release of Ceph as well as security fixes from the Operating System.

Mon Settings

  • count: Set the number of mons to be started. The number must be between 1 and 9. The recommended value is most commonly 3. For highest availability, an odd number of mons should be specified. For higher durability in case of mon loss, an even number can be specified although availability may be lower. To maintain quorum a majority of mons must be up. For example, if there are three mons, two must be up. If there are four mons, three must be up. If there are two mons, both must be up. If quorum is lost, see the disaster recovery guide to restore quorum from a single mon.
  • allowMultiplePerNode: Whether to allow the placement of multiple mons on a single node. Default is false for production. Should only be set to true in test environments.
  • volumeClaimTemplate: A PersistentVolumeSpec used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath volume mounts are used. The current set of fields from template that are used are storageClassName and the storage resource request and limit. The default storage size request for new PVCs is 10Gi. Ensure that associated storage class is configured to use volumeBindingMode: WaitForFirstConsumer. This setting only applies to new monitors that are created when the requested number of monitors increases, or when a monitor fails and is recreated. An example CRD configuration is provided below.
  • failureDomainLabel: The label that is expected on each node where the mons are expected to be deployed. The labels must be found in the list of well-known topology labels.

  • zones: The failure domain names where the Mons are expected to be deployed. There must be at least three zones specified in the list. Each zone can be backed by a different storage class by specifying the volumeClaimTemplate.

    • name: The name of the zone, which is the value of the domain label.
    • volumeClaimTemplate: A PersistentVolumeSpec used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath volume mounts are used. The current set of fields from template that are used are storageClassName and the storage resource request and limit. The default storage size request for new PVCs is 10Gi. Ensure that associated storage class is configured to use volumeBindingMode: WaitForFirstConsumer. This setting only applies to new monitors that are created when the requested number of monitors increases, or when a monitor fails and is recreated. An example CRD configuration is provided below.

  • stretchCluster: The stretch cluster settings that define the zones (or other failure domain labels) across which to configure the cluster.

    • failureDomainLabel: The label that is expected on each node where the cluster is expected to be deployed. The labels must be found in the list of well-known topology labels.
    • subFailureDomain: With a zone, the data replicas must be spread across OSDs in the subFailureDomain. The default is host.
    • zones: The failure domain names where the Mons and OSDs are expected to be deployed. There must be three zones specified in the list. This element is always named zone even if a non-default failureDomainLabel is specified. The elements have two values:
      • name: The name of the zone, which is the value of the domain label.
      • arbiter: Whether the zone is expected to be the arbiter zone which only runs a single mon. Exactly one zone must be labeled true.
      • volumeClaimTemplate: A PersistentVolumeSpec used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath volume mounts are used. The current set of fields from template that are used are storageClassName and the storage resource request and limit. The default storage size request for new PVCs is 10Gi. Ensure that associated storage class is configured to use volumeBindingMode: WaitForFirstConsumer. This setting only applies to new monitors that are created when the requested number of monitors increases, or when a monitor fails and is recreated. An example CRD configuration is provided below. The two zones that are not the arbiter zone are expected to have OSDs deployed.

If these settings are changed in the CRD the operator will update the number of mons during a periodic check of the mon health, which by default is every 45 seconds.

To change the defaults that the operator uses to determine the mon health and whether to failover a mon, refer to the health settings. The intervals should be small enough that you have confidence the mons will maintain quorum, while also being long enough to ignore network blips where mons are failed over too often.

Mgr Settings

You can use the cluster CR to enable or disable any manager module. This can be configured like so:

1
+ CephCluster CRD - Rook Ceph Documentation      
 
  
   
 
 
 
 
 
 
  
 
 
 
 
 
  
 
 
 
 
 
CephCluster CRD
 
Rook allows creation and customization of storage clusters through the custom resource definitions (CRDs). There are primarily four different modes in which to create your cluster.
 
     
  1. Host Storage Cluster: Consume storage from host paths and raw devices
    2.  
  2. PVC Storage Cluster: Dynamically provision storage underneath Rook by specifying the storage class Rook should use to consume storage (via PVCs)
    3.  
  3. Stretched Storage Cluster: Distribute Ceph mons across three zones, while storage (OSDs) is only configured in two zones
    4.  
  4. External Ceph Cluster: Connect your K8s applications to an external Ceph cluster
    5.  
 
See the separate topics for a description and examples of each of these scenarios.
 
Settings
 
Settings can be specified at the global level to apply to the cluster as a whole, while other settings can be specified at more fine-grained levels. If any setting is unspecified, a suitable default will be used automatically.
 
Cluster metadata
 
     
  • name: The name that will be used internally for the Ceph cluster. Most commonly the name is the same as the namespace since multiple clusters are not supported in the same namespace.
    •  
  • namespace: The Kubernetes namespace that will be created for the Rook cluster. The services, pods, and other resources created by the operator will be added to this namespace. The common scenario is to create a single Rook cluster. If multiple clusters are created, they must not have conflicting devices or host paths.
    •  
 
Cluster Settings
 
     
  • external:
       
    • enable: if true, the cluster will not be managed by Rook but via an external entity. This mode is intended to connect to an existing cluster. In this case, Rook will only consume the external cluster. However, Rook will be able to deploy various daemons in Kubernetes such as object gateways, mds and nfs if an image is provided and will refuse otherwise. If this setting is enabled all the other options will be ignored except cephVersion.image and dataDirHostPath. See external cluster configuration. If cephVersion.image is left blank, Rook will refuse the creation of extra CRs like object, file and nfs.
      •  
     
    •  
  • cephVersion: The version information for launching the ceph daemons.
       
    • image: The image used for running the ceph daemons. For example, quay.io/ceph/ceph:v19.2.1. For more details read the container images section. For the latest ceph images, see the Ceph DockerHub. To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version. Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag v19 will be updated each time a new Squid build is released. Using the general v19 tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster.
      •  
    • allowUnsupported: If true, allow an unsupported major version of the Ceph release. Currently Reef and Squid are supported. Future versions such as Tentacle (v20) would require this to be set to true. Should be set to false in production.
      •  
    • imagePullPolicy: The image pull policy for the ceph daemon pods. Possible values are Always, IfNotPresent, and Never. The default is IfNotPresent.
      •  
     
    •  
  • dataDirHostPath: The path on the host (hostPath) where config and data should be stored for each of the services. If there are multiple clusters, the directory must be unique for each cluster. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Following paths and any of their subpaths must not be used: /etc/ceph, /rook or /var/log/ceph.
       
    • WARNING: For test scenarios, if you delete a cluster and start a new cluster on the same hosts, the path used by dataDirHostPath must be deleted. Otherwise, stale keys and other config will remain from the previous cluster and the new mons will fail to start. If this value is empty, each pod will get an ephemeral directory to store their config files that is tied to the lifetime of the pod running on that node. More details can be found in the Kubernetes empty dir docs.
      •  
     
    •  
  • skipUpgradeChecks: if set to true Rook won't perform any upgrade checks on Ceph daemons during an upgrade. Use this at YOUR OWN RISK, only if you know what you're doing. To understand Rook's upgrade process of Ceph, read the upgrade doc.
    •  
  • continueUpgradeAfterChecksEvenIfNotHealthy: if set to true Rook will continue the OSD daemon upgrade process even if the PGs are not clean, or continue with the MDS upgrade even the file system is not healthy.
    •  
  • upgradeOSDRequiresHealthyPGs: if set to true OSD upgrade process won't start until PGs are healthy.
    •  
  • dashboard: Settings for the Ceph dashboard. To view the dashboard in your browser see the dashboard guide.
       
    • enabled: Whether to enable the dashboard to view cluster status
      •  
    • urlPrefix: Allows to serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy)
      •  
    • port: Allows to change the default port where the dashboard is served
      •  
    • ssl: Whether to serve the dashboard via SSL, ignored on Ceph versions older than 13.2.2
      •  
     
    •  
  • monitoring: Settings for monitoring Ceph using Prometheus. To enable monitoring on your cluster see the monitoring guide.
       
    • enabled: Whether to enable the prometheus service monitor for an internal cluster. For an external cluster, whether to create an endpoint port for the metrics. Default is false.
      •  
    • metricsDisabled: Whether to disable the metrics reported by Ceph. If false, the prometheus mgr module and Ceph exporter are enabled. If true, the prometheus mgr module and Ceph exporter are both disabled. Default is false.
      •  
    • externalMgrEndpoints: external cluster manager endpoints
      •  
    • externalMgrPrometheusPort: external prometheus manager module port. See external cluster configuration for more details.
      •  
    • port: The internal prometheus manager module port where the prometheus mgr module listens. The port may need to be configured when host networking is enabled.
      •  
    • interval: The interval for the prometheus module to to scrape targets.
      •  
    • exporter: Ceph exporter metrics config.
         
      • perfCountersPrioLimit: Specifies which performance counters are exported. Corresponds to --prio-limit Ceph exporter flag. 0 - all counters are exported, default is 5.
        •  
      • statsPeriodSeconds: Time to wait before sending requests again to exporter server (seconds). Corresponds to --stats-period Ceph exporter flag. Default is 5.
        •  
       
      •  
     
    •  
  • network: For the network settings for the cluster, refer to the network configuration settings
    •  
  • mon: contains mon related options mon settings For more details on the mons and when to choose a number other than 3, see the mon health doc.
    •  
  • mgr: manager top level section
       
    • count: set number of ceph managers between 1 to 2. The default value is 2. If there are two managers, it is important for all mgr services point to the active mgr and not the standby mgr. Rook automatically updates the label mgr_role on the mgr pods to be either active or standby. Therefore, services need just to add the label mgr_role=active to their selector to point to the active mgr. This applies to all services that rely on the ceph mgr such as the dashboard or the prometheus metrics collector.
      •  
    • modules: A list of Ceph manager modules to enable or disable. Note the "dashboard" and "monitoring" modules are already configured by other settings.
      •  
     
    •  
  • crashCollector: The settings for crash collector daemon(s).
       
    • disable: is set to true, the crash collector will not run on any node where a Ceph daemon runs
      •  
    • daysToRetain: specifies the number of days to keep crash entries in the Ceph cluster. By default the entries are kept indefinitely.
      •  
     
    •  
  • logCollector: The settings for log collector daemon.
       
    • enabled: if set to true, the log collector will run as a side-car next to each Ceph daemon. The Ceph configuration option log_to_file will be turned on, meaning Ceph daemons will log on files in addition to still logging to container's stdout. These logs will be rotated. In case a daemon terminates with a segfault, the coredump files will be commonly be generated in /var/lib/systemd/coredump directory on the host, depending on the underlying OS location. (default: true)
      •  
    • periodicity: how often to rotate daemon's log. (default: 24h). Specified with a time suffix which may be h for hours or d for days. Rotating too often will slightly impact the daemon's performance since the signal briefly interrupts the program.
      •  
     
    •  
  • annotations: annotations configuration settings
    •  
  • labels: labels configuration settings
    •  
  • placement: placement configuration settings
    •  
  • resources: resources configuration settings
    •  
  • priorityClassNames: priority class names configuration settings
    •  
  • storage: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes.
       
    • useAllNodes: true or false, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values. If individual nodes are specified under the nodes field, then useAllNodes must be set to false.
      •  
    • nodes: Names of individual nodes in the cluster that should have their storage included in accordance with either the cluster level configuration specified above or any node specific overrides described in the next section below. useAllNodes must be set to false to use specific nodes and their config. See node settings below.
      •  
    • config: Config settings applied to all OSDs on the node unless overridden by devices. See the config settings below.
      •  
    • allowDeviceClassUpdate: Whether to allow changing the device class of an OSD after it is created. The default is false to prevent unintentional data movement or CRUSH changes if the device class is changed accidentally.
      •  
    • allowOsdCrushWeightUpdate: Whether Rook will resize the OSD CRUSH weight when the OSD PVC size is increased. This allows cluster data to be rebalanced to make most effective use of new OSD space. The default is false since data rebalancing can cause temporary cluster slowdown.
      •  
    • storage selection settings
      •  
    • Storage Class Device Sets
      •  
    • onlyApplyOSDPlacement: Whether the placement specific for OSDs is merged with the all placement. If false, the OSD placement will be merged with the all placement. If true, the OSD placement will be applied and the all placement will be ignored. The placement for OSDs is computed from several different places depending on the type of OSD:
         
      • For non-PVCs: placement.all and placement.osd
        •  
      • For PVCs: placement.all and inside the storageClassDeviceSets from the placement or preparePlacement
        •  
       
      •  
    • flappingRestartIntervalHours: Defines the time for which an OSD pod will sleep before restarting, if it stopped due to flapping. Flapping occurs where OSDs are marked down by Ceph more than 5 times in 600 seconds. The OSDs will stay down when flapping since they likely have a bad disk or other issue that needs investigation. If the issue with the OSD is fixed manually, the OSD pod can be manually restarted. The sleep is disabled if this interval is set to 0.
      •  
    • scheduleAlways: Whether to always schedule OSD pods on nodes declared explicitly in the "nodes" section, even if they are temporarily not schedulable. If set to true, consider adding placement tolerations for unschedulable nodes.
      •  
    • fullRatio: The ratio at which Ceph should block IO if the OSDs are too full. The default is 0.95.
      •  
    • backfillFullRatio: The ratio at which Ceph should stop backfilling data if the OSDs are too full. The default is 0.90.
      •  
    • nearFullRatio: The ratio at which Ceph should raise a health warning if the cluster is almost full. The default is 0.85.
      •  
     
    •  
  • disruptionManagement: The section for configuring management of daemon disruptions
       
    • managePodBudgets: if true, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically via the strategy outlined in the design. The operator will block eviction of OSDs by default and unblock them safely when drains are detected.
      •  
    • osdMaintenanceTimeout: is a duration in minutes that determines how long an entire failureDomain like region/zone/host will be held in noout (in addition to the default DOWN/OUT interval) when it is draining. The default value is 30 minutes.
      •  
    • pgHealthCheckTimeout: A duration in minutes that the operator will wait for the placement groups to become healthy (see pgHealthyRegex) after a drain was completed and OSDs came back up. Operator will continue with the next drain if the timeout exceeds. No values or 0 means that the operator will wait until the placement groups are healthy before unblocking the next drain.
      •  
    • pgHealthyRegex: The regular expression that is used to determine which PG states should be considered healthy. The default is ^(active\+clean|active\+clean\+scrubbing|active\+clean\+scrubbing\+deep)$.
      •  
     
    •  
  • removeOSDsIfOutAndSafeToRemove: If true the operator will remove the OSDs that are down and whose data has been restored to other OSDs. In Ceph terms, the OSDs are out and safe-to-destroy when they are removed.
    •  
  • cleanupPolicy: cleanup policy settings
    •  
  • security: security page for key management configuration
    •  
  • cephConfig: Set Ceph config options using the Ceph Mon config store
    •  
  • csi: Set CSI Driver options
    •  
 
Ceph container images
 
Official releases of Ceph Container images are available from Docker Hub.
 
These are general purpose Ceph container with all necessary daemons and dependencies installed.
 
  
 TAG MEANING 
   
 vRELNUM Latest release in this series (e.g., v19 = Squid) 
 
 vRELNUM.Y Latest stable release in this stable series (e.g., v19.2) 
 
 vRELNUM.Y.Z A specific release (e.g., v19.2.1) 
 
 vRELNUM.Y.Z-YYYYMMDD A specific build (e.g., v19.2.1-20250202) 
  
 
A specific will contain a specific release of Ceph as well as security fixes from the Operating System.
 
Mon Settings
 
     
  • count: Set the number of mons to be started. The number must be between 1 and 9. The recommended value is most commonly 3. For highest availability, an odd number of mons should be specified. For higher durability in case of mon loss, an even number can be specified although availability may be lower. To maintain quorum a majority of mons must be up. For example, if there are three mons, two must be up. If there are four mons, three must be up. If there are two mons, both must be up. If quorum is lost, see the disaster recovery guide to restore quorum from a single mon.
    •  
  • allowMultiplePerNode: Whether to allow the placement of multiple mons on a single node. Default is false for production. Should only be set to true in test environments.
    •  
  • volumeClaimTemplate: A PersistentVolumeSpec used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath volume mounts are used. The current set of fields from template that are used are storageClassName and the storage resource request and limit. The default storage size request for new PVCs is 10Gi. Ensure that associated storage class is configured to use volumeBindingMode: WaitForFirstConsumer. This setting only applies to new monitors that are created when the requested number of monitors increases, or when a monitor fails and is recreated. An example CRD configuration is provided below.
    •  
  • failureDomainLabel: The label that is expected on each node where the mons are expected to be deployed. The labels must be found in the list of well-known topology labels.
    •  
  •  
    zones: The failure domain names where the Mons are expected to be deployed. There must be at least three zones specified in the list. Each zone can be backed by a different storage class by specifying the volumeClaimTemplate.
     
       
    • name: The name of the zone, which is the value of the domain label.
      •  
    • volumeClaimTemplate: A PersistentVolumeSpec used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath volume mounts are used. The current set of fields from template that are used are storageClassName and the storage resource request and limit. The default storage size request for new PVCs is 10Gi. Ensure that associated storage class is configured to use volumeBindingMode: WaitForFirstConsumer. This setting only applies to new monitors that are created when the requested number of monitors increases, or when a monitor fails and is recreated. An example CRD configuration is provided below.
      •  
     
    •  
  •  
    stretchCluster: The stretch cluster settings that define the zones (or other failure domain labels) across which to configure the cluster.
     
       
    • failureDomainLabel: The label that is expected on each node where the cluster is expected to be deployed. The labels must be found in the list of well-known topology labels.
      •  
    • subFailureDomain: With a zone, the data replicas must be spread across OSDs in the subFailureDomain. The default is host.
      •  
    • zones: The failure domain names where the Mons and OSDs are expected to be deployed. There must be three zones specified in the list. This element is always named zone even if a non-default failureDomainLabel is specified. The elements have two values:
         
      • name: The name of the zone, which is the value of the domain label.
        •  
      • arbiter: Whether the zone is expected to be the arbiter zone which only runs a single mon. Exactly one zone must be labeled true.
        •  
      • volumeClaimTemplate: A PersistentVolumeSpec used by Rook to create PVCs for monitor storage. This field is optional, and when not provided, HostPath volume mounts are used. The current set of fields from template that are used are storageClassName and the storage resource request and limit. The default storage size request for new PVCs is 10Gi. Ensure that associated storage class is configured to use volumeBindingMode: WaitForFirstConsumer. This setting only applies to new monitors that are created when the requested number of monitors increases, or when a monitor fails and is recreated. An example CRD configuration is provided below. The two zones that are not the arbiter zone are expected to have OSDs deployed.
        •  
       
      •  
     
    •  
 
If these settings are changed in the CRD the operator will update the number of mons during a periodic check of the mon health, which by default is every 45 seconds.
 
To change the defaults that the operator uses to determine the mon health and whether to failover a mon, refer to the health settings. The intervals should be small enough that you have confidence the mons will maintain quorum, while also being long enough to ignore network blips where mons are failed over too often.
 
Mgr Settings
 
You can use the cluster CR to enable or disable any manager module. This can be configured like so:
 
1
 2
 3
 4
mgr:
   modules:
   - name: <name of the module>
     enabled: true
-
 
Some modules will have special configuration to ensure the module is fully functional after being enabled. Specifically:
 
     
  • pg_autoscaler: Rook will configure all new pools with PG autoscaling by setting: osd_pool_default_pg_autoscale_mode = on
    •  
 
Network Configuration Settings
 
If not specified, the default SDN will be used. Configure the network that will be enabled for the cluster and services.
 
     
  • provider: Specifies the network provider that will be used to connect the network interface. You can choose between host, and multus.
    •  
  • selectors: Used for multus provider only. Select NetworkAttachmentDefinitions to use for Ceph networks.
       
    • public: Select the NetworkAttachmentDefinition to use for the public network.
      •  
    • cluster: Select the NetworkAttachmentDefinition to use for the cluster network.
      •  
     
    •  
  • addressRanges: Used for host or multus providers only. Allows overriding the address ranges (CIDRs) that Ceph will listen on.
       
    • public: A list of individual network ranges in CIDR format to use for Ceph's public network.
      •  
    • cluster: A list of individual network ranges in CIDR format to use for Ceph's cluster network.
      •  
     
    •  
  • ipFamily: Specifies the network stack Ceph daemons should listen on.
    •  
  • dualStack: Specifies that Ceph daemon should listen on both IPv4 and IPv6 network stacks.
    •  
  • connections: Settings for network connections using Ceph's msgr2 protocol
       
    • requireMsgr2: Whether to require communication over msgr2. If true, the msgr v1 port (6789) will be disabled and clients will be required to connect to the Ceph cluster with the v2 port (3300). Requires a kernel that supports msgr2 (kernel 5.11 or CentOS 8.4 or newer). Default is false.
      •  
    • encryption: Settings for encryption on the wire to Ceph daemons
         
      • enabled: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted. When encryption is not enabled, clients still establish a strong initial authentication and data integrity is still validated with a crc check. IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only, set "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class. The nbd and fuse drivers are not recommended in production since restarting the csi driver pod will disconnect the volumes. If this setting is enabled, CephFS volumes also require setting CSI_CEPHFS_KERNEL_MOUNT_OPTIONS to "ms_mode=secure" in operator.yaml.
        •  
       
      •  
    • compression:
         
      • enabled: Whether to compress the data in transit across the wire. The default is false. See the kernel requirements above for encryption.
        •  
       
      •  
     
    •  
 
 
Caution
 
Changing networking configuration after a Ceph cluster has been deployed is only supported for the network encryption settings. Changing other network settings is NOT supported and will likely result in a non-functioning cluster.
 
 
Provider
 
Selecting a non-default network provider is an advanced topic. Read more in the Network Providers documentation.
 
IPFamily
 
Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. To enable dual stack see the network configuration section.
 
Node Settings
 
In addition to the cluster level settings specified above, each individual node can also specify configuration to override the cluster level settings and defaults. If a node does not specify any configuration then it will inherit the cluster level settings.
 
     
  • name: The name of the node, which should match its kubernetes.io/hostname label.
    •  
  • config: Config settings applied to all OSDs on the node unless overridden by devices. See the config settings below.
    •  
  • storage selection settings
    •  
 
When useAllNodes is set to true, Rook attempts to make Ceph cluster management as hands-off as possible while still maintaining reasonable data safety. If a usable node comes online, Rook will begin to use it automatically. To maintain a balance between hands-off usability and data safety, Nodes are removed from Ceph as OSD hosts only (1) if the node is deleted from Kubernetes itself or (2) if the node has its taints or affinities modified in such a way that the node is no longer usable by Rook. Any changes to taints or affinities, intentional or unintentional, may affect the data reliability of the Ceph cluster. In order to help protect against this somewhat, deletion of nodes by taint or affinity modifications must be "confirmed" by deleting the Rook Ceph operator pod and allowing the operator deployment to restart the pod.
 
For production clusters, we recommend that useAllNodes is set to false to prevent the Ceph cluster from suffering reduced data reliability unintentionally due to a user mistake. When useAllNodes is set to false, Rook relies on the user to be explicit about when nodes are added to or removed from the Ceph cluster. Nodes are only added to the Ceph cluster if the node is added to the Ceph cluster resource. Similarly, nodes are only removed if the node is removed from the Ceph cluster resource.
 
Node Updates
 
Nodes can be added and removed over time by updating the Cluster CRD, for example with kubectl -n rook-ceph edit cephcluster rook-ceph. This will bring up your default text editor and allow you to add and remove storage nodes from the cluster. This feature is only available when useAllNodes has been set to false.
 
Storage Selection Settings
 
Below are the settings for host-based cluster. This type of cluster can specify devices for OSDs, both at the cluster and individual node level, for selecting which storage resources will be included in the cluster.
 
     
  • useAllDevices: true or false, indicating whether all devices found on nodes in the cluster should be automatically consumed by OSDs. Not recommended unless you have a very controlled environment where you will not risk formatting of devices with existing data. When true, all devices and partitions will be used. Is overridden by deviceFilter if specified. LVM logical volumes are not picked by useAllDevices.
    •  
  • deviceFilter: A regular expression for short kernel names of devices (e.g. sda) that allows selection of devices and partitions to be consumed by OSDs. LVM logical volumes are not picked by deviceFilter.If individual devices have been specified for a node then this filter will be ignored. This field uses golang regular expression syntax. For example:
       
    • sdb: Only selects the sdb device if found
      •  
    • ^sd.: Selects all devices starting with sd
      •  
    • ^sd[a-d]: Selects devices starting with sda, sdb, sdc, and sdd if found
      •  
    • ^s: Selects all devices that start with s
      •  
    • ^[^r]: Selects all devices that do not start with r
      •  
     
    •  
  • devicePathFilter: A regular expression for device paths (e.g. /dev/disk/by-path/pci-0:1:2:3-scsi-1) that allows selection of devices and partitions to be consumed by OSDs. LVM logical volumes are not picked by devicePathFilter.If individual devices or deviceFilter have been specified for a node then this filter will be ignored. This field uses golang regular expression syntax. For example:
       
    • ^/dev/sd.: Selects all devices starting with sd
      •  
    • ^/dev/disk/by-path/pci-.*: Selects all devices which are connected to PCI bus
      •  
     
    •  
  • devices: A list of individual device names belonging to this node to include in the storage cluster.
       
    • name: The name of the devices and partitions (e.g., sda). The full udev path can also be specified for devices, partitions, and logical volumes (e.g. /dev/disk/by-id/ata-ST4000DM004-XXXX - this will not change after reboots).
      •  
    • config: Device-specific config settings. See the config settings below
      •  
     
    •  
 
Host-based cluster supports raw devices, partitions, logical volumes, encrypted devices, and multipath devices. Be sure to see the quickstart doc prerequisites for additional considerations.
 
Below are the settings for a PVC-based cluster.
  
Storage Class Device Sets
 
The following are the settings for Storage Class Device Sets which can be configured to create OSDs that are backed by block mode PVs.
 
     
  • name: A name for the set.
    •  
  • count: The number of devices in the set.
    •  
  • resources: The CPU and RAM requests/limits for the devices. (Optional)
    •  
  •  
    placement: The placement criteria for the devices. (Optional) Default is no placement criteria.
     
    The syntax is the same as for other placement configuration. It supports nodeAffinity, podAffinity, podAntiAffinity and tolerations keys.
     
    It is recommended to configure the placement such that the OSDs will be as evenly spread across nodes as possible. At a minimum, anti-affinity should be added so at least one OSD will be placed on each available nodes.
     
    However, if there are more OSDs than nodes, this anti-affinity will not be effective. Another placement scheme to consider is to add labels to the nodes in such a way that the OSDs can be grouped on those nodes, create multiple storageClassDeviceSets, and add node affinity to each of the device sets that will place the OSDs in those sets of nodes.
     
    Rook will automatically add required nodeAffinity to the OSD daemons to match the topology labels that are found on the nodes where the OSD prepare jobs ran. To ensure data durability, the OSDs are required to run in the same topology that the Ceph CRUSH map expects. For example, if the nodes are labeled with rack topology labels, the OSDs will be constrained to a certain rack. Without the topology labels, Rook will not constrain the OSDs beyond what is required by the PVs, for example to run in the zone where provisioned. See the OSD Topology section for the related labels.
     
    •  
  •  
    preparePlacement: The placement criteria for the preparation of the OSD devices. Creating OSDs is a two-step process and the prepare job may require different placement than the OSD daemons. If the preparePlacement is not specified, the placement will instead be applied for consistent placement for the OSD prepare jobs and OSD deployments. The preparePlacement is only useful for portable OSDs in the device sets. OSDs that are not portable will be tied to the host where the OSD prepare job initially runs.
     
       
    • For example, provisioning may require topology spread constraints across zones, but the OSD daemons may require constraints across hosts within the zones.
      •  
     
    •  
  • portable: If true, the OSDs will be allowed to move between nodes during failover. This requires a storage class that supports portability (e.g. aws-ebs, but not the local storage provisioner). If false, the OSDs will be assigned to a node permanently. Rook will configure Ceph's CRUSH map to support the portability.
    •  
  • tuneDeviceClass: For example, Ceph cannot detect AWS volumes as HDDs from the storage class "gp2-csi", so you can improve Ceph performance by setting this to true.
    •  
  • tuneFastDeviceClass: For example, Ceph cannot detect Azure disks as SSDs from the storage class "managed-premium", so you can improve Ceph performance by setting this to true..
    •  
  • volumeClaimTemplates: A list of PVC templates to use for provisioning the underlying storage devices.
       
    • metadata.name: "data", "metadata", or "wal". If a single template is provided, the name must be "data". If the name is "metadata" or "wal", the devices are used to store the Ceph metadata or WAL respectively. In both cases, the devices must be raw devices or LVM logical volumes.
         
      • resources.requests.storage: The desired capacity for the underlying storage devices.
        •  
      • storageClassName: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass.
        •  
      • volumeMode: The volume mode to be set for the PVC. Which should be Block
        •  
      • accessModes: The access mode for the PVC to be bound by OSD.
        •  
       
      •  
     
    •  
  • schedulerName: Scheduler name for OSD pod placement. (Optional)
    •  
  • encrypted: whether to encrypt all the OSDs in a given storageClassDeviceSet
    •  
 
See the table in OSD Configuration Settings to know the allowed configurations.
 
OSD Configuration Settings
 
The following storage selection settings are specific to Ceph and do not apply to other backends. All variables are key-value pairs represented as strings.
 
     
  • metadataDevice: Name of a device, partition or lvm to use for the metadata of OSDs on each node. Performance can be improved by using a low latency device (such as SSD or NVMe) as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Provisioning will fail if the user specifies a metadataDevice but that device is not used as a metadata device by Ceph. Notably, ceph-volume will not use a device of the same device class (HDD, SSD, NVMe) as OSD devices for metadata, resulting in this failure.
    •  
  • databaseSizeMB: The size in MB of a bluestore database. Include quotes around the size.
    •  
  • walSizeMB: The size in MB of a bluestore write ahead log (WAL). Include quotes around the size.
    •  
  • deviceClass: The CRUSH device class to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either hdd, ssd, or nvme based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of the pool spec's deviceClass field. If updating the device class of an OSD after the OSD is already created, allowDeviceClassUpdate: true must be set. Otherwise updates to this deviceClass will be ignored.
    •  
  • initialWeight: The initial OSD weight in TiB units. By default, this value is derived from OSD's capacity.
    •  
  • primaryAffinity: The primary-affinity value of an OSD, within range [0, 1] (default: 1).
    •  
  • osdsPerDevice**: The number of OSDs to create on each device. High performance devices such as NVMe can handle running multiple OSDs. If desired, this can be overridden for each node and each device.
    •  
  • encryptedDevice**: Encrypt OSD volumes using dmcrypt ("true" or "false"). By default this option is disabled. See encryption for more information on encryption in Ceph. (Resizing is not supported for host-based clusters.)
    •  
  • crushRoot: The value of the root CRUSH map label. The default is default. Generally, you should not need to change this. However, if any of your topology labels may have the value default, you need to change crushRoot to avoid conflicts, since CRUSH map values need to be unique.
    •  
  • enableCrushUpdates: Enables rook to update the pool crush rule using Pool Spec. Can cause data remapping if crush rule changes, Defaults to false.
    •  
  • migration: Existing PVC based OSDs can be migrated to enable or disable encryption. Refer to the osd management topic for details.
    •  
 
Allowed configurations are:
 
  
 block device type host-based cluster PVC-based cluster 
   
 disk   
 
 part encryptedDevice must be false encrypted must be false 
 
 lvm metadataDevice must be "", osdsPerDevice must be 1, and encryptedDevice must be false metadata.name must not be metadata or wal and encrypted must be false 
 
 crypt   
 
 mpath   
  
 
Limitations of metadata device
 
     
  • If metadataDevice is specified in the global OSD configuration or in the node level OSD configuration, the metadata device will be shared between all OSDs on the same node. In other words, OSDs will be initialized by lvm batch. In this case, we can't use partition device.
    •  
  • If metadataDevice is specified in the device local configuration, we can use partition as metadata device. In other words, OSDs are initialized by lvm prepare.
    •  
 
Annotations and Labels
 
Annotations and Labels can be specified so that the Rook components will have those annotations / labels added to them.
 
You can set annotations / labels for Rook components for the list of key value pairs:
 
     
  • all: Set annotations / labels for all components except clusterMetadata.
    •  
  • mgr: Set annotations / labels for MGRs
    •  
  • mon: Set annotations / labels for mons
    •  
  • osd: Set annotations / labels for OSDs
    •  
  • dashboard: Set annotations / labels for the dashboard service
    •  
  • prepareosd: Set annotations / labels for OSD Prepare Jobs
    •  
  • monitoring: Set annotations / labels for service monitor
    •  
  • crashcollector: Set annotations / labels for crash collectors
    •  
  • clusterMetadata: Set annotations only to rook-ceph-mon-endpoints configmap and the rook-ceph-mon and rook-ceph-admin-keyring secrets. These annotations will not be merged with the all annotations. The common usage is for backing up these critical resources with kubed. Note the clusterMetadata annotation will not be merged with the all annotation. When other keys are set, all will be merged together with the specific component.
    •  
 
Placement Configuration Settings
 
Placement configuration for the cluster services. It includes the following keys: mgr, mon, arbiter, osd, prepareosd, cleanup, and all. Each service will have its placement configuration generated by merging the generic configuration under all with the most specific one (which will override any attributes).
 
In stretch clusters, if the arbiter placement is specified, that placement will only be applied to the arbiter. Neither will the arbiter placement be merged with the all placement to allow the arbiter to be fully independent of other daemon placement. The remaining mons will still use the mon and/or all sections.
 
 
Note
 
Placement of OSD pods is controlled using the Storage Class Device Set, not the general placement configuration.
 
 
A Placement configuration is specified (according to the kubernetes PodSpec) as:
  
If you use labelSelector for osd pods, you must write two rules both for rook-ceph-osd and rook-ceph-osd-prepare like the example configuration. It comes from the design that there are these two pods for an OSD. For more detail, see the osd design doc and the related issue.
 
The Rook Ceph operator creates a Job called rook-ceph-detect-version to detect the full Ceph version used by the given cephVersion.image. The placement from the mon section is used for the Job except for the PodAntiAffinity field.
 
Placement Example
 
To control where various services will be scheduled by kubernetes, use the placement configuration sections below. The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage-node. Specific node affinity and tolerations that only apply to themondaemons in this example require the labelrole=storage-mon-node` and also tolerate the control plane taint.
 
 1
+
 
Some modules will have special configuration to ensure the module is fully functional after being enabled. Specifically:
 
     
  • pg_autoscaler: Rook will configure all new pools with PG autoscaling by setting: osd_pool_default_pg_autoscale_mode = on
    •  
 
Network Configuration Settings
 
If not specified, the default SDN will be used. Configure the network that will be enabled for the cluster and services.
 
     
  • provider: Specifies the network provider that will be used to connect the network interface. You can choose between host, and multus.
    •  
  • selectors: Used for multus provider only. Select NetworkAttachmentDefinitions to use for Ceph networks.
       
    • public: Select the NetworkAttachmentDefinition to use for the public network.
      •  
    • cluster: Select the NetworkAttachmentDefinition to use for the cluster network.
      •  
     
    •  
  • addressRanges: Used for host or multus providers only. Allows overriding the address ranges (CIDRs) that Ceph will listen on.
       
    • public: A list of individual network ranges in CIDR format to use for Ceph's public network.
      •  
    • cluster: A list of individual network ranges in CIDR format to use for Ceph's cluster network.
      •  
     
    •  
  • ipFamily: Specifies the network stack Ceph daemons should listen on.
    •  
  • dualStack: Specifies that Ceph daemon should listen on both IPv4 and IPv6 network stacks.
    •  
  • connections: Settings for network connections using Ceph's msgr2 protocol
       
    • requireMsgr2: Whether to require communication over msgr2. If true, the msgr v1 port (6789) will be disabled and clients will be required to connect to the Ceph cluster with the v2 port (3300). Requires a kernel that supports msgr2 (kernel 5.11 or CentOS 8.4 or newer). Default is false.
      •  
    • encryption: Settings for encryption on the wire to Ceph daemons
         
      • enabled: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted. When encryption is not enabled, clients still establish a strong initial authentication and data integrity is still validated with a crc check. IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only, set "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class. The nbd and fuse drivers are not recommended in production since restarting the csi driver pod will disconnect the volumes. If this setting is enabled, CephFS volumes also require setting CSI_CEPHFS_KERNEL_MOUNT_OPTIONS to "ms_mode=secure" in operator.yaml.
        •  
       
      •  
    • compression:
         
      • enabled: Whether to compress the data in transit across the wire. The default is false. See the kernel requirements above for encryption.
        •  
       
      •  
     
    •  
 
 
Caution
 
Changing networking configuration after a Ceph cluster has been deployed is only supported for the network encryption settings. Changing other network settings is NOT supported and will likely result in a non-functioning cluster.
 
 
Provider
 
Selecting a non-default network provider is an advanced topic. Read more in the Network Providers documentation.
 
IPFamily
 
Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. To enable dual stack see the network configuration section.
 
Node Settings
 
In addition to the cluster level settings specified above, each individual node can also specify configuration to override the cluster level settings and defaults. If a node does not specify any configuration then it will inherit the cluster level settings.
 
     
  • name: The name of the node, which should match its kubernetes.io/hostname label.
    •  
  • config: Config settings applied to all OSDs on the node unless overridden by devices. See the config settings below.
    •  
  • storage selection settings
    •  
 
When useAllNodes is set to true, Rook attempts to make Ceph cluster management as hands-off as possible while still maintaining reasonable data safety. If a usable node comes online, Rook will begin to use it automatically. To maintain a balance between hands-off usability and data safety, Nodes are removed from Ceph as OSD hosts only (1) if the node is deleted from Kubernetes itself or (2) if the node has its taints or affinities modified in such a way that the node is no longer usable by Rook. Any changes to taints or affinities, intentional or unintentional, may affect the data reliability of the Ceph cluster. In order to help protect against this somewhat, deletion of nodes by taint or affinity modifications must be "confirmed" by deleting the Rook Ceph operator pod and allowing the operator deployment to restart the pod.
 
For production clusters, we recommend that useAllNodes is set to false to prevent the Ceph cluster from suffering reduced data reliability unintentionally due to a user mistake. When useAllNodes is set to false, Rook relies on the user to be explicit about when nodes are added to or removed from the Ceph cluster. Nodes are only added to the Ceph cluster if the node is added to the Ceph cluster resource. Similarly, nodes are only removed if the node is removed from the Ceph cluster resource.
 
Node Updates
 
Nodes can be added and removed over time by updating the Cluster CRD, for example with kubectl -n rook-ceph edit cephcluster rook-ceph. This will bring up your default text editor and allow you to add and remove storage nodes from the cluster. This feature is only available when useAllNodes has been set to false.
 
Storage Selection Settings
 
Below are the settings for host-based cluster. This type of cluster can specify devices for OSDs, both at the cluster and individual node level, for selecting which storage resources will be included in the cluster.
 
     
  • useAllDevices: true or false, indicating whether all devices found on nodes in the cluster should be automatically consumed by OSDs. Not recommended unless you have a very controlled environment where you will not risk formatting of devices with existing data. When true, all devices and partitions will be used. Is overridden by deviceFilter if specified. LVM logical volumes are not picked by useAllDevices.
    •  
  • deviceFilter: A regular expression for short kernel names of devices (e.g. sda) that allows selection of devices and partitions to be consumed by OSDs. LVM logical volumes are not picked by deviceFilter.If individual devices have been specified for a node then this filter will be ignored. This field uses golang regular expression syntax. For example:
       
    • sdb: Only selects the sdb device if found
      •  
    • ^sd.: Selects all devices starting with sd
      •  
    • ^sd[a-d]: Selects devices starting with sda, sdb, sdc, and sdd if found
      •  
    • ^s: Selects all devices that start with s
      •  
    • ^[^r]: Selects all devices that do not start with r
      •  
     
    •  
  • devicePathFilter: A regular expression for device paths (e.g. /dev/disk/by-path/pci-0:1:2:3-scsi-1) that allows selection of devices and partitions to be consumed by OSDs. LVM logical volumes are not picked by devicePathFilter.If individual devices or deviceFilter have been specified for a node then this filter will be ignored. This field uses golang regular expression syntax. For example:
       
    • ^/dev/sd.: Selects all devices starting with sd
      •  
    • ^/dev/disk/by-path/pci-.*: Selects all devices which are connected to PCI bus
      •  
     
    •  
  • devices: A list of individual device names belonging to this node to include in the storage cluster.
       
    • name: The name of the devices and partitions (e.g., sda). The full udev path can also be specified for devices, partitions, and logical volumes (e.g. /dev/disk/by-id/ata-ST4000DM004-XXXX - this will not change after reboots).
      •  
    • config: Device-specific config settings. See the config settings below
      •  
     
    •  
 
Host-based cluster supports raw devices, partitions, logical volumes, encrypted devices, and multipath devices. Be sure to see the quickstart doc prerequisites for additional considerations.
 
Below are the settings for a PVC-based cluster.
  
Storage Class Device Sets
 
The following are the settings for Storage Class Device Sets which can be configured to create OSDs that are backed by block mode PVs.
 
     
  • name: A name for the set.
    •  
  • count: The number of devices in the set.
    •  
  • resources: The CPU and RAM requests/limits for the devices. (Optional)
    •  
  •  
    placement: The placement criteria for the devices. (Optional) Default is no placement criteria.
     
    The syntax is the same as for other placement configuration. It supports nodeAffinity, podAffinity, podAntiAffinity and tolerations keys.
     
    It is recommended to configure the placement such that the OSDs will be as evenly spread across nodes as possible. At a minimum, anti-affinity should be added so at least one OSD will be placed on each available nodes.
     
    However, if there are more OSDs than nodes, this anti-affinity will not be effective. Another placement scheme to consider is to add labels to the nodes in such a way that the OSDs can be grouped on those nodes, create multiple storageClassDeviceSets, and add node affinity to each of the device sets that will place the OSDs in those sets of nodes.
     
    Rook will automatically add required nodeAffinity to the OSD daemons to match the topology labels that are found on the nodes where the OSD prepare jobs ran. To ensure data durability, the OSDs are required to run in the same topology that the Ceph CRUSH map expects. For example, if the nodes are labeled with rack topology labels, the OSDs will be constrained to a certain rack. Without the topology labels, Rook will not constrain the OSDs beyond what is required by the PVs, for example to run in the zone where provisioned. See the OSD Topology section for the related labels.
     
    •  
  •  
    preparePlacement: The placement criteria for the preparation of the OSD devices. Creating OSDs is a two-step process and the prepare job may require different placement than the OSD daemons. If the preparePlacement is not specified, the placement will instead be applied for consistent placement for the OSD prepare jobs and OSD deployments. The preparePlacement is only useful for portable OSDs in the device sets. OSDs that are not portable will be tied to the host where the OSD prepare job initially runs.
     
       
    • For example, provisioning may require topology spread constraints across zones, but the OSD daemons may require constraints across hosts within the zones.
      •  
     
    •  
  • portable: If true, the OSDs will be allowed to move between nodes during failover. This requires a storage class that supports portability (e.g. aws-ebs, but not the local storage provisioner). If false, the OSDs will be assigned to a node permanently. Rook will configure Ceph's CRUSH map to support the portability.
    •  
  • tuneDeviceClass: For example, Ceph cannot detect AWS volumes as HDDs from the storage class "gp2-csi", so you can improve Ceph performance by setting this to true.
    •  
  • tuneFastDeviceClass: For example, Ceph cannot detect Azure disks as SSDs from the storage class "managed-premium", so you can improve Ceph performance by setting this to true..
    •  
  • volumeClaimTemplates: A list of PVC templates to use for provisioning the underlying storage devices.
       
    • metadata.name: "data", "metadata", or "wal". If a single template is provided, the name must be "data". If the name is "metadata" or "wal", the devices are used to store the Ceph metadata or WAL respectively. In both cases, the devices must be raw devices or LVM logical volumes.
         
      • resources.requests.storage: The desired capacity for the underlying storage devices.
        •  
      • storageClassName: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass.
        •  
      • volumeMode: The volume mode to be set for the PVC. Which should be Block
        •  
      • accessModes: The access mode for the PVC to be bound by OSD.
        •  
       
      •  
     
    •  
  • schedulerName: Scheduler name for OSD pod placement. (Optional)
    •  
  • encrypted: whether to encrypt all the OSDs in a given storageClassDeviceSet
    •  
 
See the table in OSD Configuration Settings to know the allowed configurations.
 
OSD Configuration Settings
 
The following storage selection settings are specific to Ceph and do not apply to other backends. All variables are key-value pairs represented as strings.
 
     
  • metadataDevice: Name of a device, partition or lvm to use for the metadata of OSDs on each node. Performance can be improved by using a low latency device (such as SSD or NVMe) as the metadata device, while other spinning platter (HDD) devices on a node are used to store data. Provisioning will fail if the user specifies a metadataDevice but that device is not used as a metadata device by Ceph. Notably, ceph-volume will not use a device of the same device class (HDD, SSD, NVMe) as OSD devices for metadata, resulting in this failure.
    •  
  • databaseSizeMB: The size in MB of a bluestore database. Include quotes around the size.
    •  
  • walSizeMB: The size in MB of a bluestore write ahead log (WAL). Include quotes around the size.
    •  
  • deviceClass: The CRUSH device class to use for this selection of storage devices. (By default, if a device's class has not already been set, OSDs will automatically set a device's class to either hdd, ssd, or nvme based on the hardware properties exposed by the Linux kernel.) These storage classes can then be used to select the devices backing a storage pool by specifying them as the value of the pool spec's deviceClass field. If updating the device class of an OSD after the OSD is already created, allowDeviceClassUpdate: true must be set. Otherwise updates to this deviceClass will be ignored.
    •  
  • initialWeight: The initial OSD weight in TiB units. By default, this value is derived from OSD's capacity.
    •  
  • primaryAffinity: The primary-affinity value of an OSD, within range [0, 1] (default: 1).
    •  
  • osdsPerDevice**: The number of OSDs to create on each device. High performance devices such as NVMe can handle running multiple OSDs. If desired, this can be overridden for each node and each device.
    •  
  • encryptedDevice**: Encrypt OSD volumes using dmcrypt ("true" or "false"). By default this option is disabled. See encryption for more information on encryption in Ceph. (Resizing is not supported for host-based clusters.)
    •  
  • crushRoot: The value of the root CRUSH map label. The default is default. Generally, you should not need to change this. However, if any of your topology labels may have the value default, you need to change crushRoot to avoid conflicts, since CRUSH map values need to be unique.
    •  
  • enableCrushUpdates: Enables rook to update the pool crush rule using Pool Spec. Can cause data remapping if crush rule changes, Defaults to false.
    •  
  • migration: Existing PVC based OSDs can be migrated to enable or disable encryption. Refer to the osd management topic for details.
    •  
 
Allowed configurations are:
 
  
 block device type host-based cluster PVC-based cluster 
   
 disk   
 
 part encryptedDevice must be false encrypted must be false 
 
 lvm metadataDevice must be "", osdsPerDevice must be 1, and encryptedDevice must be false metadata.name must not be metadata or wal and encrypted must be false 
 
 crypt   
 
 mpath   
  
 
Limitations of metadata device
 
     
  • If metadataDevice is specified in the global OSD configuration or in the node level OSD configuration, the metadata device will be shared between all OSDs on the same node. In other words, OSDs will be initialized by lvm batch. In this case, we can't use partition device.
    •  
  • If metadataDevice is specified in the device local configuration, we can use partition as metadata device. In other words, OSDs are initialized by lvm prepare.
    •  
 
Annotations and Labels
 
Annotations and Labels can be specified so that the Rook components will have those annotations / labels added to them.
 
You can set annotations / labels for Rook components for the list of key value pairs:
 
     
  • all: Set annotations / labels for all components except clusterMetadata.
    •  
  • mgr: Set annotations / labels for MGRs
    •  
  • mon: Set annotations / labels for mons
    •  
  • osd: Set annotations / labels for OSDs
    •  
  • dashboard: Set annotations / labels for the dashboard service
    •  
  • prepareosd: Set annotations / labels for OSD Prepare Jobs
    •  
  • monitoring: Set annotations / labels for service monitor
    •  
  • crashcollector: Set annotations / labels for crash collectors
    •  
  • clusterMetadata: Set annotations only to rook-ceph-mon-endpoints configmap and the rook-ceph-mon and rook-ceph-admin-keyring secrets. These annotations will not be merged with the all annotations. The common usage is for backing up these critical resources with kubed. Note the clusterMetadata annotation will not be merged with the all annotation. When other keys are set, all will be merged together with the specific component.
    •  
 
Placement Configuration Settings
 
Placement configuration for the cluster services. It includes the following keys: mgr, mon, arbiter, osd, prepareosd, cleanup, and all. Each service will have its placement configuration generated by merging the generic configuration under all with the most specific one (which will override any attributes).
 
In stretch clusters, if the arbiter placement is specified, that placement will only be applied to the arbiter. Neither will the arbiter placement be merged with the all placement to allow the arbiter to be fully independent of other daemon placement. The remaining mons will still use the mon and/or all sections.
 
 
Note
 
Placement of OSD pods is controlled using the Storage Class Device Set, not the general placement configuration.
 
 
A Placement configuration is specified (according to the kubernetes PodSpec) as:
  
If you use labelSelector for osd pods, you must write two rules both for rook-ceph-osd and rook-ceph-osd-prepare like the example configuration. It comes from the design that there are these two pods for an OSD. For more detail, see the osd design doc and the related issue.
 
The Rook Ceph operator creates a Job called rook-ceph-detect-version to detect the full Ceph version used by the given cephVersion.image. The placement from the mon section is used for the Job except for the PodAntiAffinity field.
 
Placement Example
 
To control where various services will be scheduled by kubernetes, use the placement configuration sections below. The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage-node. Specific node affinity and tolerations that only apply to themondaemons in this example require the labelrole=storage-mon-node` and also tolerate the control plane taint.
 
 1
  2
  3
  4
diff --git a/docs/rook/v1.16/CRDs/Cluster/external-cluster/advance-external/index.html b/docs/rook/v1.16/CRDs/Cluster/external-cluster/advance-external/index.html
index 6852579f5..4709b3a2b 100644
--- a/docs/rook/v1.16/CRDs/Cluster/external-cluster/advance-external/index.html
+++ b/docs/rook/v1.16/CRDs/Cluster/external-cluster/advance-external/index.html
@@ -2,13 +2,13 @@
 2
toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
 kubectl -n rook-ceph cp deploy/examples/external/create-external-cluster-resources.py $toolbox:/etc/ceph
 
  
  •  
    Exec to the toolbox pod and execute create-external-cluster-resources.py with needed options to create required users and keys.
     
    •   
     
    Important
     
    For other clusters to connect to storage in this cluster, Rook must be configured with a networking configuration that is accessible from other clusters. Most commonly this is done by enabling host networking in the CephCluster CR so the Ceph daemons will be addressable by their host IPs.
     
     
    Admin privileges
     
    If in case the cluster needs the admin keyring to configure, update the admin key rook-ceph-mon secret with client.admin keyring
     
     
    Note
     
    Sharing the admin key with the external cluster is not generally recommended
     
     
       
    1.  
      Get the client.admin keyring from the ceph cluster
       
      1
      ceph auth get client.admin
-
       
      2.  
    2.  
      Update two values in the rook-ceph-mon secret:
       
         
      • ceph-username: Set to client.admin
        •  
      • ceph-secret: Set the client.admin keyring
        •  
       
      3.  
     
    After restarting the rook operator (and the toolbox if in use), rook will configure ceph with admin privileges.
     
    Connect to an External Object Store
     
    Create the external object store CR to configure connection to external gateways.
     
    1
+
      
  •  
    Update two values in the rook-ceph-mon secret:
     
       
    • ceph-username: Set to client.admin
      •  
    • ceph-secret: Set the client.admin keyring
      •  
     
    •   
    After restarting the rook operator (and the toolbox if in use), rook will configure ceph with admin privileges.
     
    Connect to an External Object Store
     
    Create the external object store CR to configure connection to external gateways.
     
    1
 2
    cd deploy/examples/external
 kubectl create -f object-external.yaml
-
     
    Consume the S3 Storage, in two different ways:
     
       
    1.  
      Create an Object store user for credentials to access the S3 endpoint.
       
      1
+
       
      Consume the S3 Storage, in two different ways:
       
         
      1.  
        Create an Object store user for credentials to access the S3 endpoint.
         
        1
 2
        cd deploy/examples
 kubectl create -f object-user.yaml
-
         
        2.  
      2.  
        Create a bucket storage class where a client can request creating buckets and then create the Object Bucket Claim, which will create an individual bucket for reading and writing objects.
         
        1
+
         
        3.  
      3.  
        Create a bucket storage class where a client can request creating buckets and then create the Object Bucket Claim, which will create an individual bucket for reading and writing objects.
         
        1
 2
 3
        cd deploy/examples/external
 kubectl create -f storageclass-bucket-delete.yaml
diff --git a/docs/rook/v1.16/CRDs/Cluster/external-cluster/consumer-import/index.html b/docs/rook/v1.16/CRDs/Cluster/external-cluster/consumer-import/index.html
index 320d41b73..1b8a8f7a9 100644
--- a/docs/rook/v1.16/CRDs/Cluster/external-cluster/consumer-import/index.html
+++ b/docs/rook/v1.16/CRDs/Cluster/external-cluster/consumer-import/index.html
@@ -11,8 +11,8 @@
 helm install --create-namespace --namespace $clusterNamespace rook-ceph rook-release/rook-ceph -f values.yaml
 helm install --create-namespace --namespace $clusterNamespace rook-ceph-cluster \
 --set operatorNamespace=$operatorNamespace rook-release/rook-ceph-cluster -f values-external.yaml
-
         
        Manifest Installation
         
        If not installing with Helm, here are the steps to install with manifests.
         
           
        1.  
          Deploy Rook, create common.yaml, crds.yaml and operator.yaml manifests.
           
          2.  
        2.  
          Create common-external.yaml and cluster-external.yaml
           
          3.  
         
        Import the Provider Data
         
           
        1.  
          Paste the above output from create-external-cluster-resources.py into your current shell to allow importing the provider data.
           
          2.  
        2.  
          The import script in the next step uses the current kubeconfig context by default. If you want to specify the kubernetes cluster to use without changing the current context, you can specify the cluster name by setting the KUBECONTEXT environment variable.
           
          1
          export KUBECONTEXT=<cluster-name>
-
           
          3.  
        3.  
          Here is the link for import script. The script has used the rook-ceph namespace and few parameters that also have referenced from namespace variable. If user's external cluster has a different namespace, change the namespace parameter in the script according to their external cluster. For example with new-namespace namespace, this change is needed on the namespace parameter in the script.
           
          1
          NAMESPACE=${NAMESPACE:="new-namespace"}
+
           
          Manifest Installation
           
          If not installing with Helm, here are the steps to install with manifests.
           
             
          1.  
            Deploy Rook, create common.yaml, crds.yaml and operator.yaml manifests.
             
            2.  
          2.  
            Create common-external.yaml and cluster-external.yaml
             
            3.  
           
          Import the Provider Data
           
             
          1.  
            Paste the above output from create-external-cluster-resources.py into your current shell to allow importing the provider data.
             
            2.  
          2.  
            The import script in the next step uses the current kubeconfig context by default. If you want to specify the kubernetes cluster to use without changing the current context, you can specify the cluster name by setting the KUBECONTEXT environment variable.
             
            1
            export KUBECONTEXT=<cluster-name>
+
             
            3.  
          3.  
            Here is the link for import script. The script has used the rook-ceph namespace and few parameters that also have referenced from namespace variable. If user's external cluster has a different namespace, change the namespace parameter in the script according to their external cluster. For example with new-namespace namespace, this change is needed on the namespace parameter in the script.
             
            1
            NAMESPACE=${NAMESPACE:="new-namespace"}
 
             
            4.  
          4.  
            Run the import script.
             
             
            Note
             
            If your Rook cluster nodes are running a kernel earlier than or equivalent to 5.4, remove fast-diff, object-map, deep-flatten,exclusive-lock from the imageFeatures line.
             
             
            1
            . import-external-cluster.sh
 
             
            5.  
           
          Cluster Verification
           
             
          1.  
            Verify the consumer cluster is connected to the provider ceph cluster:
             
            1
 2
@@ -20,4 +20,4 @@
 NAME                 DATADIRHOSTPATH   MONCOUNT   AGE    STATE       HEALTH
 rook-ceph-external   /var/lib/rook                162m   Connected   HEALTH_OK
 
             
            2.  
          2.  
            Verify the creation of the storage class depending on the rbd pools and filesystem provided. ceph-rbd and cephfs would be the respective names for the RBD and CephFS storage classes.
             
            1
            kubectl -n rook-ceph get sc
-
             
            3.  
          3.  
            Create a persistent volume based on these StorageClass.
             
            4.  
           
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
+

  • Create a persistent volume based on these StorageClass.

    • © Rook Authors 2022. Documentation distributed under CC-BY-4.0.

    © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page.

    Made with Material for MkDocs
    \ No newline at end of file diff --git a/docs/rook/v1.16/CRDs/Cluster/external-cluster/provider-export/index.html b/docs/rook/v1.16/CRDs/Cluster/external-cluster/provider-export/index.html index 4a4e8616a..b95b7da37 100644 --- a/docs/rook/v1.16/CRDs/Cluster/external-cluster/provider-export/index.html +++ b/docs/rook/v1.16/CRDs/Cluster/external-cluster/provider-export/index.html @@ -1,4 +1,4 @@ - Export config from the Ceph provider cluster - Rook Ceph Documentation
    Skip to content

    Export config from the Ceph provider cluster

    In order to configure an external Ceph cluster with Rook, we need to extract some information in order to connect to that cluster.

    1. Create all users and keys

    Run the python script create-external-cluster-resources.py in the provider Ceph cluster cephadm shell, to have access to create the necessary users and keys.

    1
    python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --cephfs-filesystem-name <filesystem-name> --rgw-endpoint  <rgw-endpoint> --namespace <namespace> --format bash
+ Export config from the Ceph provider cluster - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Export config from the Ceph provider cluster
     
    In order to configure an external Ceph cluster with Rook, we need to extract some information in order to connect to that cluster.
     
    1. Create all users and keys
     
    Run the python script create-external-cluster-resources.py in the provider Ceph cluster cephadm shell, to have access to create the necessary users and keys.
     
    1
    python3 create-external-cluster-resources.py --rbd-data-pool-name <pool_name> --cephfs-filesystem-name <filesystem-name> --rgw-endpoint  <rgw-endpoint> --namespace <namespace> --format bash
 
     
       
    • --namespace: Namespace where CephCluster will run, for example rook-ceph
      •  
    • --format bash: The format of the output
      •  
    • --rbd-data-pool-name: The name of the RBD data pool
      •  
    • --alias-rbd-data-pool-name: Provides an alias for the RBD data pool name, necessary if a special character is present in the pool name such as a period or underscore
      •  
    • --rgw-endpoint: (optional) The RADOS Gateway endpoint in the format <IP>:<PORT> or <FQDN>:<PORT>.
      •  
    • --rgw-pool-prefix: (optional) The prefix of the RGW pools. If not specified, the default prefix is default
      •  
    • --rgw-tls-cert-path: (optional) RADOS Gateway endpoint TLS certificate (or intermediate signing certificate) file path
      •  
    • --rgw-skip-tls: (optional) Ignore TLS certification validation when a self-signed certificate is provided (NOT RECOMMENDED)
      •  
    • --rbd-metadata-ec-pool-name: (optional) Provides the name of erasure coded RBD metadata pool, used for creating ECRBDStorageClass.
      •  
    • --monitoring-endpoint: (optional) Ceph Manager prometheus exporter endpoints (comma separated list of IP entries of active and standby mgrs)
      •  
    • --monitoring-endpoint-port: (optional) Ceph Manager prometheus exporter port
      •  
    • --skip-monitoring-endpoint: (optional) Skip prometheus exporter endpoints, even if they are available. Useful if the prometheus module is not enabled
      •  
    • --ceph-conf: (optional) Provide a Ceph conf file
      •  
    • --keyring: (optional) Path to Ceph keyring file, to be used with --ceph-conf
      •  
    • --k8s-cluster-name: (optional) Kubernetes cluster name
      •  
    • --output: (optional) Output will be stored into the provided file
      •  
    • --dry-run: (optional) Prints the executed commands without running them
      •  
    • --run-as-user: (optional) Provides a user name to check the cluster's health status, must be prefixed by client.
      •  
    • --cephfs-metadata-pool-name: (optional) Provides the name of the cephfs metadata pool
      •  
    • --cephfs-filesystem-name: (optional) The name of the filesystem, used for creating CephFS StorageClass
      •  
    • --cephfs-data-pool-name: (optional) Provides the name of the CephFS data pool, used for creating CephFS StorageClass
      •  
    • --rados-namespace: (optional) Divides a pool into separate logical namespaces, used for creating RBD PVC in a CephBlockPoolRadosNamespace (should be lower case)
      •  
    • --subvolume-group: (optional) Provides the name of the subvolume group, used for creating CephFS PVC in a subvolumeGroup
      •  
    • --rgw-realm-name: (optional) Provides the name of the rgw-realm
      •  
    • --rgw-zone-name: (optional) Provides the name of the rgw-zone
      •  
    • --rgw-zonegroup-name: (optional) Provides the name of the rgw-zone-group
      •  
    • --upgrade: (optional) Upgrades the cephCSIKeyrings(For example: client.csi-cephfs-provisioner) and client.healthchecker ceph users with new permissions needed for the new cluster version and older permission will still be applied.
      •  
    • --restricted-auth-permission: (optional) Restrict cephCSIKeyrings auth permissions to specific pools, and cluster. Mandatory flags that need to be set are --rbd-data-pool-name, and --k8s-cluster-name. --cephfs-filesystem-name flag can also be passed in case of CephFS user restriction, so it can restrict users to particular CephFS filesystem.
      •  
    • --v2-port-enable: (optional) Enables the v2 mon port (3300) for mons.
      •  
    • --topology-pools: (optional) Comma-separated list of topology-constrained rbd pools
      •  
    • --topology-failure-domain-label: (optional) K8s cluster failure domain label (example: zone, rack, or host) for the topology-pools that match the ceph domain
      •  
    • --topology-failure-domain-values: (optional) Comma-separated list of the k8s cluster failure domain values corresponding to each of the pools in the topology-pools list
      •  
    • --config-file: Path to the configuration file, Priority: command-line-args > config.ini values > default values
      •  
     
    2. Copy the bash output
     
    Example Output:
     
     1
  2
  3
diff --git a/docs/rook/v1.16/CRDs/Cluster/external-cluster/topology-for-external-mode/index.html b/docs/rook/v1.16/CRDs/Cluster/external-cluster/topology-for-external-mode/index.html
index fbc0001b4..662e22a61 100644
--- a/docs/rook/v1.16/CRDs/Cluster/external-cluster/topology-for-external-mode/index.html
+++ b/docs/rook/v1.16/CRDs/Cluster/external-cluster/topology-for-external-mode/index.html
@@ -106,4 +106,4 @@
 provisioner: rook-ceph.rbd.csi.ceph.com
 reclaimPolicy: Delete
 volumeBindingMode: WaitForFirstConsumer
-
     
    Set two values in the rook-ceph-operator-config configmap:
     
       
    • CSI_ENABLE_TOPOLOGY: "true": Enable the feature
      •  
    • CSI_TOPOLOGY_DOMAIN_LABELS: "topology.kubernetes.io/zone": Set the topology domain labels that the CSI driver will analyze on the nodes during scheduling.
      •  
     
    Create a Topology-Based PVC
     
    The topology-based storage class is ready to be consumed! Create a PVC from the ceph-rbd-topology storage class above, and watch the OSD usage to see how the data is spread only among the topology-based CRUSH buckets.
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
+

    Set two values in the rook-ceph-operator-config configmap:

    • CSI_ENABLE_TOPOLOGY: "true": Enable the feature
    • CSI_TOPOLOGY_DOMAIN_LABELS: "topology.kubernetes.io/zone": Set the topology domain labels that the CSI driver will analyze on the nodes during scheduling.

    Create a Topology-Based PVC

    The topology-based storage class is ready to be consumed! Create a PVC from the ceph-rbd-topology storage class above, and watch the OSD usage to see how the data is spread only among the topology-based CRUSH buckets.

    © Rook Authors 2022. Documentation distributed under CC-BY-4.0.

    © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page.

    Made with Material for MkDocs
    \ No newline at end of file diff --git a/docs/rook/v1.16/CRDs/Cluster/network-providers/index.html b/docs/rook/v1.16/CRDs/Cluster/network-providers/index.html index f845287f9..3dd79b987 100644 --- a/docs/rook/v1.16/CRDs/Cluster/network-providers/index.html +++ b/docs/rook/v1.16/CRDs/Cluster/network-providers/index.html @@ -67,7 +67,7 @@

    Validating Multus configuration

    We highly recommend validating your Multus configuration before you install a CephCluster. A tool exists to facilitate validating the Multus configuration. After installing the Rook operator and before installing any Custom Resources, run the tool from the operator pod.

    The tool's CLI is designed to be as helpful as possible. Get help text for the multus validation tool like so:

    1. Exec into the Rook operator pod

      1
      kubectl --namespace rook-ceph exec -it deploy/rook-ceph-operator -- bash

    2. Output and read the tool's help text

      1
      rook multus validation run --help

    3. Use the validation tool config file for advanced configuration.

      1
      rook multus validation config --help
-

      Generate a sample config, that includes commented help text, using one of the available templates.

    4. Run the tool after configuring. If the tool fails, it will suggest what things may be preventing Multus networks from working properly, and it will request the logs and outputs that will help debug issues.

    Note

    The tool requires host network access. Many Kubernetes distros have security limitations. Use the tool's serviceAccountName config option or --service-account-name CLI flag to instruct the tool to run using a particular ServiceAccount in order to allow necessary permissions. An example compatible with openshift is provided in the Rook repository at deploy/examples/multus-validation-test-openshift.yaml

    Known limitations with Multus

    Daemons leveraging Kubernetes service IPs (Monitors, Managers, Rados Gateways) are not listening on the NAD specified in the selectors. Instead the daemon listens on the default network, however the NAD is attached to the container, allowing the daemon to communicate with the rest of the cluster. There is work in progress to fix this issue in the multus-service repository. At the time of writing it's unclear when this will be supported.

    Multus examples

    Macvlan, Whereabouts, Node Dynamic IPs

    The network plan for this cluster will be as follows:

    Node configuration must allow nodes to route to pods on the Multus public network.

    Because pods will be connecting via Macvlan, and because Macvlan does not allow hosts and pods to route between each other, the host must also be connected via Macvlan.

    Because the host IP range is different from the pod IP range, a route must be added to include the pod range.

    Such a configuration should be equivalent to the following:

    1
+

    Generate a sample config, that includes commented help text, using one of the available templates.

  • Run the tool after configuring. If the tool fails, it will suggest what things may be preventing Multus networks from working properly, and it will request the logs and outputs that will help debug issues.

    • Note

    The tool requires host network access. Many Kubernetes distros have security limitations. Use the tool's serviceAccountName config option or --service-account-name CLI flag to instruct the tool to run using a particular ServiceAccount in order to allow necessary permissions. An example compatible with openshift is provided in the Rook repository at deploy/examples/multus-validation-test-openshift.yaml

    Known limitations with Multus

    Daemons leveraging Kubernetes service IPs (Monitors, Managers, Rados Gateways) are not listening on the NAD specified in the selectors. Instead the daemon listens on the default network, however the NAD is attached to the container, allowing the daemon to communicate with the rest of the cluster. There is work in progress to fix this issue in the multus-service repository. At the time of writing it's unclear when this will be supported.

    Multus examples

    Macvlan, Whereabouts, Node Dynamic IPs

    The network plan for this cluster will be as follows:

    Node configuration must allow nodes to route to pods on the Multus public network.

    Because pods will be connecting via Macvlan, and because Macvlan does not allow hosts and pods to route between each other, the host must also be connected via Macvlan.

    Because the host IP range is different from the pod IP range, a route must be added to include the pod range.

    Such a configuration should be equivalent to the following:

    1
 2
 3
 4
    ip link add public-shim link eth0 type macvlan mode bridge
diff --git a/docs/rook/v1.16/CRDs/Cluster/stretch-cluster/index.html b/docs/rook/v1.16/CRDs/Cluster/stretch-cluster/index.html
index 2c07c9617..7a014cfe7 100644
--- a/docs/rook/v1.16/CRDs/Cluster/stretch-cluster/index.html
+++ b/docs/rook/v1.16/CRDs/Cluster/stretch-cluster/index.html
@@ -77,4 +77,4 @@
               values:
               - b
               - c
-

    For more details, see the Stretch Cluster design doc.

    \ No newline at end of file +

    For more details, see the Stretch Cluster design doc.

    \ No newline at end of file diff --git a/docs/rook/v1.16/CRDs/Shared-Filesystem/ceph-filesystem-crd/index.html b/docs/rook/v1.16/CRDs/Shared-Filesystem/ceph-filesystem-crd/index.html index 7ec478e1a..bd882a034 100644 --- a/docs/rook/v1.16/CRDs/Shared-Filesystem/ceph-filesystem-crd/index.html +++ b/docs/rook/v1.16/CRDs/Shared-Filesystem/ceph-filesystem-crd/index.html @@ -83,7 +83,7 @@ # requests: # cpu: "500m" # memory: "1024Mi" -

    (These definitions can also be found in the filesystem.yaml file)

    Erasure Coded

    Erasure coded pools require the OSDs to use bluestore for the configured storeType. Additionally, erasure coded pools can only be used with dataPools. The metadataPool must use a replicated pool.

    Note

    This sample requires at least 3 bluestore OSDs, with each OSD located on a different node.

    The OSDs must be located on different nodes, because the failureDomain will be set to host by default, and the erasureCoded chunk settings require at least 3 different OSDs (2 dataChunks + 1 codingChunks).

     1
+

    (These definitions can also be found in the filesystem.yaml file)

    Erasure Coded

    Erasure coded pools require the OSDs to use bluestore for the configured storeType. Additionally, erasure coded pools can only be used with dataPools. The metadataPool must use a replicated pool.

    Note

    This sample requires at least 3 bluestore OSDs, with each OSD located on a different node.

    The OSDs must be located on different nodes, because the failureDomain will be set to host by default, and the erasureCoded chunk settings require at least 3 different OSDs (2 dataChunks + 1 codingChunks).

     1
  2
  3
  4
@@ -122,4 +122,4 @@
   metadataServer:
     activeCount: 1
     activeStandby: true
-

    IMPORTANT: For erasure coded pools, we have to create a replicated pool as the default data pool and an erasure-coded pool as a secondary pool.

    (These definitions can also be found in the filesystem-ec.yaml file. Also see an example in the storageclass-ec.yaml for how to configure the volume.)

    Filesystem Settings

    Metadata

    Pools

    The pools allow all of the settings defined in the Pool CRD spec. For more details, see the Pool CRD settings. In the example above, there must be at least three hosts (size 3) and at least eight devices (6 data + 2 coding chunks) in the cluster.

    Generated Pool Names

    Both metadataPool and dataPools support defining names as required. The final pool name will consist of the filesystem name and pool name, e.g., <fsName>-<poolName> or <fsName>-metadata for metadataPool. For more granular configuration you may want to set preservePoolNames to true in pools to disable generation of names. In that case all pool names defined are used as given.

    Metadata Server Settings

    The metadata server settings correspond to the MDS daemon settings.

    MDS Resources Configuration Settings

    The format of the resource requests/limits structure is the same as described in the Ceph Cluster CRD documentation.

    If the memory resource limit is declared Rook will automatically set the MDS configuration mds_cache_memory_limit. The configuration value is calculated with the aim that the actual MDS memory consumption remains consistent with the MDS pods' resource declaration.

    In order to provide the best possible experience running Ceph in containers, Rook internally recommends the memory for MDS daemons to be at least 4096MB. If a user configures a limit or request value that is too low, Rook will still run the pod(s) and print a warning to the operator log.

    \ No newline at end of file +

    IMPORTANT: For erasure coded pools, we have to create a replicated pool as the default data pool and an erasure-coded pool as a secondary pool.

    (These definitions can also be found in the filesystem-ec.yaml file. Also see an example in the storageclass-ec.yaml for how to configure the volume.)

    Filesystem Settings

    Metadata

    Pools

    The pools allow all of the settings defined in the Pool CRD spec. For more details, see the Pool CRD settings. In the example above, there must be at least three hosts (size 3) and at least eight devices (6 data + 2 coding chunks) in the cluster.

    Generated Pool Names

    Both metadataPool and dataPools support defining names as required. The final pool name will consist of the filesystem name and pool name, e.g., <fsName>-<poolName> or <fsName>-metadata for metadataPool. For more granular configuration you may want to set preservePoolNames to true in pools to disable generation of names. In that case all pool names defined are used as given.

    Metadata Server Settings

    The metadata server settings correspond to the MDS daemon settings.

    MDS Resources Configuration Settings

    The format of the resource requests/limits structure is the same as described in the Ceph Cluster CRD documentation.

    If the memory resource limit is declared Rook will automatically set the MDS configuration mds_cache_memory_limit. The configuration value is calculated with the aim that the actual MDS memory consumption remains consistent with the MDS pods' resource declaration.

    In order to provide the best possible experience running Ceph in containers, Rook internally recommends the memory for MDS daemons to be at least 4096MB. If a user configures a limit or request value that is too low, Rook will still run the pod(s) and print a warning to the operator log.

    \ No newline at end of file diff --git a/docs/rook/v1.16/CRDs/Shared-Filesystem/ceph-fs-mirror-crd/index.html b/docs/rook/v1.16/CRDs/Shared-Filesystem/ceph-fs-mirror-crd/index.html index cf18d9575..1a4b2412c 100644 --- a/docs/rook/v1.16/CRDs/Shared-Filesystem/ceph-fs-mirror-crd/index.html +++ b/docs/rook/v1.16/CRDs/Shared-Filesystem/ceph-fs-mirror-crd/index.html @@ -9,4 +9,4 @@ name: my-fs-mirror namespace: rook-ceph spec: {} -

    Settings

    If any setting is unspecified, a suitable default will be used automatically.

    FilesystemMirror metadata

    FilesystemMirror Settings

    Configuring mirroring peers

    In order to configure mirroring peers, please refer to the CephFilesystem documentation.

    \ No newline at end of file +

    Settings

    If any setting is unspecified, a suitable default will be used automatically.

    FilesystemMirror metadata

    FilesystemMirror Settings

    Configuring mirroring peers

    In order to configure mirroring peers, please refer to the CephFilesystem documentation.

    \ No newline at end of file diff --git a/docs/rook/v1.16/CRDs/ceph-client-crd/index.html b/docs/rook/v1.16/CRDs/ceph-client-crd/index.html index b61b2880b..18ae26d10 100644 --- a/docs/rook/v1.16/CRDs/ceph-client-crd/index.html +++ b/docs/rook/v1.16/CRDs/ceph-client-crd/index.html @@ -44,7 +44,7 @@ export CEPH_KEYRING=/libsqliteceph/ceph.keyring; export CEPH_ARGS=--id example; ceph status -

    With this config, the ceph tools (ceph CLI, in-program access, etc) can connect to and utilize the Ceph cluster.

    Use Case: SQLite

    The Ceph project contains a SQLite VFS that interacts with RADOS directly, called libcephsqlite.

    First, on your workload ensure that you have the appropriate packages installed that make libcephsqlite.so available:

    Without the appropriate package (or a from-scratch build of SQLite), you will be unable to load libcephsqlite.so.

    After creating a CephClient similar to deploy/examples/sqlitevfs-client.yaml and retrieving it's credentials, you may set the following ENV variables:

    1
+

    With this config, the ceph tools (ceph CLI, in-program access, etc) can connect to and utilize the Ceph cluster.

    Use Case: SQLite

    The Ceph project contains a SQLite VFS that interacts with RADOS directly, called libcephsqlite.

    First, on your workload ensure that you have the appropriate packages installed that make libcephsqlite.so available:

    Without the appropriate package (or a from-scratch build of SQLite), you will be unable to load libcephsqlite.so.

    After creating a CephClient similar to deploy/examples/sqlitevfs-client.yaml and retrieving it's credentials, you may set the following ENV variables:

    1
 2
 3
    export CEPH_CONF=/libsqliteceph/ceph.conf;
 export CEPH_KEYRING=/libsqliteceph/ceph.keyring;
diff --git a/docs/rook/v1.16/CRDs/ceph-nfs-crd/index.html b/docs/rook/v1.16/CRDs/ceph-nfs-crd/index.html
index e8387e275..658d19288 100644
--- a/docs/rook/v1.16/CRDs/ceph-nfs-crd/index.html
+++ b/docs/rook/v1.16/CRDs/ceph-nfs-crd/index.html
@@ -141,4 +141,4 @@
         debugLevel: 0
 
         resources: {}
-

    NFS Settings

    Server

    The server spec sets configuration for Rook-created NFS-Ganesha server pods.

    Security

    The security spec sets security configuration for the NFS cluster.

    Scaling the active server count

    It is possible to scale the size of the cluster up or down by modifying the spec.server.active field. Scaling the cluster size up can be done at will. Once the new server comes up, clients can be assigned to it immediately.

    The CRD always eliminates the highest index servers first, in reverse order from how they were started. Scaling down the cluster requires that clients be migrated from servers that will be eliminated to others. That process is currently a manual one and should be performed before reducing the size of the cluster.

    Warning

    See the known issue below about setting this value greater than one.

    Known issues

    server.active count greater than 1

    \ No newline at end of file +

    NFS Settings

    Server

    The server spec sets configuration for Rook-created NFS-Ganesha server pods.

    Security

    The security spec sets security configuration for the NFS cluster.

    Scaling the active server count

    It is possible to scale the size of the cluster up or down by modifying the spec.server.active field. Scaling the cluster size up can be done at will. Once the new server comes up, clients can be assigned to it immediately.

    The CRD always eliminates the highest index servers first, in reverse order from how they were started. Scaling down the cluster requires that clients be migrated from servers that will be eliminated to others. That process is currently a manual one and should be performed before reducing the size of the cluster.

    Warning

    See the known issue below about setting this value greater than one.

    Known issues

    server.active count greater than 1

    \ No newline at end of file diff --git a/docs/rook/v1.16/Contributing/ci-configuration/index.html b/docs/rook/v1.16/Contributing/ci-configuration/index.html index 603ae5f4f..aa95653bd 100644 --- a/docs/rook/v1.16/Contributing/ci-configuration/index.html +++ b/docs/rook/v1.16/Contributing/ci-configuration/index.html @@ -1 +1 @@ - CI Configuration - Rook Ceph Documentation
    Skip to content

    CI Configuration

    This page contains information regarding the CI configuration used for the Rook project to test, build and release the project.

    Secrets

    • Snyk (Security Scan):
    • Testing:
    • Publishing:
      • DOCKER_USERNAME + DOCKER_PASSWORD: Username and password of registry.
      • DOCKER_REGISTRY: Target registry namespace (e.g., rook)
      • AWS_USR + AWS_PSW: AWS credentials with access to S3 for Helm chart publishing.
      • GIT_API_TOKEN: GitHub access token, used to push docs changes to the docs repositories gh-pages branch.

    © Rook Authors 2022. Documentation distributed under CC-BY-4.0.

    © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page.

    Made with Material for MkDocs
    \ No newline at end of file + CI Configuration - Rook Ceph Documentation
    Skip to content

    CI Configuration

    This page contains information regarding the CI configuration used for the Rook project to test, build and release the project.

    Secrets

    • Snyk (Security Scan):
    • Testing:
    • Publishing:
      • DOCKER_USERNAME + DOCKER_PASSWORD: Username and password of registry.
      • DOCKER_REGISTRY: Target registry namespace (e.g., rook)
      • AWS_USR + AWS_PSW: AWS credentials with access to S3 for Helm chart publishing.
      • GIT_API_TOKEN: GitHub access token, used to push docs changes to the docs repositories gh-pages branch.

    © Rook Authors 2022. Documentation distributed under CC-BY-4.0.

    © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page.

    Made with Material for MkDocs
    \ No newline at end of file diff --git a/docs/rook/v1.16/Contributing/development-environment/index.html b/docs/rook/v1.16/Contributing/development-environment/index.html index fb1b7825b..5d94d0766 100644 --- a/docs/rook/v1.16/Contributing/development-environment/index.html +++ b/docs/rook/v1.16/Contributing/development-environment/index.html @@ -14,7 +14,7 @@ # On MacOS with Apple silicon minikube start --disk-size=40g --extra-disks 1 --driver qemu

    It is recommended to install a Docker client on your host system too. Depending on your operating system follow the official guide.

    Stopping the cluster and destroying the Minikube virtual machine can be done with:

    1
    minikube delete
-

    Install Helm

    Use helm.sh to install Helm and set up Rook charts defined under _output/charts (generated by build):

    Note

    These helper scripts depend on some artifacts under the _output/ directory generated during build time. These scripts should be run from the project root.

    Note

    If Helm is not available in your PATH, Helm will be downloaded to a temporary directory (/tmp/rook-tests-scripts-helm) and used from that directory.

    Using local Rook image on minikube cluster

    Developers can test quickly their changes by building and using the local Rook image on their minikube cluster.

    1) Set the local Docker environment to use minikube:

    1
+

    Install Helm

    Use helm.sh to install Helm and set up Rook charts defined under _output/charts (generated by build):

    Note

    These helper scripts depend on some artifacts under the _output/ directory generated during build time. These scripts should be run from the project root.

    Note

    If Helm is not available in your PATH, Helm will be downloaded to a temporary directory (/tmp/rook-tests-scripts-helm) and used from that directory.

    Using local Rook image on minikube cluster

    Developers can test quickly their changes by building and using the local Rook image on their minikube cluster.

    1) Set the local Docker environment to use minikube:

    1
 2
 3
    ```console
 eval $(minikube docker-env -p minikube)
diff --git a/docs/rook/v1.16/Contributing/development-flow/index.html b/docs/rook/v1.16/Contributing/development-flow/index.html
index 2f4e1ac58..8d91c0e49 100644
--- a/docs/rook/v1.16/Contributing/development-flow/index.html
+++ b/docs/rook/v1.16/Contributing/development-flow/index.html
@@ -108,7 +108,7 @@
     │   └── utils
     ├── integration               # integration test cases that will be invoked during golang testing
     └── scripts                   # scripts for setting up integration and manual testing environments
-

    Development

    To submit a change, create a branch in your fork and then submit a pull request (PR) from the branch.

    Design Document

    For new features of significant scope and complexity, a design document is recommended before work begins on the implementation. Create a design document if:

    For smaller, straightforward features and bug fixes, there is no need for a design document. Authoring a design document has many advantages:

    Note

    Writing code to prototype the feature while working on the design may be very useful to help flesh out the approach.

    A design document should be written as a markdown file in the design folder. Follow the process outlined in the design template. There are many examples of previous design documents in that folder. Submit a pull request for the design to be discussed and approved by the community, just like any other change to the repository.

    Create a Branch

    From a console, create a new branch based on your fork where changes will be developed:

    1
+

    Development

    To submit a change, create a branch in your fork and then submit a pull request (PR) from the branch.

    Design Document

    For new features of significant scope and complexity, a design document is recommended before work begins on the implementation. Create a design document if:

    For smaller, straightforward features and bug fixes, there is no need for a design document. Authoring a design document has many advantages:

    Note

    Writing code to prototype the feature while working on the design may be very useful to help flesh out the approach.

    A design document should be written as a markdown file in the design folder. Follow the process outlined in the design template. There are many examples of previous design documents in that folder. Submit a pull request for the design to be discussed and approved by the community, just like any other change to the repository.

    Create a Branch

    From a console, create a new branch based on your fork where changes will be developed:

    1
 2
 3
 4
@@ -126,7 +126,7 @@

    Unit tests for individual packages can be run with the standard go test command.

    To see code coverage on the packages that you changed, view the coverage.html in a browser to inspect your new code.

    1
 2
    go test -coverprofile=coverage.out
 go tool cover -html=coverage.out -o coverage.html
-

    Writing unit tests

    Good unit tests start with easily testable code. Small chunks ("units") of code can be easily tested for every possible input. Higher-level code units that are built from smaller, already-tested units can more easily verify that the units are combined together correctly.

    Common cases that may need tests:

    Integration Tests

    Rook's upstream continuous integration (CI) tests will run integration tests against your changes automatically.

    Tmate Session

    Integration tests will be run in Github actions. If an integration test fails, enable a tmate session to troubleshoot the issue by one of the following steps:

    See the action details for an ssh connection to the Github runner.

    Commit structure

    Rook maintainers value clear, lengthy and explanatory commit messages.

    Requirements for commits:

    An example acceptable commit message:

    1
+

    Writing unit tests

    Good unit tests start with easily testable code. Small chunks ("units") of code can be easily tested for every possible input. Higher-level code units that are built from smaller, already-tested units can more easily verify that the units are combined together correctly.

    Common cases that may need tests:

    Integration Tests

    Rook's upstream continuous integration (CI) tests will run integration tests against your changes automatically.

    Tmate Session

    Integration tests will be run in Github actions. If an integration test fails, enable a tmate session to troubleshoot the issue by one of the following steps:

    See the action details for an ssh connection to the Github runner.

    Commit structure

    Rook maintainers value clear, lengthy and explanatory commit messages.

    Requirements for commits:

    An example acceptable commit message:

    1
 2
 3
 4
diff --git a/docs/rook/v1.16/Getting-Started/ceph-openshift/index.html b/docs/rook/v1.16/Getting-Started/ceph-openshift/index.html
index 996d294ab..bfd7ca321 100644
--- a/docs/rook/v1.16/Getting-Started/ceph-openshift/index.html
+++ b/docs/rook/v1.16/Getting-Started/ceph-openshift/index.html
@@ -1,4 +1,4 @@
- OpenShift - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    OpenShift
     
    OpenShift adds a number of security and other enhancements to Kubernetes. In particular, security context constraints allow the cluster admin to define exactly which permissions are allowed to pods running in the cluster. You will need to define those permissions that allow the Rook pods to run.
     
    The settings for Rook in OpenShift are described below, and are also included in the example yaml files:
      
    TL;DR
     
    To create an OpenShift cluster, the commands basically include:
     
    1
+ OpenShift - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    OpenShift
     
    OpenShift adds a number of security and other enhancements to Kubernetes. In particular, security context constraints allow the cluster admin to define exactly which permissions are allowed to pods running in the cluster. You will need to define those permissions that allow the Rook pods to run.
     
    The settings for Rook in OpenShift are described below, and are also included in the example yaml files:
      
    TL;DR
     
    To create an OpenShift cluster, the commands basically include:
     
    1
 2
 3
    oc create -f crds.yaml -f common.yaml
 oc create -f operator-openshift.yaml
diff --git a/docs/rook/v1.16/Getting-Started/example-configurations/index.html b/docs/rook/v1.16/Getting-Started/example-configurations/index.html
index 188acb140..c3f307f03 100644
--- a/docs/rook/v1.16/Getting-Started/example-configurations/index.html
+++ b/docs/rook/v1.16/Getting-Started/example-configurations/index.html
@@ -1,2 +1,2 @@
- Example Configurations - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Example Configurations
     
    Configuration for Rook and Ceph can be configured in multiple ways to provide block devices, shared filesystem volumes or object storage in a kubernetes namespace. While several examples are provided to simplify storage setup, settings are available to optimize various production environments.
     
    See the example yaml files folder for all the rook/ceph setup example spec files.
     
    Common Resources
     
    The first step to deploy Rook is to create the CRDs and other common resources. The configuration for these resources will be the same for most deployments. The crds.yaml and common.yaml sets these resources up.
     
    1
    kubectl create -f crds.yaml -f common.yaml
-
     
    The examples all assume the operator and all Ceph daemons will be started in the same namespace. If deploying the operator in a separate namespace, see the comments throughout common.yaml.
     
    Operator
     
    After the common resources are created, the next step is to create the Operator deployment. Several spec file examples are provided in this directory:
     
       
    • operator.yaml: The most common settings for production deployments
         
      • kubectl create -f operator.yaml
        •  
       
      •  
    • operator-openshift.yaml: Includes all of the operator settings for running a basic Rook cluster in an OpenShift environment. You will also want to review the OpenShift Prerequisites to confirm the settings.
         
      • oc create -f operator-openshift.yaml
        •  
       
      •  
     
    Settings for the operator are configured through environment variables on the operator deployment. The individual settings are documented in operator.yaml.
     
    Cluster CRD
     
    Now that the operator is running, create the Ceph storage cluster with the CephCluster CR. This CR contains the most critical settings that will influence how the operator configures the storage. It is important to understand the various ways to configure the cluster. These examples represent several different ways to configure the storage.
      
    See the Cluster CRD topic for more details and more examples for the settings.
     
    Setting up consumable storage
     
    Now we are ready to setup Block, Shared Filesystem or Object storage in the Rook cluster. These storage types are respectively created with the CephBlockPool, CephFilesystem and CephObjectStore CRs.
     
    Block Devices
     
    Ceph provides raw block device volumes to pods. Each example below sets up a storage class which can then be used to provision a block device in application pods. The storage class is defined with a Ceph pool which defines the level of data redundancy in Ceph:
     
       
    • storageclass.yaml: This example illustrates replication of 3 for production scenarios and requires at least three worker nodes. Data is replicated on three different kubernetes worker nodes. Intermittent or long-lasting single node failures will not result in data unavailability or loss.
      •  
    • storageclass-ec.yaml: Configures erasure coding for data durability rather than replication. Ceph's erasure coding is more efficient than replication so you can get high reliability without the 3x replication cost of the preceding example (but at the cost of higher computational encoding and decoding costs on the worker nodes). Erasure coding requires at least three worker nodes. See the Erasure coding documentation.
      •  
    • storageclass-test.yaml: Replication of 1 for test scenarios. Requires only a single node. Do not use this for production applications. A single node failure can result in full data loss.
      •  
     
    The block storage classes are found in the examples directory:
     
       
    • csi/rbd: the CSI driver examples for block devices
      •  
     
    See the CephBlockPool CRD topic for more block storage settings.
     
    Shared Filesystem
     
    Ceph filesystem (CephFS) allows the user to mount a shared posix-compliant folder into one or more application pods. This storage is similar to NFS shared storage or CIFS shared folders, as explained here.
     
    Shared Filesystem storage contains configurable pools for different scenarios:
     
       
    • filesystem.yaml: Replication of 3 for production scenarios. Requires at least three worker nodes.
      •  
    • filesystem-ec.yaml: Erasure coding for production scenarios. Requires at least three worker nodes.
      •  
    • filesystem-test.yaml: Replication of 1 for test scenarios. Requires only a single node.
      •  
     
    Dynamic provisioning is possible with the CSI driver. The storage class for shared filesystems is found in the csi/cephfs directory.
     
    See the Shared Filesystem CRD topic for more details on the settings.
     
    Object Storage
     
    Ceph supports storing blobs of data called objects that support HTTP(s)-type get/put/post and delete semantics. This storage is similar to AWS S3 storage, for example.
     
    Object storage contains multiple pools that can be configured for different scenarios:
     
       
    • object.yaml: Replication of 3 for production scenarios. Requires at least three worker nodes.
      •  
    • object-openshift.yaml: Replication of 3 with rgw in a port range valid for OpenShift. Requires at least three worker nodes.
      •  
    • object-ec.yaml: Erasure coding rather than replication for production scenarios. Requires at least three worker nodes.
      •  
    • object-test.yaml: Replication of 1 for test scenarios. Requires only a single node.
      •  
     
    See the Object Store CRD topic for more details on the settings.
     
    Object Storage User
     
       
    • object-user.yaml: Creates a simple object storage user and generates credentials for the S3 API
      •  
     
    Object Storage Buckets
     
    The Ceph operator also runs an object store bucket provisioner which can grant access to existing buckets or dynamically provision new buckets.
     
       
    • object-bucket-claim-retain.yaml Creates a request for a new bucket by referencing a StorageClass which saves the bucket when the initiating OBC is deleted.
      •  
    • object-bucket-claim-delete.yaml Creates a request for a new bucket by referencing a StorageClass which deletes the bucket when the initiating OBC is deleted.
      •  
    • storageclass-bucket-retain.yaml Creates a new StorageClass which defines the Ceph Object Store and retains the bucket after the initiating OBC is deleted.
      •  
    • storageclass-bucket-delete.yaml Creates a new StorageClass which defines the Ceph Object Store and deletes the bucket after the initiating OBC is deleted.
      •  
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
+ Example Configurations - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Example Configurations
     
    Configuration for Rook and Ceph can be configured in multiple ways to provide block devices, shared filesystem volumes or object storage in a kubernetes namespace. While several examples are provided to simplify storage setup, settings are available to optimize various production environments.
     
    See the example yaml files folder for all the rook/ceph setup example spec files.
     
    Common Resources
     
    The first step to deploy Rook is to create the CRDs and other common resources. The configuration for these resources will be the same for most deployments. The crds.yaml and common.yaml sets these resources up.
     
    1
    kubectl create -f crds.yaml -f common.yaml
+
     
    The examples all assume the operator and all Ceph daemons will be started in the same namespace. If deploying the operator in a separate namespace, see the comments throughout common.yaml.
     
    Operator
     
    After the common resources are created, the next step is to create the Operator deployment. Several spec file examples are provided in this directory:
     
       
    • operator.yaml: The most common settings for production deployments
         
      • kubectl create -f operator.yaml
        •  
       
      •  
    • operator-openshift.yaml: Includes all of the operator settings for running a basic Rook cluster in an OpenShift environment. You will also want to review the OpenShift Prerequisites to confirm the settings.
         
      • oc create -f operator-openshift.yaml
        •  
       
      •  
     
    Settings for the operator are configured through environment variables on the operator deployment. The individual settings are documented in operator.yaml.
     
    Cluster CRD
     
    Now that the operator is running, create the Ceph storage cluster with the CephCluster CR. This CR contains the most critical settings that will influence how the operator configures the storage. It is important to understand the various ways to configure the cluster. These examples represent several different ways to configure the storage.
      
    See the Cluster CRD topic for more details and more examples for the settings.
     
    Setting up consumable storage
     
    Now we are ready to setup Block, Shared Filesystem or Object storage in the Rook cluster. These storage types are respectively created with the CephBlockPool, CephFilesystem and CephObjectStore CRs.
     
    Block Devices
     
    Ceph provides raw block device volumes to pods. Each example below sets up a storage class which can then be used to provision a block device in application pods. The storage class is defined with a Ceph pool which defines the level of data redundancy in Ceph:
     
       
    • storageclass.yaml: This example illustrates replication of 3 for production scenarios and requires at least three worker nodes. Data is replicated on three different kubernetes worker nodes. Intermittent or long-lasting single node failures will not result in data unavailability or loss.
      •  
    • storageclass-ec.yaml: Configures erasure coding for data durability rather than replication. Ceph's erasure coding is more efficient than replication so you can get high reliability without the 3x replication cost of the preceding example (but at the cost of higher computational encoding and decoding costs on the worker nodes). Erasure coding requires at least three worker nodes. See the Erasure coding documentation.
      •  
    • storageclass-test.yaml: Replication of 1 for test scenarios. Requires only a single node. Do not use this for production applications. A single node failure can result in full data loss.
      •  
     
    The block storage classes are found in the examples directory:
     
       
    • csi/rbd: the CSI driver examples for block devices
      •  
     
    See the CephBlockPool CRD topic for more block storage settings.
     
    Shared Filesystem
     
    Ceph filesystem (CephFS) allows the user to mount a shared posix-compliant folder into one or more application pods. This storage is similar to NFS shared storage or CIFS shared folders, as explained here.
     
    Shared Filesystem storage contains configurable pools for different scenarios:
     
       
    • filesystem.yaml: Replication of 3 for production scenarios. Requires at least three worker nodes.
      •  
    • filesystem-ec.yaml: Erasure coding for production scenarios. Requires at least three worker nodes.
      •  
    • filesystem-test.yaml: Replication of 1 for test scenarios. Requires only a single node.
      •  
     
    Dynamic provisioning is possible with the CSI driver. The storage class for shared filesystems is found in the csi/cephfs directory.
     
    See the Shared Filesystem CRD topic for more details on the settings.
     
    Object Storage
     
    Ceph supports storing blobs of data called objects that support HTTP(s)-type get/put/post and delete semantics. This storage is similar to AWS S3 storage, for example.
     
    Object storage contains multiple pools that can be configured for different scenarios:
     
       
    • object.yaml: Replication of 3 for production scenarios. Requires at least three worker nodes.
      •  
    • object-openshift.yaml: Replication of 3 with rgw in a port range valid for OpenShift. Requires at least three worker nodes.
      •  
    • object-ec.yaml: Erasure coding rather than replication for production scenarios. Requires at least three worker nodes.
      •  
    • object-test.yaml: Replication of 1 for test scenarios. Requires only a single node.
      •  
     
    See the Object Store CRD topic for more details on the settings.
     
    Object Storage User
     
       
    • object-user.yaml: Creates a simple object storage user and generates credentials for the S3 API
      •  
     
    Object Storage Buckets
     
    The Ceph operator also runs an object store bucket provisioner which can grant access to existing buckets or dynamically provision new buckets.
     
       
    • object-bucket-claim-retain.yaml Creates a request for a new bucket by referencing a StorageClass which saves the bucket when the initiating OBC is deleted.
      •  
    • object-bucket-claim-delete.yaml Creates a request for a new bucket by referencing a StorageClass which deletes the bucket when the initiating OBC is deleted.
      •  
    • storageclass-bucket-retain.yaml Creates a new StorageClass which defines the Ceph Object Store and retains the bucket after the initiating OBC is deleted.
      •  
    • storageclass-bucket-delete.yaml Creates a new StorageClass which defines the Ceph Object Store and deletes the bucket after the initiating OBC is deleted.
      •  
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
diff --git a/docs/rook/v1.16/Getting-Started/intro/index.html b/docs/rook/v1.16/Getting-Started/intro/index.html
index 901fa3f06..6c19e1caa 100644
--- a/docs/rook/v1.16/Getting-Started/intro/index.html
+++ b/docs/rook/v1.16/Getting-Started/intro/index.html
@@ -1 +1 @@
- Rook - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Rook
     
    Rook is an open source cloud-native storage orchestrator, providing the platform, framework, and support for Ceph storage to natively integrate with cloud-native environments.
     
    Ceph is a distributed storage system that provides file, block and object storage and is deployed in large scale production clusters.
     
    Rook automates deployment and management of Ceph to provide self-managing, self-scaling, and self-healing storage services. The Rook operator does this by building on Kubernetes resources to deploy, configure, provision, scale, upgrade, and monitor Ceph.
     
    The Ceph operator was declared stable in December 2018 in the Rook v0.9 release, providing a production storage platform for many years. Rook is hosted by the Cloud Native Computing Foundation (CNCF) as a graduated level project.
     
    Quick Start Guide
     
    Starting Ceph in your cluster is as simple as a few kubectl commands. See our Quickstart guide to get started with the Ceph operator!
     
    Designs
     
    Ceph is a highly scalable distributed storage solution for block storage, object storage, and shared filesystems with years of production deployments. See the Ceph overview.
     
    For detailed design documentation, see also the design docs.
     
    Need help? Be sure to join the Rook Slack
     
    If you have any questions along the way, don't hesitate to ask in our Slack channel. Sign up for the Rook Slack here.
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
+ Rook - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Rook
     
    Rook is an open source cloud-native storage orchestrator, providing the platform, framework, and support for Ceph storage to natively integrate with cloud-native environments.
     
    Ceph is a distributed storage system that provides file, block and object storage and is deployed in large scale production clusters.
     
    Rook automates deployment and management of Ceph to provide self-managing, self-scaling, and self-healing storage services. The Rook operator does this by building on Kubernetes resources to deploy, configure, provision, scale, upgrade, and monitor Ceph.
     
    The Ceph operator was declared stable in December 2018 in the Rook v0.9 release, providing a production storage platform for many years. Rook is hosted by the Cloud Native Computing Foundation (CNCF) as a graduated level project.
     
    Quick Start Guide
     
    Starting Ceph in your cluster is as simple as a few kubectl commands. See our Quickstart guide to get started with the Ceph operator!
     
    Designs
     
    Ceph is a highly scalable distributed storage solution for block storage, object storage, and shared filesystems with years of production deployments. See the Ceph overview.
     
    For detailed design documentation, see also the design docs.
     
    Need help? Be sure to join the Rook Slack
     
    If you have any questions along the way, don't hesitate to ask in our Slack channel. Sign up for the Rook Slack here.
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
diff --git a/docs/rook/v1.16/Getting-Started/quickstart/index.html b/docs/rook/v1.16/Getting-Started/quickstart/index.html
index 9d14a1d06..d6ae4a224 100644
--- a/docs/rook/v1.16/Getting-Started/quickstart/index.html
+++ b/docs/rook/v1.16/Getting-Started/quickstart/index.html
@@ -1,11 +1,11 @@
- Quickstart - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Quickstart
     
    Welcome to Rook! We hope you have a great experience installing the Rook cloud-native storage orchestrator platform to enable highly available, durable Ceph storage in Kubernetes clusters.
     
    Don't hesitate to ask questions in our Slack channel. Sign up for the Rook Slack here.
     
    This guide will walk through the basic setup of a Ceph cluster and enable K8s applications to consume block, object, and file storage.
     
    Always use a virtual machine when testing Rook. Never use a host system where local devices may mistakenly be consumed.
     
    Kubernetes Version
     
    Kubernetes versions v1.27 through v1.32 are supported.
     
    CPU Architecture
     
    Architectures released are amd64 / x86_64 and arm64.
     
    Prerequisites
     
    To check if a Kubernetes cluster is ready for Rook, see the prerequisites.
     
    To configure the Ceph storage cluster, at least one of these local storage options are required:
     
       
    • Raw devices (no partitions or formatted filesystem)
      •  
    • Raw partitions (no formatted filesystem)
      •  
    • LVM Logical Volumes (no formatted filesystem)
      •  
    • Encrypted devices (no formatted filesystem)
      •  
    • Multipath devices (no formatted filesystem)
      •  
    • Persistent Volumes available from a storage class in block mode
      •  
     
    TL;DR
     
    A simple Rook cluster is created for Kubernetes with the following kubectl commands and example manifests.
     
    1
+ Quickstart - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Quickstart
     
    Welcome to Rook! We hope you have a great experience installing the Rook cloud-native storage orchestrator platform to enable highly available, durable Ceph storage in Kubernetes clusters.
     
    Don't hesitate to ask questions in our Slack channel. Sign up for the Rook Slack here.
     
    This guide will walk through the basic setup of a Ceph cluster and enable K8s applications to consume block, object, and file storage.
     
    Always use a virtual machine when testing Rook. Never use a host system where local devices may mistakenly be consumed.
     
    Kubernetes Version
     
    Kubernetes versions v1.27 through v1.32 are supported.
     
    CPU Architecture
     
    Architectures released are amd64 / x86_64 and arm64.
     
    Prerequisites
     
    To check if a Kubernetes cluster is ready for Rook, see the prerequisites.
     
    To configure the Ceph storage cluster, at least one of these local storage options are required:
     
       
    • Raw devices (no partitions or formatted filesystem)
      •  
    • Raw partitions (no formatted filesystem)
      •  
    • LVM Logical Volumes (no formatted filesystem)
      •  
    • Encrypted devices (no formatted filesystem)
      •  
    • Multipath devices (no formatted filesystem)
      •  
    • Persistent Volumes available from a storage class in block mode
      •  
     
    TL;DR
     
    A simple Rook cluster is created for Kubernetes with the following kubectl commands and example manifests.
     
    1
 2
 3
 4
    $ git clone --single-branch --branch v1.16.4 https://github.com/rook/rook.git
 cd rook/deploy/examples
 kubectl create -f crds.yaml -f common.yaml -f operator.yaml
 kubectl create -f cluster.yaml
-
     
    After the cluster is running, applications can consume block, object, or file storage.
     
    Deploy the Rook Operator
     
    The first step is to deploy the Rook operator.
     
     
    Important
     
    The Rook Helm Chart is available to deploy the operator instead of creating the below manifests.
     
     
     
    Note
     
    Check that the example yaml files are from a tagged release of Rook.
     
     
     
    Note
     
    These steps are for a standard production Rook deployment in Kubernetes. For Openshift, testing, or more options, see the example configurations documentation.
     
     
    1
+
     
    After the cluster is running, applications can consume block, object, or file storage.
     
    Deploy the Rook Operator
     
    The first step is to deploy the Rook operator.
     
     
    Important
     
    The Rook Helm Chart is available to deploy the operator instead of creating the below manifests.
     
     
     
    Note
     
    Check that the example yaml files are from a tagged release of Rook.
     
     
     
    Note
     
    These steps are for a standard production Rook deployment in Kubernetes. For Openshift, testing, or more options, see the example configurations documentation.
     
     
    1
 2
 3
 4
@@ -14,7 +14,7 @@
 
 # verify the rook-ceph-operator is in the `Running` state before proceeding
 kubectl -n rook-ceph get pod
-
     
    Before starting the operator in production, consider these settings:
     
       
    1. Some Rook features are disabled by default. See the operator.yaml for these and other advanced settings.
         
      1. Device discovery: Rook will watch for new devices to configure if the ROOK_ENABLE_DISCOVERY_DAEMON setting is enabled, commonly used in bare metal clusters.
        2.  
      2. Node affinity and tolerations: The CSI driver by default will run on any node in the cluster. To restrict the CSI driver affinity, several settings are available.
        3.  
       
      2.  
    2. If deploying Rook into a namespace other than the default rook-ceph, see the topic on using an alternative namespace.
      3.  
     
    Cluster Environments
     
    The Rook documentation is focused around starting Rook in a variety of environments. While creating the cluster in this guide, consider these example cluster manifests:
     
       
    • cluster.yaml: Cluster settings for a production cluster running on bare metal. Requires at least three worker nodes.
      •  
    • cluster-on-pvc.yaml: Cluster settings for a production cluster running in a dynamic cloud environment.
      •  
    • cluster-test.yaml: Cluster settings for a test environment such as minikube.
      •  
     
    See the Ceph example configurations for more details.
     
    Create a Ceph Cluster
     
    Now that the Rook operator is running we can create the Ceph cluster.
     
     
    Important
     
    The Rook Cluster Helm Chart is available to deploy the operator instead of creating the below manifests.
     
     
     
    Important
     
    For the cluster to survive reboots, set the dataDirHostPath property that is valid for the hosts. For more settings, see the documentation on configuring the cluster.
     
     
    Create the cluster:
     
    1
    kubectl create -f cluster.yaml
+
     
    Before starting the operator in production, consider these settings:
     
       
    1. Some Rook features are disabled by default. See the operator.yaml for these and other advanced settings.
         
      1. Device discovery: Rook will watch for new devices to configure if the ROOK_ENABLE_DISCOVERY_DAEMON setting is enabled, commonly used in bare metal clusters.
        2.  
      2. Node affinity and tolerations: The CSI driver by default will run on any node in the cluster. To restrict the CSI driver affinity, several settings are available.
        3.  
       
      2.  
    2. If deploying Rook into a namespace other than the default rook-ceph, see the topic on using an alternative namespace.
      3.  
     
    Cluster Environments
     
    The Rook documentation is focused around starting Rook in a variety of environments. While creating the cluster in this guide, consider these example cluster manifests:
     
       
    • cluster.yaml: Cluster settings for a production cluster running on bare metal. Requires at least three worker nodes.
      •  
    • cluster-on-pvc.yaml: Cluster settings for a production cluster running in a dynamic cloud environment.
      •  
    • cluster-test.yaml: Cluster settings for a test environment such as minikube.
      •  
     
    See the Ceph example configurations for more details.
     
    Create a Ceph Cluster
     
    Now that the Rook operator is running we can create the Ceph cluster.
     
     
    Important
     
    The Rook Cluster Helm Chart is available to deploy the operator instead of creating the below manifests.
     
     
     
    Important
     
    For the cluster to survive reboots, set the dataDirHostPath property that is valid for the hosts. For more settings, see the documentation on configuring the cluster.
     
     
    Create the cluster:
     
    1
    kubectl create -f cluster.yaml
 
     
    Verify the cluster is running by viewing the pods in the rook-ceph namespace.
     
    The number of osd pods will depend on the number of nodes in the cluster and the number of devices configured. For the default cluster.yaml above, one OSD will be created for each available device found on each node.
     
     
    Hint
     
    If the rook-ceph-mon, rook-ceph-mgr, or rook-ceph-osd pods are not created, please refer to the Ceph common issues for more details and potential solutions.
     
     
     1
  2
  3
diff --git a/docs/rook/v1.16/Helm-Charts/ceph-cluster-chart/index.html b/docs/rook/v1.16/Helm-Charts/ceph-cluster-chart/index.html
index de61969c1..de83671fc 100644
--- a/docs/rook/v1.16/Helm-Charts/ceph-cluster-chart/index.html
+++ b/docs/rook/v1.16/Helm-Charts/ceph-cluster-chart/index.html
@@ -1,4 +1,4 @@
- Ceph Cluster Helm Chart - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Ceph Cluster Helm Chart
     
    Creates Rook resources to configure a Ceph cluster using the Helm package manager. This chart is a simple packaging of templates that will optionally create Rook resources such as:
     
       
    • CephCluster, CephFilesystem, and CephObjectStore CRs
      •  
    • Storage classes to expose Ceph RBD volumes, CephFS volumes, and RGW buckets
      •  
    • Ingress for external access to the dashboard
      •  
    • Toolbox
      •  
     
    Prerequisites
      
    Installing
     
    The helm install command deploys rook on the Kubernetes cluster in the default configuration. The configuration section lists the parameters that can be configured during installation. It is recommended that the rook operator be installed into the rook-ceph namespace. The clusters can be installed into the same namespace as the operator or a separate namespace.
     
    Rook currently publishes builds of this chart to the release and master channels.
     
    Before installing, review the values.yaml to confirm if the default settings need to be updated.
     
       
    • If the operator was installed in a namespace other than rook-ceph, the namespace must be set in the operatorNamespace variable.
      •  
    • Set the desired settings in the cephClusterSpec. The defaults are only an example and not likely to apply to your cluster.
      •  
    • The monitoring section should be removed from the cephClusterSpec, as it is specified separately in the helm settings.
      •  
    • The default values for cephBlockPools, cephFileSystems, and CephObjectStores will create one of each, and their corresponding storage classes.
      •  
    • All Ceph components now have default values for the pod resources. The resources may need to be adjusted in production clusters depending on the load. The resources can also be disabled if Ceph should not be limited (e.g. test clusters).
      •  
     
    Release
     
    The release channel is the most recent release of Rook that is considered stable for the community.
     
    The example install assumes you have first installed the Rook Operator Helm Chart and created your customized values.yaml.
     
    1
+ Ceph Cluster Helm Chart - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Ceph Cluster Helm Chart
     
    Creates Rook resources to configure a Ceph cluster using the Helm package manager. This chart is a simple packaging of templates that will optionally create Rook resources such as:
     
       
    • CephCluster, CephFilesystem, and CephObjectStore CRs
      •  
    • Storage classes to expose Ceph RBD volumes, CephFS volumes, and RGW buckets
      •  
    • Ingress for external access to the dashboard
      •  
    • Toolbox
      •  
     
    Prerequisites
      
    Installing
     
    The helm install command deploys rook on the Kubernetes cluster in the default configuration. The configuration section lists the parameters that can be configured during installation. It is recommended that the rook operator be installed into the rook-ceph namespace. The clusters can be installed into the same namespace as the operator or a separate namespace.
     
    Rook currently publishes builds of this chart to the release and master channels.
     
    Before installing, review the values.yaml to confirm if the default settings need to be updated.
     
       
    • If the operator was installed in a namespace other than rook-ceph, the namespace must be set in the operatorNamespace variable.
      •  
    • Set the desired settings in the cephClusterSpec. The defaults are only an example and not likely to apply to your cluster.
      •  
    • The monitoring section should be removed from the cephClusterSpec, as it is specified separately in the helm settings.
      •  
    • The default values for cephBlockPools, cephFileSystems, and CephObjectStores will create one of each, and their corresponding storage classes.
      •  
    • All Ceph components now have default values for the pod resources. The resources may need to be adjusted in production clusters depending on the load. The resources can also be disabled if Ceph should not be limited (e.g. test clusters).
      •  
     
    Release
     
    The release channel is the most recent release of Rook that is considered stable for the community.
     
    The example install assumes you have first installed the Rook Operator Helm Chart and created your customized values.yaml.
     
    1
 2
 3
    helm repo add rook-release https://charts.rook.io/release
 helm install --create-namespace --namespace rook-ceph rook-ceph-cluster \
diff --git a/docs/rook/v1.16/Helm-Charts/helm-charts/index.html b/docs/rook/v1.16/Helm-Charts/helm-charts/index.html
index 8fbb20872..56c535f68 100644
--- a/docs/rook/v1.16/Helm-Charts/helm-charts/index.html
+++ b/docs/rook/v1.16/Helm-Charts/helm-charts/index.html
@@ -1 +1 @@
- Helm Charts Overview - Rook Ceph Documentation     
     
     
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Helm Charts Overview
     
    Rook has published the following Helm charts for the Ceph storage provider:
     
       
    • Rook Ceph Operator: Starts the Ceph Operator, which will watch for Ceph CRs (custom resources)
      •  
    • Rook Ceph Cluster: Creates Ceph CRs that the operator will use to configure the cluster
      •  
     
    The Helm charts are intended to simplify deployment and upgrades. Configuring the Rook resources without Helm is also fully supported by creating the manifests directly.
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
+ Helm Charts Overview - Rook Ceph Documentation     
     
     
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Helm Charts Overview
     
    Rook has published the following Helm charts for the Ceph storage provider:
     
       
    • Rook Ceph Operator: Starts the Ceph Operator, which will watch for Ceph CRs (custom resources)
      •  
    • Rook Ceph Cluster: Creates Ceph CRs that the operator will use to configure the cluster
      •  
     
    The Helm charts are intended to simplify deployment and upgrades. Configuring the Rook resources without Helm is also fully supported by creating the manifests directly.
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
diff --git a/docs/rook/v1.16/Helm-Charts/operator-chart/index.html b/docs/rook/v1.16/Helm-Charts/operator-chart/index.html
index 0bacad42a..3b362e29a 100644
--- a/docs/rook/v1.16/Helm-Charts/operator-chart/index.html
+++ b/docs/rook/v1.16/Helm-Charts/operator-chart/index.html
@@ -1,7 +1,7 @@
  Ceph Operator Helm Chart - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Ceph Operator Helm Chart
     
    Installs rook to create, configure, and manage Ceph clusters on Kubernetes.
     
    Introduction
     
    This chart bootstraps a rook-ceph-operator deployment on a Kubernetes cluster using the Helm package manager.
     
    Prerequisites
     
       
    • Kubernetes 1.22+
      •  
    • Helm 3.x
      •  
     
    See the Helm support matrix for more details.
     
    Installing
     
    The Ceph Operator helm chart will install the basic components necessary to create a storage platform for your Kubernetes cluster.
     
       
    1. Install the Helm chart
      2.  
    2. Create a Rook cluster.
      3.  
     
    The helm install command deploys rook on the Kubernetes cluster in the default configuration. The configuration section lists the parameters that can be configured during installation. It is recommended that the rook operator be installed into the rook-ceph namespace (you will install your clusters into separate namespaces).
     
    Rook currently publishes builds of the Ceph operator to the release and master channels.
     
    Release
     
    The release channel is the most recent release of Rook that is considered stable for the community.
     
    1
 2
    helm repo add rook-release https://charts.rook.io/release
 helm install --create-namespace --namespace rook-ceph rook-ceph rook-release/rook-ceph -f values.yaml
-
     
    For example settings, see the next section or values.yaml
     
    Configuration
     
    The following table lists the configurable parameters of the rook-operator chart and their default values.
     
      
     Parameter Description Default 
           
     allowLoopDevices If true, loop devices are allowed to be used for osds in test clusters false 
     
     annotations Pod annotations {} 
     
     cephCommandsTimeoutSeconds The timeout for ceph commands in seconds "15" 
     
     containerSecurityContext Set the container security context for the operator {"capabilities":{"drop":["ALL"]},"runAsGroup":2016,"runAsNonRoot":true,"runAsUser":2016} 
     
     crds.enabled Whether the helm chart should create and update the CRDs. If false, the CRDs must be managed independently with deploy/examples/crds.yaml. WARNING Only set during first deployment. If later disabled the cluster may be DESTROYED. If the CRDs are deleted in this case, see the disaster recovery guide to restore them. true 
     
     csi.attacher.repository Kubernetes CSI Attacher image repository "registry.k8s.io/sig-storage/csi-attacher" 
     
     csi.attacher.tag Attacher image tag "v4.8.0" 
     
     csi.cephFSAttachRequired Whether to skip any attach operation altogether for CephFS PVCs. See more details here. If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation of pods using the CephFS PVC fast. WARNING It's highly discouraged to use this for CephFS RWO volumes. Refer to this issue for more details. true 
     
     csi.cephFSFSGroupPolicy Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html "File" 
     
     csi.cephFSKernelMountOptions Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options. Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR nil 
     
     csi.cephFSPluginUpdateStrategy CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate RollingUpdate 
     
     csi.cephFSPluginUpdateStrategyMaxUnavailable A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. 1 
     
     csi.cephcsi.repository Ceph CSI image repository "quay.io/cephcsi/cephcsi" 
     
     csi.cephcsi.tag Ceph CSI image tag "v3.13.0" 
     
     csi.cephfsLivenessMetricsPort CSI CephFS driver metrics port 9081 
     
     csi.cephfsPodLabels Labels to add to the CSI CephFS Deployments and DaemonSets Pods nil 
     
     csi.clusterName Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster nil 
     
     csi.csiAddons.enabled Enable CSIAddons false 
     
     csi.csiAddons.repository CSIAddons sidecar image repository "quay.io/csiaddons/k8s-sidecar" 
     
     csi.csiAddons.tag CSIAddons sidecar image tag "v0.11.0" 
     
     csi.csiAddonsPort CSI Addons server port 9070 
     
     csi.csiCephFSPluginResource CEPH CSI CephFS plugin resource requirement list see values.yaml 
     
     csi.csiCephFSPluginVolume The volume of the CephCSI CephFS plugin DaemonSet nil 
     
     csi.csiCephFSPluginVolumeMount The volume mounts of the CephCSI CephFS plugin DaemonSet nil 
     
     csi.csiCephFSProvisionerResource CEPH CSI CephFS provisioner resource requirement list see values.yaml 
     
     csi.csiDriverNamePrefix CSI driver name prefix for cephfs, rbd and nfs. namespace name where rook-ceph operator is deployed 
     
     csi.csiLeaderElectionLeaseDuration Duration in seconds that non-leader candidates will wait to force acquire leadership. 137s 
     
     csi.csiLeaderElectionRenewDeadline Deadline in seconds that the acting leader will retry refreshing leadership before giving up. 107s 
     
     csi.csiLeaderElectionRetryPeriod Retry period in seconds the LeaderElector clients should wait between tries of actions. 26s 
     
     csi.csiNFSPluginResource CEPH CSI NFS plugin resource requirement list see values.yaml 
     
     csi.csiNFSProvisionerResource CEPH CSI NFS provisioner resource requirement list see values.yaml 
     
     csi.csiRBDPluginResource CEPH CSI RBD plugin resource requirement list see values.yaml 
     
     csi.csiRBDPluginVolume The volume of the CephCSI RBD plugin DaemonSet nil 
     
     csi.csiRBDPluginVolumeMount The volume mounts of the CephCSI RBD plugin DaemonSet nil 
     
     csi.csiRBDProvisionerResource CEPH CSI RBD provisioner resource requirement list csi-omap-generator resources will be applied only if enableOMAPGenerator is set to true see values.yaml 
     
     csi.disableCsiDriver Disable the CSI driver. "false" 
     
     csi.enableCSIEncryption Enable Ceph CSI PVC encryption support false 
     
     csi.enableCSIHostNetwork Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary in some network configurations where the SDN does not provide access to an external cluster or there is significant drop in read/write performance true 
     
     csi.enableCephfsDriver Enable Ceph CSI CephFS driver true 
     
     csi.enableCephfsSnapshotter Enable Snapshotter in CephFS provisioner pod true 
     
     csi.enableLiveness Enable Ceph CSI Liveness sidecar deployment false 
     
     csi.enableMetadata Enable adding volume metadata on the CephFS subvolumes and RBD images. Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. Hence enable metadata is false by default false 
     
     csi.enableNFSSnapshotter Enable Snapshotter in NFS provisioner pod true 
     
     csi.enableOMAPGenerator OMAP generator generates the omap mapping between the PV name and the RBD image which helps CSI to identify the rbd images for CSI operations. CSI_ENABLE_OMAP_GENERATOR needs to be enabled when we are using rbd mirroring feature. By default OMAP generator is disabled and when enabled, it will be deployed as a sidecar with CSI provisioner pod, to enable set it to true. false 
     
     csi.enablePluginSelinuxHostMount Enable Host mount for /etc/selinux directory for Ceph CSI nodeplugins false 
     
     csi.enableRBDSnapshotter Enable Snapshotter in RBD provisioner pod true 
     
     csi.enableRbdDriver Enable Ceph CSI RBD driver true 
     
     csi.enableVolumeGroupSnapshot Enable volume group snapshot feature. This feature is enabled by default as long as the necessary CRDs are available in the cluster. true 
     
     csi.forceCephFSKernelClient Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS you may want to disable this setting. However, this will cause an issue during upgrades with the FUSE client. See the upgrade guide true 
     
     csi.grpcTimeoutInSeconds Set GRPC timeout for csi containers (in seconds). It should be >= 120. If this value is not set or is invalid, it defaults to 150 150 
     
     csi.imagePullPolicy Image pull policy "IfNotPresent" 
     
     csi.kubeApiBurst Burst to use while communicating with the kubernetes apiserver. nil 
     
     csi.kubeApiQPS QPS to use while communicating with the kubernetes apiserver. nil 
     
     csi.kubeletDirPath Kubelet root directory path (if the Kubelet uses a different path for the --root-dir flag) /var/lib/kubelet 
     
     csi.logLevel Set logging level for cephCSI containers maintained by the cephCSI. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. 0 
     
     csi.nfs.enabled Enable the nfs csi driver false 
     
     csi.nfsAttachRequired Whether to skip any attach operation altogether for NFS PVCs. See more details here. If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation of pods using the NFS PVC fast. WARNING It's highly discouraged to use this for NFS RWO volumes. Refer to this issue for more details. true 
     
     csi.nfsFSGroupPolicy Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html "File" 
     
     csi.nfsPluginUpdateStrategy CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate RollingUpdate 
     
     csi.nfsPodLabels Labels to add to the CSI NFS Deployments and DaemonSets Pods nil 
     
     csi.pluginNodeAffinity The node labels for affinity of the CephCSI RBD plugin DaemonSet 1 nil 
     
     csi.pluginPriorityClassName PriorityClassName to be set on csi driver plugin pods "system-node-critical" 
     
     csi.pluginTolerations Array of tolerations in YAML format which will be added to CephCSI plugin DaemonSet nil 
     
     csi.provisioner.repository Kubernetes CSI provisioner image repository "registry.k8s.io/sig-storage/csi-provisioner" 
     
     csi.provisioner.tag Provisioner image tag "v5.1.0" 
     
     csi.provisionerNodeAffinity The node labels for affinity of the CSI provisioner deployment 1 nil 
     
     csi.provisionerPriorityClassName PriorityClassName to be set on csi driver provisioner pods "system-cluster-critical" 
     
     csi.provisionerReplicas Set replicas for csi provisioner deployment 2 
     
     csi.provisionerTolerations Array of tolerations in YAML format which will be added to CSI provisioner deployment nil 
     
     csi.rbdAttachRequired Whether to skip any attach operation altogether for RBD PVCs. See more details here. If set to false it skips the volume attachments and makes the creation of pods using the RBD PVC fast. WARNING It's highly discouraged to use this for RWO volumes as it can cause data corruption. csi-addons operations like Reclaimspace and PVC Keyrotation will also not be supported if set to false since we'll have no VolumeAttachments to determine which node the PVC is mounted on. Refer to this issue for more details. true 
     
     csi.rbdFSGroupPolicy Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html "File" 
     
     csi.rbdLivenessMetricsPort Ceph CSI RBD driver metrics port 8080 
     
     csi.rbdPluginUpdateStrategy CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate RollingUpdate 
     
     csi.rbdPluginUpdateStrategyMaxUnavailable A maxUnavailable parameter of CSI RBD plugin daemonset update strategy. 1 
     
     csi.rbdPodLabels Labels to add to the CSI RBD Deployments and DaemonSets Pods nil 
     
     csi.registrar.repository Kubernetes CSI registrar image repository "registry.k8s.io/sig-storage/csi-node-driver-registrar" 
     
     csi.registrar.tag Registrar image tag "v2.13.0" 
     
     csi.resizer.repository Kubernetes CSI resizer image repository "registry.k8s.io/sig-storage/csi-resizer" 
     
     csi.resizer.tag Resizer image tag "v1.13.1" 
     
     csi.serviceMonitor.enabled Enable ServiceMonitor for Ceph CSI drivers false 
     
     csi.serviceMonitor.interval Service monitor scrape interval "10s" 
     
     csi.serviceMonitor.labels ServiceMonitor additional labels {} 
     
     csi.serviceMonitor.namespace Use a different namespace for the ServiceMonitor nil 
     
     csi.sidecarLogLevel Set logging level for Kubernetes-csi sidecar containers. Supported values from 0 to 5. 0 for general useful logs (the default), 5 for trace level verbosity. 0 
     
     csi.snapshotter.repository Kubernetes CSI snapshotter image repository "registry.k8s.io/sig-storage/csi-snapshotter" 
     
     csi.snapshotter.tag Snapshotter image tag "v8.2.0" 
     
     csi.topology.domainLabels domainLabels define which node labels to use as domains for CSI nodeplugins to advertise their domains nil 
     
     csi.topology.enabled Enable topology based provisioning false 
     
     currentNamespaceOnly Whether the operator should watch cluster CRD in its own namespace or not false 
     
     customHostnameLabel Custom label to identify node hostname. If not set kubernetes.io/hostname will be used nil 
     
     disableDeviceHotplug Disable automatic orchestration when new devices are discovered. false 
     
     discover.nodeAffinity The node labels for affinity of discover-agent 1 nil 
     
     discover.podLabels Labels to add to the discover pods nil 
     
     discover.resources Add resources to discover daemon pods nil 
     
     discover.toleration Toleration for the discover pods. Options: NoSchedule, PreferNoSchedule or NoExecute nil 
     
     discover.tolerationKey The specific key of the taint to tolerate nil 
     
     discover.tolerations Array of tolerations in YAML format which will be added to discover deployment nil 
     
     discoverDaemonUdev Blacklist certain disks according to the regex provided. nil 
     
     discoveryDaemonInterval Set the discovery daemon device discovery interval (default to 60m) "60m" 
     
     enableDiscoveryDaemon Enable discovery daemon false 
     
     enableOBCWatchOperatorNamespace Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used true 
     
     enforceHostNetwork Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled false 
     
     hostpathRequiresPrivileged Runs Ceph Pods as privileged to be able to write to hostPaths in OpenShift with SELinux restrictions. false 
     
     image.pullPolicy Image pull policy "IfNotPresent" 
     
     image.repository Image "docker.io/rook/ceph" 
     
     image.tag Image tag master 
     
     imagePullSecrets imagePullSecrets option allow to pull docker images from private docker registry. Option will be passed to all service accounts. nil 
     
     logLevel Global log level for the operator. Options: ERROR, WARNING, INFO, DEBUG "INFO" 
     
     monitoring.enabled Enable monitoring. Requires Prometheus to be pre-installed. Enabling will also create RBAC rules to allow Operator to create ServiceMonitors false 
     
     nodeSelector Kubernetes nodeSelector to add to the Deployment. {} 
     
     obcProvisionerNamePrefix Specify the prefix for the OBC provisioner in place of the cluster namespace ceph cluster namespace 
     
     operatorPodLabels Custom pod labels for the operator {} 
     
     priorityClassName Set the priority class for the rook operator deployment if desired nil 
     
     pspEnable If true, create & use PSP resources false 
     
     rbacAggregate.enableOBCs If true, create a ClusterRole aggregated to user facing roles for objectbucketclaims false 
     
     rbacEnable If true, create & use RBAC resources true 
     
     resources Pod resource requests & limits {"limits":{"memory":"512Mi"},"requests":{"cpu":"200m","memory":"128Mi"}} 
     
     revisionHistoryLimit The revision history limit for all pods created by Rook. If blank, the K8s default is 10. nil 
     
     scaleDownOperator If true, scale down the rook operator. This is useful for administrative actions where the rook operator must be scaled down, while using gitops style tooling to deploy your helm charts. false 
     
     tolerations List of Kubernetes tolerations to add to the Deployment. [] 
     
     unreachableNodeTolerationSeconds Delay to use for the node.kubernetes.io/unreachable pod failure toleration to override the Kubernetes default of 5 minutes 5 
     
     useOperatorHostNetwork If true, run rook operator on the host network nil 
          
     
    Development Build
     
    To deploy from a local build from your development environment:
     
       
    1. Build the Rook docker image: make
      2.  
    2. Copy the image to your K8s cluster, such as with the docker save then the docker load commands
      3.  
    3. Install the helm chart:
      4.  
     
    1
+
     
    For example settings, see the next section or values.yaml
     
    Configuration
     
    The following table lists the configurable parameters of the rook-operator chart and their default values.
     
      
     Parameter Description Default 
           
     allowLoopDevices If true, loop devices are allowed to be used for osds in test clusters false 
     
     annotations Pod annotations {} 
     
     cephCommandsTimeoutSeconds The timeout for ceph commands in seconds "15" 
     
     containerSecurityContext Set the container security context for the operator {"capabilities":{"drop":["ALL"]},"runAsGroup":2016,"runAsNonRoot":true,"runAsUser":2016} 
     
     crds.enabled Whether the helm chart should create and update the CRDs. If false, the CRDs must be managed independently with deploy/examples/crds.yaml. WARNING Only set during first deployment. If later disabled the cluster may be DESTROYED. If the CRDs are deleted in this case, see the disaster recovery guide to restore them. true 
     
     csi.attacher.repository Kubernetes CSI Attacher image repository "registry.k8s.io/sig-storage/csi-attacher" 
     
     csi.attacher.tag Attacher image tag "v4.8.0" 
     
     csi.cephFSAttachRequired Whether to skip any attach operation altogether for CephFS PVCs. See more details here. If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation of pods using the CephFS PVC fast. WARNING It's highly discouraged to use this for CephFS RWO volumes. Refer to this issue for more details. true 
     
     csi.cephFSFSGroupPolicy Policy for modifying a volume's ownership or permissions when the CephFS PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html "File" 
     
     csi.cephFSKernelMountOptions Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options. Set to "ms_mode=secure" when connections.encrypted is enabled in CephCluster CR nil 
     
     csi.cephFSPluginUpdateStrategy CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate RollingUpdate 
     
     csi.cephFSPluginUpdateStrategyMaxUnavailable A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. 1 
     
     csi.cephcsi.repository Ceph CSI image repository "quay.io/cephcsi/cephcsi" 
     
     csi.cephcsi.tag Ceph CSI image tag "v3.13.0" 
     
     csi.cephfsLivenessMetricsPort CSI CephFS driver metrics port 9081 
     
     csi.cephfsPodLabels Labels to add to the CSI CephFS Deployments and DaemonSets Pods nil 
     
     csi.clusterName Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster nil 
     
     csi.csiAddons.enabled Enable CSIAddons false 
     
     csi.csiAddons.repository CSIAddons sidecar image repository "quay.io/csiaddons/k8s-sidecar" 
     
     csi.csiAddons.tag CSIAddons sidecar image tag "v0.11.0" 
     
     csi.csiAddonsPort CSI Addons server port 9070 
     
     csi.csiCephFSPluginResource CEPH CSI CephFS plugin resource requirement list see values.yaml 
     
     csi.csiCephFSPluginVolume The volume of the CephCSI CephFS plugin DaemonSet nil 
     
     csi.csiCephFSPluginVolumeMount The volume mounts of the CephCSI CephFS plugin DaemonSet nil 
     
     csi.csiCephFSProvisionerResource CEPH CSI CephFS provisioner resource requirement list see values.yaml 
     
     csi.csiDriverNamePrefix CSI driver name prefix for cephfs, rbd and nfs. namespace name where rook-ceph operator is deployed 
     
     csi.csiLeaderElectionLeaseDuration Duration in seconds that non-leader candidates will wait to force acquire leadership. 137s 
     
     csi.csiLeaderElectionRenewDeadline Deadline in seconds that the acting leader will retry refreshing leadership before giving up. 107s 
     
     csi.csiLeaderElectionRetryPeriod Retry period in seconds the LeaderElector clients should wait between tries of actions. 26s 
     
     csi.csiNFSPluginResource CEPH CSI NFS plugin resource requirement list see values.yaml 
     
     csi.csiNFSProvisionerResource CEPH CSI NFS provisioner resource requirement list see values.yaml 
     
     csi.csiRBDPluginResource CEPH CSI RBD plugin resource requirement list see values.yaml 
     
     csi.csiRBDPluginVolume The volume of the CephCSI RBD plugin DaemonSet nil 
     
     csi.csiRBDPluginVolumeMount The volume mounts of the CephCSI RBD plugin DaemonSet nil 
     
     csi.csiRBDProvisionerResource CEPH CSI RBD provisioner resource requirement list csi-omap-generator resources will be applied only if enableOMAPGenerator is set to true see values.yaml 
     
     csi.disableCsiDriver Disable the CSI driver. "false" 
     
     csi.enableCSIEncryption Enable Ceph CSI PVC encryption support false 
     
     csi.enableCSIHostNetwork Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary in some network configurations where the SDN does not provide access to an external cluster or there is significant drop in read/write performance true 
     
     csi.enableCephfsDriver Enable Ceph CSI CephFS driver true 
     
     csi.enableCephfsSnapshotter Enable Snapshotter in CephFS provisioner pod true 
     
     csi.enableLiveness Enable Ceph CSI Liveness sidecar deployment false 
     
     csi.enableMetadata Enable adding volume metadata on the CephFS subvolumes and RBD images. Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. Hence enable metadata is false by default false 
     
     csi.enableNFSSnapshotter Enable Snapshotter in NFS provisioner pod true 
     
     csi.enableOMAPGenerator OMAP generator generates the omap mapping between the PV name and the RBD image which helps CSI to identify the rbd images for CSI operations. CSI_ENABLE_OMAP_GENERATOR needs to be enabled when we are using rbd mirroring feature. By default OMAP generator is disabled and when enabled, it will be deployed as a sidecar with CSI provisioner pod, to enable set it to true. false 
     
     csi.enablePluginSelinuxHostMount Enable Host mount for /etc/selinux directory for Ceph CSI nodeplugins false 
     
     csi.enableRBDSnapshotter Enable Snapshotter in RBD provisioner pod true 
     
     csi.enableRbdDriver Enable Ceph CSI RBD driver true 
     
     csi.enableVolumeGroupSnapshot Enable volume group snapshot feature. This feature is enabled by default as long as the necessary CRDs are available in the cluster. true 
     
     csi.forceCephFSKernelClient Enable Ceph Kernel clients on kernel < 4.17. If your kernel does not support quotas for CephFS you may want to disable this setting. However, this will cause an issue during upgrades with the FUSE client. See the upgrade guide true 
     
     csi.grpcTimeoutInSeconds Set GRPC timeout for csi containers (in seconds). It should be >= 120. If this value is not set or is invalid, it defaults to 150 150 
     
     csi.imagePullPolicy Image pull policy "IfNotPresent" 
     
     csi.kubeApiBurst Burst to use while communicating with the kubernetes apiserver. nil 
     
     csi.kubeApiQPS QPS to use while communicating with the kubernetes apiserver. nil 
     
     csi.kubeletDirPath Kubelet root directory path (if the Kubelet uses a different path for the --root-dir flag) /var/lib/kubelet 
     
     csi.logLevel Set logging level for cephCSI containers maintained by the cephCSI. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity. 0 
     
     csi.nfs.enabled Enable the nfs csi driver false 
     
     csi.nfsAttachRequired Whether to skip any attach operation altogether for NFS PVCs. See more details here. If cephFSAttachRequired is set to false it skips the volume attachments and makes the creation of pods using the NFS PVC fast. WARNING It's highly discouraged to use this for NFS RWO volumes. Refer to this issue for more details. true 
     
     csi.nfsFSGroupPolicy Policy for modifying a volume's ownership or permissions when the NFS PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html "File" 
     
     csi.nfsPluginUpdateStrategy CSI NFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate RollingUpdate 
     
     csi.nfsPodLabels Labels to add to the CSI NFS Deployments and DaemonSets Pods nil 
     
     csi.pluginNodeAffinity The node labels for affinity of the CephCSI RBD plugin DaemonSet 1 nil 
     
     csi.pluginPriorityClassName PriorityClassName to be set on csi driver plugin pods "system-node-critical" 
     
     csi.pluginTolerations Array of tolerations in YAML format which will be added to CephCSI plugin DaemonSet nil 
     
     csi.provisioner.repository Kubernetes CSI provisioner image repository "registry.k8s.io/sig-storage/csi-provisioner" 
     
     csi.provisioner.tag Provisioner image tag "v5.1.0" 
     
     csi.provisionerNodeAffinity The node labels for affinity of the CSI provisioner deployment 1 nil 
     
     csi.provisionerPriorityClassName PriorityClassName to be set on csi driver provisioner pods "system-cluster-critical" 
     
     csi.provisionerReplicas Set replicas for csi provisioner deployment 2 
     
     csi.provisionerTolerations Array of tolerations in YAML format which will be added to CSI provisioner deployment nil 
     
     csi.rbdAttachRequired Whether to skip any attach operation altogether for RBD PVCs. See more details here. If set to false it skips the volume attachments and makes the creation of pods using the RBD PVC fast. WARNING It's highly discouraged to use this for RWO volumes as it can cause data corruption. csi-addons operations like Reclaimspace and PVC Keyrotation will also not be supported if set to false since we'll have no VolumeAttachments to determine which node the PVC is mounted on. Refer to this issue for more details. true 
     
     csi.rbdFSGroupPolicy Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html "File" 
     
     csi.rbdLivenessMetricsPort Ceph CSI RBD driver metrics port 8080 
     
     csi.rbdPluginUpdateStrategy CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate RollingUpdate 
     
     csi.rbdPluginUpdateStrategyMaxUnavailable A maxUnavailable parameter of CSI RBD plugin daemonset update strategy. 1 
     
     csi.rbdPodLabels Labels to add to the CSI RBD Deployments and DaemonSets Pods nil 
     
     csi.registrar.repository Kubernetes CSI registrar image repository "registry.k8s.io/sig-storage/csi-node-driver-registrar" 
     
     csi.registrar.tag Registrar image tag "v2.13.0" 
     
     csi.resizer.repository Kubernetes CSI resizer image repository "registry.k8s.io/sig-storage/csi-resizer" 
     
     csi.resizer.tag Resizer image tag "v1.13.1" 
     
     csi.serviceMonitor.enabled Enable ServiceMonitor for Ceph CSI drivers false 
     
     csi.serviceMonitor.interval Service monitor scrape interval "10s" 
     
     csi.serviceMonitor.labels ServiceMonitor additional labels {} 
     
     csi.serviceMonitor.namespace Use a different namespace for the ServiceMonitor nil 
     
     csi.sidecarLogLevel Set logging level for Kubernetes-csi sidecar containers. Supported values from 0 to 5. 0 for general useful logs (the default), 5 for trace level verbosity. 0 
     
     csi.snapshotter.repository Kubernetes CSI snapshotter image repository "registry.k8s.io/sig-storage/csi-snapshotter" 
     
     csi.snapshotter.tag Snapshotter image tag "v8.2.0" 
     
     csi.topology.domainLabels domainLabels define which node labels to use as domains for CSI nodeplugins to advertise their domains nil 
     
     csi.topology.enabled Enable topology based provisioning false 
     
     currentNamespaceOnly Whether the operator should watch cluster CRD in its own namespace or not false 
     
     customHostnameLabel Custom label to identify node hostname. If not set kubernetes.io/hostname will be used nil 
     
     disableDeviceHotplug Disable automatic orchestration when new devices are discovered. false 
     
     discover.nodeAffinity The node labels for affinity of discover-agent 1 nil 
     
     discover.podLabels Labels to add to the discover pods nil 
     
     discover.resources Add resources to discover daemon pods nil 
     
     discover.toleration Toleration for the discover pods. Options: NoSchedule, PreferNoSchedule or NoExecute nil 
     
     discover.tolerationKey The specific key of the taint to tolerate nil 
     
     discover.tolerations Array of tolerations in YAML format which will be added to discover deployment nil 
     
     discoverDaemonUdev Blacklist certain disks according to the regex provided. nil 
     
     discoveryDaemonInterval Set the discovery daemon device discovery interval (default to 60m) "60m" 
     
     enableDiscoveryDaemon Enable discovery daemon false 
     
     enableOBCWatchOperatorNamespace Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used true 
     
     enforceHostNetwork Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled false 
     
     hostpathRequiresPrivileged Runs Ceph Pods as privileged to be able to write to hostPaths in OpenShift with SELinux restrictions. false 
     
     image.pullPolicy Image pull policy "IfNotPresent" 
     
     image.repository Image "docker.io/rook/ceph" 
     
     image.tag Image tag master 
     
     imagePullSecrets imagePullSecrets option allow to pull docker images from private docker registry. Option will be passed to all service accounts. nil 
     
     logLevel Global log level for the operator. Options: ERROR, WARNING, INFO, DEBUG "INFO" 
     
     monitoring.enabled Enable monitoring. Requires Prometheus to be pre-installed. Enabling will also create RBAC rules to allow Operator to create ServiceMonitors false 
     
     nodeSelector Kubernetes nodeSelector to add to the Deployment. {} 
     
     obcProvisionerNamePrefix Specify the prefix for the OBC provisioner in place of the cluster namespace ceph cluster namespace 
     
     operatorPodLabels Custom pod labels for the operator {} 
     
     priorityClassName Set the priority class for the rook operator deployment if desired nil 
     
     pspEnable If true, create & use PSP resources false 
     
     rbacAggregate.enableOBCs If true, create a ClusterRole aggregated to user facing roles for objectbucketclaims false 
     
     rbacEnable If true, create & use RBAC resources true 
     
     resources Pod resource requests & limits {"limits":{"memory":"512Mi"},"requests":{"cpu":"200m","memory":"128Mi"}} 
     
     revisionHistoryLimit The revision history limit for all pods created by Rook. If blank, the K8s default is 10. nil 
     
     scaleDownOperator If true, scale down the rook operator. This is useful for administrative actions where the rook operator must be scaled down, while using gitops style tooling to deploy your helm charts. false 
     
     tolerations List of Kubernetes tolerations to add to the Deployment. [] 
     
     unreachableNodeTolerationSeconds Delay to use for the node.kubernetes.io/unreachable pod failure toleration to override the Kubernetes default of 5 minutes 5 
     
     useOperatorHostNetwork If true, run rook operator on the host network nil 
          
     
    Development Build
     
    To deploy from a local build from your development environment:
     
       
    1. Build the Rook docker image: make
      2.  
    2. Copy the image to your K8s cluster, such as with the docker save then the docker load commands
      3.  
    3. Install the helm chart:
      4.  
     
    1
 2
    cd deploy/charts/rook-ceph
 helm install --create-namespace --namespace rook-ceph rook-ceph .
 
     
    Uninstalling the Chart
     
    To see the currently installed Rook chart:
     
    1
    helm ls --namespace rook-ceph
diff --git a/docs/rook/v1.16/Storage-Configuration/Advanced/ceph-configuration/index.html b/docs/rook/v1.16/Storage-Configuration/Advanced/ceph-configuration/index.html
index ca6c18af5..18772c493 100644
--- a/docs/rook/v1.16/Storage-Configuration/Advanced/ceph-configuration/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Advanced/ceph-configuration/index.html
@@ -193,7 +193,7 @@
     [global]
     osd crush update on start = false
     osd pool default size = 2
-
     
    Custom CSI ceph.conf Settings
     
     
    Warning
     
    It is highly recommended to use the default setting that comes with CephCSI and this can only be used when absolutely necessary. The ceph.conf should be reset back to default values if/when the configurations are no longer necessary.
     
     
    If the csi-ceph-conf-override ConfigMap is created before the cluster is started, the CephCSI pods will automatically pick up the settings. If you add the settings to the ConfigMap after the cluster has been initialized, you can restart the Rook operator pod and wait for Rook to recreate CSI pods to take immediate effect.
     
    After the CSI pods are restarted, the new settings should be in effect.
     
    Example CSI ceph.conf Settings
     
    In this Example we will set the rbd_validate_pool to false to skip rbd pool validation.
     
     
    Warning
     
    Modify Ceph settings carefully to avoid modifying the default configuration. Changing the settings could result in unexpected results if used incorrectly.
     
     
    1
    kubectl create -f csi-ceph-conf-override.yaml
+
     
    Custom CSI ceph.conf Settings
     
     
    Warning
     
    It is highly recommended to use the default setting that comes with CephCSI and this can only be used when absolutely necessary. The ceph.conf should be reset back to default values if/when the configurations are no longer necessary.
     
     
    If the csi-ceph-conf-override ConfigMap is created before the cluster is started, the CephCSI pods will automatically pick up the settings. If you add the settings to the ConfigMap after the cluster has been initialized, you can restart the Rook operator pod and wait for Rook to recreate CSI pods to take immediate effect.
     
    After the CSI pods are restarted, the new settings should be in effect.
     
    Example CSI ceph.conf Settings
     
    In this Example we will set the rbd_validate_pool to false to skip rbd pool validation.
     
     
    Warning
     
    Modify Ceph settings carefully to avoid modifying the default configuration. Changing the settings could result in unexpected results if used incorrectly.
     
     
    1
    kubectl create -f csi-ceph-conf-override.yaml
 
     
    Restart the Rook operator pod and wait for CSI pods to be recreated.
     
    OSD CRUSH Settings
     
    A useful view of the CRUSH Map is generated with the following command:
     
    1
    ceph osd tree
 
     
    In this section we will be tweaking some of the values seen in the output.
     
    OSD Weight
     
    The CRUSH weight controls the ratio of data that should be distributed to each OSD. This also means a higher or lower amount of disk I/O operations for an OSD with higher/lower weight, respectively.
     
    By default OSDs get a weight relative to their storage capacity, which maximizes overall cluster capacity by filling all drives at the same rate, even if drive sizes vary. This should work for most use-cases, but the following situations could warrant weight changes:
     
       
    • Your cluster has some relatively slow OSDs or nodes. Lowering their weight can reduce the impact of this bottleneck.
      •  
    • You're using bluestore drives provisioned with Rook v0.3.1 or older. In this case you may notice OSD weights did not get set relative to their storage capacity. Changing the weight can fix this and maximize cluster capacity.
      •  
     
    This example sets the weight of osd.0 which is 600GiB
     
    1
    ceph osd crush reweight osd.0 .600
 
     
    OSD Primary Affinity
     
    When pools are set with a size setting greater than one, data is replicated between nodes and OSDs. For every chunk of data a Primary OSD is selected to be used for reading that data to be sent to clients. You can control how likely it is for an OSD to become a Primary using the Primary Affinity setting. This is similar to the OSD weight setting, except it only affects reads on the storage device, not capacity or writes.
     
    In this example we will ensure that osd.0 is only selected as Primary if all other OSDs holding data replicas are unavailable:
     
    1
    ceph osd primary-affinity osd.0 0
diff --git a/docs/rook/v1.16/Storage-Configuration/Advanced/ceph-osd-mgmt/index.html b/docs/rook/v1.16/Storage-Configuration/Advanced/ceph-osd-mgmt/index.html
index 3d6cf3df0..81e86526e 100644
--- a/docs/rook/v1.16/Storage-Configuration/Advanced/ceph-osd-mgmt/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Advanced/ceph-osd-mgmt/index.html
@@ -32,9 +32,9 @@
 # 2022-09-14 08:58:28.899567 I | cephclient: generated admin config in /var/lib/rook/rook-ceph
 # 2022-09-14 08:58:29.421345 I | cephosd: validating status of osd.0
 ---
-
     
    Purge the OSD with a Job
     
    OSD removal can be automated with the example found in the rook-ceph-purge-osd job. In the osd-purge.yaml, change the <OSD-IDs> to the ID(s) of the OSDs you want to remove.
     
       
    1. Run the job: kubectl create -f osd-purge.yaml
      2.  
    2. When the job is completed, review the logs to ensure success: kubectl -n rook-ceph logs -l app=rook-ceph-purge-osd
      3.  
    3. When finished, you can delete the job: kubectl delete -f osd-purge.yaml
      4.  
     
    If you want to remove OSDs by hand, continue with the following sections. However, we recommend you use the above-mentioned steps to avoid operation errors.
     
    Purge the OSD manually
     
    If the OSD purge job fails or you need fine-grained control of the removal, here are the individual commands that can be run from the toolbox.
     
       
    1. Detach the OSD PVC from Rook
         
      • kubectl -n rook-ceph label pvc <orphaned-pvc> ceph.rook.io/DeviceSetPVCId-
        •  
       
      2.  
    2. Mark the OSD as out if not already marked as such by Ceph. This signals Ceph to start moving (backfilling) the data that was on that OSD to another OSD.
         
      • ceph osd out osd.<ID> (for example if the OSD ID is 23 this would be ceph osd out osd.23)
        •  
       
      3.  
    3. Wait for the data to finish backfilling to other OSDs.
         
      • ceph status will indicate the backfilling is done when all of the PGs are active+clean. If desired, it's safe to remove the disk after that.
        •  
       
      4.  
    4. Remove the OSD from the Ceph cluster
         
      • ceph osd purge <ID> --yes-i-really-mean-it
        •  
       
      5.  
    5. Verify the OSD is removed from the node in the CRUSH map
         
      • ceph osd tree
        •  
       
      6.  
     
    The operator can automatically remove OSD deployments that are considered "safe-to-destroy" by Ceph. After the steps above, the OSD will be considered safe to remove since the data has all been moved to other OSDs. But this will only be done automatically by the operator if you have this setting in the cluster CR:
     
    1
    removeOSDsIfOutAndSafeToRemove: true
+
     
    Purge the OSD with a Job
     
    OSD removal can be automated with the example found in the rook-ceph-purge-osd job. In the osd-purge.yaml, change the <OSD-IDs> to the ID(s) of the OSDs you want to remove.
     
       
    1. Run the job: kubectl create -f osd-purge.yaml
      2.  
    2. When the job is completed, review the logs to ensure success: kubectl -n rook-ceph logs -l app=rook-ceph-purge-osd
      3.  
    3. When finished, you can delete the job: kubectl delete -f osd-purge.yaml
      4.  
     
    If you want to remove OSDs by hand, continue with the following sections. However, we recommend you use the above-mentioned steps to avoid operation errors.
     
    Purge the OSD manually
     
    If the OSD purge job fails or you need fine-grained control of the removal, here are the individual commands that can be run from the toolbox.
     
       
    1. Detach the OSD PVC from Rook
         
      • kubectl -n rook-ceph label pvc <orphaned-pvc> ceph.rook.io/DeviceSetPVCId-
        •  
       
      2.  
    2. Mark the OSD as out if not already marked as such by Ceph. This signals Ceph to start moving (backfilling) the data that was on that OSD to another OSD.
         
      • ceph osd out osd.<ID> (for example if the OSD ID is 23 this would be ceph osd out osd.23)
        •  
       
      3.  
    3. Wait for the data to finish backfilling to other OSDs.
         
      • ceph status will indicate the backfilling is done when all of the PGs are active+clean. If desired, it's safe to remove the disk after that.
        •  
       
      4.  
    4. Remove the OSD from the Ceph cluster
         
      • ceph osd purge <ID> --yes-i-really-mean-it
        •  
       
      5.  
    5. Verify the OSD is removed from the node in the CRUSH map
         
      • ceph osd tree
        •  
       
      6.  
     
    The operator can automatically remove OSD deployments that are considered "safe-to-destroy" by Ceph. After the steps above, the OSD will be considered safe to remove since the data has all been moved to other OSDs. But this will only be done automatically by the operator if you have this setting in the cluster CR:
     
    1
    removeOSDsIfOutAndSafeToRemove: true
 
     
    Otherwise, you will need to delete the deployment directly:
     
    1
    kubectl delete deployment -n rook-ceph rook-ceph-osd-<ID>
-
     
    In PVC-based cluster, remove the orphaned PVC, if necessary.
     
    Delete the underlying data
     
    If you want to clean the device where the OSD was running, see in the instructions to wipe a disk on the Cleaning up a Cluster topic.
     
    Replace an OSD
     
    To replace a disk that has failed:
     
       
    1. Run the steps in the previous section to Remove an OSD.
      2.  
    2. Replace the physical device and verify the new device is attached.
      3.  
    3. Check if your cluster CR will find the new device. If you are using useAllDevices: true you can skip this step. If your cluster CR lists individual devices or uses a device filter you may need to update the CR.
      4.  
    4. The operator ideally will automatically create the new OSD within a few minutes of adding the new device or updating the CR. If you don't see a new OSD automatically created, restart the operator (by deleting the operator pod) to trigger the OSD creation.
      5.  
    5. Verify if the OSD is created on the node by running ceph osd tree from the toolbox.
      6.  
     
     
    Note
     
    The OSD might have a different ID than the previous OSD that was replaced.
     
     
    OSD Migration
     
    Ceph does not support changing certain settings on existing OSDs. To support changing these settings on an OSD, the OSD must be destroyed and re-created with the new settings. Rook will automate this by migrating only one OSD at a time. The operator waits for the data to rebalance (PGs to become active+clean) before migrating the next OSD. This ensures that there is no data loss. Refer to the OSD migration design doc for more information. 
     
    The following scenarios are supported for OSD migration:
     
       
    • Enable or disable OSD encryption for existing PVC-based OSDs by changing the encrypted setting under the storageClassDeviceSets
      •  
     
    For example:
     
    1
+
     
    In PVC-based cluster, remove the orphaned PVC, if necessary.
     
    Delete the underlying data
     
    If you want to clean the device where the OSD was running, see in the instructions to wipe a disk on the Cleaning up a Cluster topic.
     
    Replace an OSD
     
    To replace a disk that has failed:
     
       
    1. Run the steps in the previous section to Remove an OSD.
      2.  
    2. Replace the physical device and verify the new device is attached.
      3.  
    3. Check if your cluster CR will find the new device. If you are using useAllDevices: true you can skip this step. If your cluster CR lists individual devices or uses a device filter you may need to update the CR.
      4.  
    4. The operator ideally will automatically create the new OSD within a few minutes of adding the new device or updating the CR. If you don't see a new OSD automatically created, restart the operator (by deleting the operator pod) to trigger the OSD creation.
      5.  
    5. Verify if the OSD is created on the node by running ceph osd tree from the toolbox.
      6.  
     
     
    Note
     
    The OSD might have a different ID than the previous OSD that was replaced.
     
     
    OSD Migration
     
    Ceph does not support changing certain settings on existing OSDs. To support changing these settings on an OSD, the OSD must be destroyed and re-created with the new settings. Rook will automate this by migrating only one OSD at a time. The operator waits for the data to rebalance (PGs to become active+clean) before migrating the next OSD. This ensures that there is no data loss. Refer to the OSD migration design doc for more information. 
     
    The following scenarios are supported for OSD migration:
     
       
    • Enable or disable OSD encryption for existing PVC-based OSDs by changing the encrypted setting under the storageClassDeviceSets
      •  
     
    For example:
     
    1
 2
 3
 4
diff --git a/docs/rook/v1.16/Storage-Configuration/Block-Storage-RBD/block-storage/index.html b/docs/rook/v1.16/Storage-Configuration/Block-Storage-RBD/block-storage/index.html
index 2c86e5be1..34d8fe2c8 100644
--- a/docs/rook/v1.16/Storage-Configuration/Block-Storage-RBD/block-storage/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Block-Storage-RBD/block-storage/index.html
@@ -161,7 +161,7 @@
 kubectl delete -f mysql.yaml
 kubectl delete -n rook-ceph cephblockpools.ceph.rook.io replicapool
 kubectl delete storageclass rook-ceph-block
-
     
    Advanced Example: Erasure Coded Block Storage
     
    If you want to use erasure coded pool with RBD, your OSDs must use bluestore as their storeType. Additionally the nodes that are going to mount the erasure coded RBD block storage must have Linux kernel >= 4.11.
     
     
    Attention
     
    This example requires at least 3 bluestore OSDs, with each OSD located on a different node.
     
     
    The OSDs must be located on different nodes, because the failureDomain is set to host and the erasureCoded chunk settings require at least 3 different OSDs (2 dataChunks + 1 codingChunks).
     
    To be able to use an erasure coded pool you need to create two pools (as seen below in the definitions): one erasure coded and one replicated.
     
     
    Attention
     
    This example requires at least 3 bluestore OSDs, with each OSD located on a different node.
     
     
    The OSDs must be located on different nodes, because the failureDomain is set to host and the erasureCoded chunk settings require at least 3 different OSDs (2 dataChunks + 1 codingChunks).
     
    Erasure Coded CSI Driver
     
    The erasure coded pool must be set as the dataPool parameter in storageclass-ec.yaml It is used for the data of the RBD images.
     
    Node Loss
     
    If a node goes down where a pod is running where a RBD RWO volume is mounted, the volume cannot automatically be mounted on another node. The node must be guaranteed to be offline before the volume can be mounted on another node.
     
    Configure CSI-Addons
     
    Deploy csi-addons controller and enable csi-addons sidecar as mentioned in the CSI Addons guide.
     
    Handling Node Loss
     
     
    Warning
     
    Automated node loss handling is currently disabled, please refer to the manual steps to recover from the node loss. We are actively working on a new design for this feature. For more details see the tracking issue.
     
     
    When a node is confirmed to be down, add the following taints to the node:
     
    1
+
     
    Advanced Example: Erasure Coded Block Storage
     
    If you want to use erasure coded pool with RBD, your OSDs must use bluestore as their storeType. Additionally the nodes that are going to mount the erasure coded RBD block storage must have Linux kernel >= 4.11.
     
     
    Attention
     
    This example requires at least 3 bluestore OSDs, with each OSD located on a different node.
     
     
    The OSDs must be located on different nodes, because the failureDomain is set to host and the erasureCoded chunk settings require at least 3 different OSDs (2 dataChunks + 1 codingChunks).
     
    To be able to use an erasure coded pool you need to create two pools (as seen below in the definitions): one erasure coded and one replicated.
     
     
    Attention
     
    This example requires at least 3 bluestore OSDs, with each OSD located on a different node.
     
     
    The OSDs must be located on different nodes, because the failureDomain is set to host and the erasureCoded chunk settings require at least 3 different OSDs (2 dataChunks + 1 codingChunks).
     
    Erasure Coded CSI Driver
     
    The erasure coded pool must be set as the dataPool parameter in storageclass-ec.yaml It is used for the data of the RBD images.
     
    Node Loss
     
    If a node goes down where a pod is running where a RBD RWO volume is mounted, the volume cannot automatically be mounted on another node. The node must be guaranteed to be offline before the volume can be mounted on another node.
     
    Configure CSI-Addons
     
    Deploy csi-addons controller and enable csi-addons sidecar as mentioned in the CSI Addons guide.
     
    Handling Node Loss
     
     
    Warning
     
    Automated node loss handling is currently disabled, please refer to the manual steps to recover from the node loss. We are actively working on a new design for this feature. For more details see the tracking issue.
     
     
    When a node is confirmed to be down, add the following taints to the node:
     
    1
 2
    kubectl taint nodes <node-name> node.kubernetes.io/out-of-service=nodeshutdown:NoExecute
 kubectl taint nodes <node-name> node.kubernetes.io/out-of-service=nodeshutdown:NoSchedule
 
     
    After the taint is added to the node, Rook will automatically blocklist the node to prevent connections to Ceph from the RBD volume on that node. To verify a node is blocklisted:
     
    1
diff --git a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/index.html b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/index.html
index ccbed3d65..87529b606 100644
--- a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/index.html
@@ -46,7 +46,7 @@
             resources:
               requests:
                 storage: 1Gi
-
     
    A volume claim template is defined inside the pod spec, and defines a volume to be provisioned and used by the pod within its lifecycle. Volumes are provisioned when a pod is spawned and destroyed when the pod is deleted.
     
    Refer to the ephemeral-doc for more info. See example manifests for an RBD ephemeral volume and a CephFS ephemeral volume.
     
    CSI-Addons Controller
     
    The CSI-Addons Controller handles requests from users. Users create a CR that the controller inspects and forwards to one or more CSI-Addons sidecars for execution.
     
    Deploying the controller
     
    Deploy the controller by running the following commands:
     
    1
+
     
    A volume claim template is defined inside the pod spec, and defines a volume to be provisioned and used by the pod within its lifecycle. Volumes are provisioned when a pod is spawned and destroyed when the pod is deleted.
     
    Refer to the ephemeral-doc for more info. See example manifests for an RBD ephemeral volume and a CephFS ephemeral volume.
     
    CSI-Addons Controller
     
    The CSI-Addons Controller handles requests from users. Users create a CR that the controller inspects and forwards to one or more CSI-Addons sidecars for execution.
     
    Deploying the controller
     
    Deploy the controller by running the following commands:
     
    1
 2
 3
    kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/v0.11.0/crds.yaml
 kubectl create -f https://github.com/csi-addons/kubernetes-csi-addons/releases/download/v0.11.0/rbac.yaml
@@ -91,7 +91,7 @@
   namespace: rook-ceph
 stringData:
   encryptionPassphrase: test-encryption
-
      
    1
+
      
    1
 2
 3
 4
diff --git a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot/index.html b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot/index.html
index ec1992c16..6c0f3d12c 100644
--- a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot/index.html
@@ -1,5 +1,5 @@
- Snapshots - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Snapshots
     
    Prerequisites
      
     
    Info
     
    Just like StorageClass provides a way for administrators to describe the "classes" of storage they offer when provisioning a volume, VolumeSnapshotClass provides a way to describe the "classes" of storage when provisioning a volume snapshot.
     
     
    RBD Snapshots
     
    RBD VolumeSnapshotClass
     
    In VolumeSnapshotClass, the csi.storage.k8s.io/snapshotter-secret-name parameter should reference the name of the secret created for the rbdplugin and pool to reflect the Ceph pool name.
     
    Update the value of the clusterID field to match the namespace that Rook is running in. When Ceph CSI is deployed by Rook, the operator will automatically maintain a configmap whose contents will match this key. By default this is "rook-ceph".
     
    1
    kubectl create -f deploy/examples/csi/rbd/snapshotclass.yaml
-
     
    Volumesnapshot
     
    In snapshot, volumeSnapshotClassName should be the name of the VolumeSnapshotClass previously created. The persistentVolumeClaimName should be the name of the PVC which is already created by the RBD CSI driver.
     
    1
    kubectl create -f deploy/examples/csi/rbd/snapshot.yaml
+ Snapshots - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Snapshots
     
    Prerequisites
      
     
    Info
     
    Just like StorageClass provides a way for administrators to describe the "classes" of storage they offer when provisioning a volume, VolumeSnapshotClass provides a way to describe the "classes" of storage when provisioning a volume snapshot.
     
     
    RBD Snapshots
     
    RBD VolumeSnapshotClass
     
    In VolumeSnapshotClass, the csi.storage.k8s.io/snapshotter-secret-name parameter should reference the name of the secret created for the rbdplugin and pool to reflect the Ceph pool name.
     
    Update the value of the clusterID field to match the namespace that Rook is running in. When Ceph CSI is deployed by Rook, the operator will automatically maintain a configmap whose contents will match this key. By default this is "rook-ceph".
     
    1
    kubectl create -f deploy/examples/csi/rbd/snapshotclass.yaml
+
     
    Volumesnapshot
     
    In snapshot, volumeSnapshotClassName should be the name of the VolumeSnapshotClass previously created. The persistentVolumeClaimName should be the name of the PVC which is already created by the RBD CSI driver.
     
    1
    kubectl create -f deploy/examples/csi/rbd/snapshot.yaml
 
     
    Verify RBD Snapshot Creation
     
    1
 2
 3
    $ kubectl get volumesnapshotclass
@@ -10,7 +10,7 @@
 3
    $ kubectl get volumesnapshot
 NAME               READYTOUSE   SOURCEPVC   SOURCESNAPSHOTCONTENT   RESTORESIZE   SNAPSHOTCLASS             SNAPSHOTCONTENT                                    CREATIONTIME   AGE
 rbd-pvc-snapshot   true         rbd-pvc                             1Gi           csi-rbdplugin-snapclass   snapcontent-79090db0-7c66-4b18-bf4a-634772c7cac7   3h50m          3h51m
-
     
    The snapshot will be ready to restore to a new PVC when the READYTOUSE field of the volumesnapshot is set to true.
     
    Restore the RBD snapshot to a new PVC
     
    In pvc-restore, dataSource should be the name of the VolumeSnapshot previously created. The dataSource kind should be the VolumeSnapshot. The storageClassName can be any RBD storageclass.
     
    Please Note: * provisioner must be the same for both the Parent PVC and the restored PVC. * The non-encrypted PVC cannot be restored to an encrypted one and vice-versa. * encrypted -> encrypted (possible) * non-encrypted -> non-encrypted (possible) * encrypted -> non-encrypted (not possible) * non-encrypted -> encrypted (not possible)
     
    Create a new PVC from the snapshot
     
    1
    kubectl create -f deploy/examples/csi/rbd/pvc-restore.yaml
+
     
    The snapshot will be ready to restore to a new PVC when the READYTOUSE field of the volumesnapshot is set to true.
     
    Restore the RBD snapshot to a new PVC
     
    In pvc-restore, dataSource should be the name of the VolumeSnapshot previously created. The dataSource kind should be the VolumeSnapshot. The storageClassName can be any RBD storageclass.
     
    Please Note: * provisioner must be the same for both the Parent PVC and the restored PVC. * The non-encrypted PVC cannot be restored to an encrypted one and vice-versa. * encrypted -> encrypted (possible) * non-encrypted -> non-encrypted (possible) * encrypted -> non-encrypted (not possible) * non-encrypted -> encrypted (not possible)
     
    Create a new PVC from the snapshot
     
    1
    kubectl create -f deploy/examples/csi/rbd/pvc-restore.yaml
 
     
    Verify RBD Clone PVC Creation
     
    1
 2
 3
@@ -23,8 +23,8 @@
 3
    kubectl delete -f deploy/examples/csi/rbd/pvc-restore.yaml
 kubectl delete -f deploy/examples/csi/rbd/snapshot.yaml
 kubectl delete -f deploy/examples/csi/rbd/snapshotclass.yaml
-
     
    CephFS Snapshots
     
    CephFS VolumeSnapshotClass
     
    In VolumeSnapshotClass, the csi.storage.k8s.io/snapshotter-secret-name parameter should reference the name of the secret created for the cephfsplugin.
     
    In the volumesnapshotclass, update the value of the clusterID field to match the namespace that Rook is running in. When Ceph CSI is deployed by Rook, the operator will automatically maintain a configmap whose contents will match this key. By default this is "rook-ceph".
     
    1
    kubectl create -f deploy/examples/csi/cephfs/snapshotclass.yaml
-
     
    VolumeSnapshot
     
    In snapshot, volumeSnapshotClassName should be the name of the VolumeSnapshotClass previously created. The persistentVolumeClaimName should be the name of the PVC which is already created by the CephFS CSI driver.
     
    1
    kubectl create -f deploy/examples/csi/cephfs/snapshot.yaml
+
     
    CephFS Snapshots
     
    CephFS VolumeSnapshotClass
     
    In VolumeSnapshotClass, the csi.storage.k8s.io/snapshotter-secret-name parameter should reference the name of the secret created for the cephfsplugin.
     
    In the volumesnapshotclass, update the value of the clusterID field to match the namespace that Rook is running in. When Ceph CSI is deployed by Rook, the operator will automatically maintain a configmap whose contents will match this key. By default this is "rook-ceph".
     
    1
    kubectl create -f deploy/examples/csi/cephfs/snapshotclass.yaml
+
     
    VolumeSnapshot
     
    In snapshot, volumeSnapshotClassName should be the name of the VolumeSnapshotClass previously created. The persistentVolumeClaimName should be the name of the PVC which is already created by the CephFS CSI driver.
     
    1
    kubectl create -f deploy/examples/csi/cephfs/snapshot.yaml
 
     
    Verify CephFS Snapshot Creation
     
    1
 2
 3
    $ kubectl get volumesnapshotclass
@@ -35,7 +35,7 @@
 3
    $ kubectl get volumesnapshot
 NAME                  READYTOUSE   SOURCEPVC   SOURCESNAPSHOTCONTENT  RESTORESIZE   SNAPSHOTCLASS                SNAPSHOTCONTENT                                   CREATIONTIME   AGE
 cephfs-pvc-snapshot   true         cephfs-pvc                         1Gi           csi-cephfsplugin-snapclass   snapcontent-34476204-a14a-4d59-bfbc-2bbba695652c  3h50m          3h51m
-
     
    The snapshot will be ready to restore to a new PVC when READYTOUSE field of the volumesnapshot is set to true.
     
    Restore the CephFS snapshot to a new PVC
     
    In pvc-restore, dataSource should be the name of the VolumeSnapshot previously created. The dataSource kind should be the VolumeSnapshot.
     
    Create a new PVC from the snapshot
     
    1
    kubectl create -f deploy/examples/csi/cephfs/pvc-restore.yaml
+
     
    The snapshot will be ready to restore to a new PVC when READYTOUSE field of the volumesnapshot is set to true.
     
    Restore the CephFS snapshot to a new PVC
     
    In pvc-restore, dataSource should be the name of the VolumeSnapshot previously created. The dataSource kind should be the VolumeSnapshot.
     
    Create a new PVC from the snapshot
     
    1
    kubectl create -f deploy/examples/csi/cephfs/pvc-restore.yaml
 
     
    Verify CephFS Restore PVC Creation
     
    1
 2
 3
diff --git a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone/index.html b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone/index.html
index 8dd22e6d1..0e6762b42 100644
--- a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone/index.html
@@ -1,4 +1,4 @@
- Volume clone - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Volume clone
     
    The CSI Volume Cloning feature adds support for specifying existing PVCs in the dataSource field to indicate a user would like to clone a Volume.
     
    A Clone is defined as a duplicate of an existing Kubernetes Volume that can be consumed as any standard Volume would be. The only difference is that upon provisioning, rather than creating a "new" empty Volume, the back end device creates an exact duplicate of the specified Volume.
     
    Refer to clone-doc for more info.
     
    RBD Volume Cloning
     
    In pvc-clone, dataSource should be the name of the PVC which is already created by RBD CSI driver. The dataSource kind should be the PersistentVolumeClaim. The storageClassName can be any RBD storageclass (not necessarily same as Parent PVC)
     
    Please note: * provisioner must be the same for both the Parent PVC and the Clone PVC. * The non-encrypted PVC cannot be cloned to an encrypted one and vice-versa. * encrypted -> encrypted (possible) * non-encrypted -> non-encrypted (possible) * encrypted -> non-encrypted (not possible) * non-encrypted -> encrypted (not possible)
     
    Create a new PVC Clone from the PVC
     
    1
    kubectl create -f deploy/examples/csi/rbd/pvc-clone.yaml
+ Volume clone - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Volume clone
     
    The CSI Volume Cloning feature adds support for specifying existing PVCs in the dataSource field to indicate a user would like to clone a Volume.
     
    A Clone is defined as a duplicate of an existing Kubernetes Volume that can be consumed as any standard Volume would be. The only difference is that upon provisioning, rather than creating a "new" empty Volume, the back end device creates an exact duplicate of the specified Volume.
     
    Refer to clone-doc for more info.
     
    RBD Volume Cloning
     
    In pvc-clone, dataSource should be the name of the PVC which is already created by RBD CSI driver. The dataSource kind should be the PersistentVolumeClaim. The storageClassName can be any RBD storageclass (not necessarily same as Parent PVC)
     
    Please note: * provisioner must be the same for both the Parent PVC and the Clone PVC. * The non-encrypted PVC cannot be cloned to an encrypted one and vice-versa. * encrypted -> encrypted (possible) * non-encrypted -> non-encrypted (possible) * encrypted -> non-encrypted (not possible) * non-encrypted -> encrypted (not possible)
     
    Create a new PVC Clone from the PVC
     
    1
    kubectl create -f deploy/examples/csi/rbd/pvc-clone.yaml
 
     
    Verify RBD volume Clone PVC Creation
     
    1
    kubectl get pvc
 
     
    1
 2
@@ -6,7 +6,7 @@
 rbd-pvc           Bound    pvc-74734901-577a-11e9-b34f-525400581048   1Gi        RWO            rook-ceph-block       34m
 rbd-pvc-clone     Bound    pvc-70473135-577f-11e9-b34f-525400581048   1Gi        RWO            rook-ceph-block       8s
 
     
    RBD clone resource Cleanup
     
    To clean your cluster of the resources created by this example, run the following:
     
    1
    kubectl delete -f deploy/examples/csi/rbd/pvc-clone.yaml
-
     
    CephFS Volume Cloning
     
    Volume Clone Prerequisites
     
       
    1. Requires Kubernetes v1.16+ which supports volume clone.
      2.  
    2. Ceph-csi diver v3.1.0+ which supports volume clone.
      3.  
     
    In pvc-clone, dataSource should be the name of the PVC which is already created by CephFS CSI driver. The dataSource kind should be the PersistentVolumeClaim and also storageclass should be same as the source PVC.
     
    Create a new PVC Clone from the PVC
     
    1
    kubectl create -f deploy/examples/csi/cephfs/pvc-clone.yaml
+
     
    CephFS Volume Cloning
     
    Volume Clone Prerequisites
     
       
    1. Requires Kubernetes v1.16+ which supports volume clone.
      2.  
    2. Ceph-csi diver v3.1.0+ which supports volume clone.
      3.  
     
    In pvc-clone, dataSource should be the name of the PVC which is already created by CephFS CSI driver. The dataSource kind should be the PersistentVolumeClaim and also storageclass should be same as the source PVC.
     
    Create a new PVC Clone from the PVC
     
    1
    kubectl create -f deploy/examples/csi/cephfs/pvc-clone.yaml
 
     
    Verify CephFS volume Clone PVC Creation
     
    1
    kubectl get pvc
 
     
    1
 2
diff --git a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-group-snapshot/index.html b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-group-snapshot/index.html
index 038bef8b7..3c7d9df6a 100644
--- a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-group-snapshot/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-group-snapshot/index.html
@@ -1,5 +1,5 @@
- Volume Group Snapshots - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Volume Group Snapshots
     
    Ceph provides the ability to create crash-consistent snapshots of multiple volumes. A group snapshot represents “copies” from multiple volumes that are taken at the same point in time. A group snapshot can be used either to rehydrate new volumes (pre-populated with the snapshot data) or to restore existing volumes to a previous state (represented by the snapshots)
     
    Prerequisites
      
     
    Info
     
    Created by cluster administrators to describe how volume group snapshots should be created. including the driver information, the deletion policy, etc.
     
     
    Volume Group Snapshots
     
    CephFS VolumeGroupSnapshotClass
     
    In VolumeGroupSnapshotClass, the csi.storage.k8s.io/group-snapshotter-secret-name parameter should reference the name of the secret created for the cephfs-plugin.
     
    In the VolumeGroupSnapshotClass, update the value of the clusterID field to match the namespace that Rook is running in. When Ceph CSI is deployed by Rook, the operator will automatically maintain a configmap whose contents will match this key. By default this is "rook-ceph".
     
    1
    kubectl create -f deploy/examples/csi/cephfs/groupsnapshotclass.yaml
-
     
    CephFS VolumeGroupSnapshot
     
    In VolumeGroupSnapshot, volumeGroupSnapshotClassName should be the name of the VolumeGroupSnapshotClass previously created. The labels inside matchLabels should be present on the PVCs that are already created by the CephFS CSI driver.
     
    1
    kubectl create -f deploy/examples/csi/cephfs/groupsnapshot.yaml
+ Volume Group Snapshots - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Volume Group Snapshots
     
    Ceph provides the ability to create crash-consistent snapshots of multiple volumes. A group snapshot represents “copies” from multiple volumes that are taken at the same point in time. A group snapshot can be used either to rehydrate new volumes (pre-populated with the snapshot data) or to restore existing volumes to a previous state (represented by the snapshots)
     
    Prerequisites
      
     
    Info
     
    Created by cluster administrators to describe how volume group snapshots should be created. including the driver information, the deletion policy, etc.
     
     
    Volume Group Snapshots
     
    CephFS VolumeGroupSnapshotClass
     
    In VolumeGroupSnapshotClass, the csi.storage.k8s.io/group-snapshotter-secret-name parameter should reference the name of the secret created for the cephfs-plugin.
     
    In the VolumeGroupSnapshotClass, update the value of the clusterID field to match the namespace that Rook is running in. When Ceph CSI is deployed by Rook, the operator will automatically maintain a configmap whose contents will match this key. By default this is "rook-ceph".
     
    1
    kubectl create -f deploy/examples/csi/cephfs/groupsnapshotclass.yaml
+
     
    CephFS VolumeGroupSnapshot
     
    In VolumeGroupSnapshot, volumeGroupSnapshotClassName should be the name of the VolumeGroupSnapshotClass previously created. The labels inside matchLabels should be present on the PVCs that are already created by the CephFS CSI driver.
     
    1
    kubectl create -f deploy/examples/csi/cephfs/groupsnapshot.yaml
 
     
    Verify CephFS GroupSnapshot Creation
     
    1
 2
 3
    $ kubectl get volumegroupsnapshotclass
@@ -13,7 +13,7 @@
 
     
    The snapshot will be ready to restore to a new PVC when READYTOUSE field of the volumegroupsnapshot is set to true.
     
    Restore the CephFS volume group snapshot to a new PVC
     
    Find the name of the snapshots created by the VolumeGroupSnapshot first by running:
     
    1
 2
    $ kubectl get volumesnapshot -o=jsonpath='{range .items[?(@.metadata.ownerReferences[0].name=="cephfs-groupsnapshot")]}{.metadata.name}{"\n"}{end}'
 snapshot-a79d08c7b7e18953ec321e77be9c9646234593411136a3671d72e8a26ffd419c
-
     
    It will list the names of the snapshots created as part of the group.
     
    In pvc-restore, dataSource should be one of the Snapshot that we just found. The dataSource kind should be the VolumeSnapshot.
     
    Create a new PVC from the snapshot
     
    1
    kubectl create -f deploy/examples/csi/cephfs/pvc-restore.yaml
+
     
    It will list the names of the snapshots created as part of the group.
     
    In pvc-restore, dataSource should be one of the Snapshot that we just found. The dataSource kind should be the VolumeSnapshot.
     
    Create a new PVC from the snapshot
     
    1
    kubectl create -f deploy/examples/csi/cephfs/pvc-restore.yaml
 
     
    Verify CephFS Restore PVC Creation
     
    1
 2
 3
    $ kubectl get pvc
diff --git a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/custom-images/index.html b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/custom-images/index.html
index bbc1cb2f9..477184bbb 100644
--- a/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/custom-images/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Ceph-CSI/custom-images/index.html
@@ -13,4 +13,4 @@
 ROOK_CSI_SNAPSHOTTER_IMAGE: "registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0"
 ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.11.0"
 
     
    Use private repository
     
    If image version is not passed along with the image name in any of the variables above, Rook will add the corresponding default version to that image. Example: if ROOK_CSI_CEPH_IMAGE: "quay.io/private-repo/cephcsi" is passed, Rook will add internal default version and consume it as "quay.io/private-repo/cephcsi:v3.12.0".
     
    Use default images
     
    If you would like Rook to use the default upstream images, then you may simply remove all variables matching ROOK_CSI_*_IMAGE from the above ConfigMap and/or the operator deployment.
     
    Verifying updates
     
    You can use the below command to see the CSI images currently being used in the cluster. Note that not all images (like volumereplication-operator) may be present in every cluster depending on which CSI features are enabled.
     
    1
    kubectl --namespace rook-ceph get pod -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{"\n"}' -l 'app in (csi-rbdplugin,csi-rbdplugin-provisioner,csi-cephfsplugin,csi-cephfsplugin-provisioner)' | sort | uniq
-
     
    The default images can also be found with each release in the images list
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
+
     
    The default images can also be found with each release in the images list
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
diff --git a/docs/rook/v1.16/Storage-Configuration/Monitoring/ceph-monitoring/index.html b/docs/rook/v1.16/Storage-Configuration/Monitoring/ceph-monitoring/index.html
index 8125de9c8..5ceb043d2 100644
--- a/docs/rook/v1.16/Storage-Configuration/Monitoring/ceph-monitoring/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Monitoring/ceph-monitoring/index.html
@@ -81,7 +81,7 @@
 
     
    Finally, run kustomize to update the desired prometheus rules:
     
    1
 2
    kustomize build . > updated-chart.yaml
 kubectl create -f updated-chart.yaml
-
     
    Grafana Dashboards
     
    The dashboards have been created by @galexrt. For feedback on the dashboards please reach out to him on the Rook.io Slack.
     
     
    Note
     
    The dashboards are only compatible with Grafana 7.2.0 or higher. Also note that the dashboards are updated from time to time, to fix issues and improve them.
     
     
    The following Grafana dashboards are available:
      
    The dashboard JSON files are also available on GitHub here /deploy/examples/monitoring/grafana/.
     
    Updates and Upgrades
     
    When updating Rook, there may be updates to RBAC for monitoring. It is easy to apply the changes with each update or upgrade. This should be done at the same time you update Rook common resources like common.yaml.
     
    1
    kubectl apply -f deploy/examples/monitoring/rbac.yaml
+
     
    Grafana Dashboards
     
    The dashboards have been created by @galexrt. For feedback on the dashboards please reach out to him on the Rook.io Slack.
     
     
    Note
     
    The dashboards are only compatible with Grafana 7.2.0 or higher. Also note that the dashboards are updated from time to time, to fix issues and improve them.
     
     
    The following Grafana dashboards are available:
      
    The dashboard JSON files are also available on GitHub here /deploy/examples/monitoring/grafana/.
     
    Updates and Upgrades
     
    When updating Rook, there may be updates to RBAC for monitoring. It is easy to apply the changes with each update or upgrade. This should be done at the same time you update Rook common resources like common.yaml.
     
    1
    kubectl apply -f deploy/examples/monitoring/rbac.yaml
 
     
     
    Hint
     
    This is updated automatically if you are upgrading via the helm chart
     
     
    Teardown
     
    To clean up all the artifacts created by the monitoring walk-through, copy/paste the entire block below (note that errors about resources "not found" can be ignored):
     
    1
 2
 3
diff --git a/docs/rook/v1.16/Storage-Configuration/NFS/nfs-advanced/index.html b/docs/rook/v1.16/Storage-Configuration/NFS/nfs-advanced/index.html
index 861145c13..1393b4c19 100644
--- a/docs/rook/v1.16/Storage-Configuration/NFS/nfs-advanced/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/NFS/nfs-advanced/index.html
@@ -1,4 +1,4 @@
- Advanced configuration - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Advanced configuration
     
    All CephNFS daemons are configured using shared RADOS objects stored in a Ceph pool named .nfs. Users can modify the configuration object for each CephNFS cluster if they wish to customize the configuration.
     
    Changing configuration of the .nfs pool
     
    By default, Rook creates the .nfs pool with Ceph's default configuration. If you wish to change the configuration of this pool (for example to change its failure domain or replication factor), you can create a CephBlockPool with the spec.name field set to .nfs. This pool must be replicated and cannot be erasure coded. deploy/examples/nfs.yaml contains a sample for reference.
     
    Adding custom NFS-Ganesha config file changes
     
    Ceph uses NFS-Ganesha servers. The config file format for these objects is documented in the NFS-Ganesha project.
     
    Use Ceph's rados tool from the toolbox to interact with the configuration object. The below command will get you started by dumping the contents of the config object to stdout. The output will look something like the example shown if you have already created two exports as documented above. It is best not to modify any of the export objects created by Ceph so as not to cause errors with Ceph's export management.
     
    1
+ Advanced configuration - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Advanced configuration
     
    All CephNFS daemons are configured using shared RADOS objects stored in a Ceph pool named .nfs. Users can modify the configuration object for each CephNFS cluster if they wish to customize the configuration.
     
    Changing configuration of the .nfs pool
     
    By default, Rook creates the .nfs pool with Ceph's default configuration. If you wish to change the configuration of this pool (for example to change its failure domain or replication factor), you can create a CephBlockPool with the spec.name field set to .nfs. This pool must be replicated and cannot be erasure coded. deploy/examples/nfs.yaml contains a sample for reference.
     
    Adding custom NFS-Ganesha config file changes
     
    Ceph uses NFS-Ganesha servers. The config file format for these objects is documented in the NFS-Ganesha project.
     
    Use Ceph's rados tool from the toolbox to interact with the configuration object. The below command will get you started by dumping the contents of the config object to stdout. The output will look something like the example shown if you have already created two exports as documented above. It is best not to modify any of the export objects created by Ceph so as not to cause errors with Ceph's export management.
     
    1
 2
 3
    $ rados --pool <pool> --namespace <namespace> get conf-nfs.<cephnfs-name> -
 %url "rados://<pool>/<namespace>/export-1"
diff --git a/docs/rook/v1.16/Storage-Configuration/NFS/nfs-csi-driver/index.html b/docs/rook/v1.16/Storage-Configuration/NFS/nfs-csi-driver/index.html
index 70342b2bc..3f5c183b7 100644
--- a/docs/rook/v1.16/Storage-Configuration/NFS/nfs-csi-driver/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/NFS/nfs-csi-driver/index.html
@@ -1,4 +1,4 @@
- CSI provisioner and driver - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    CSI provisioner and driver
     
     
    Attention
     
    This feature is experimental and will not support upgrades to future versions.
     
     
    For this section, we will refer to Rook's deployment examples in the deploy/examples directory.
     
    Enabling the CSI drivers
     
    The Ceph CSI NFS provisioner and driver require additional RBAC to operate. Apply the deploy/examples/csi/nfs/rbac.yaml manifest to deploy the additional resources.
     
    Rook will only deploy the Ceph CSI NFS provisioner and driver components when the ROOK_CSI_ENABLE_NFS config is set to "true" in the rook-ceph-operator-config configmap. Change the value in your manifest, or patch the resource as below.
     
    1
    kubectl --namespace rook-ceph patch configmap rook-ceph-operator-config --type merge --patch '{"data":{"ROOK_CSI_ENABLE_NFS": "true"}}'
+ CSI provisioner and driver - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    CSI provisioner and driver
     
     
    Attention
     
    This feature is experimental and will not support upgrades to future versions.
     
     
    For this section, we will refer to Rook's deployment examples in the deploy/examples directory.
     
    Enabling the CSI drivers
     
    The Ceph CSI NFS provisioner and driver require additional RBAC to operate. Apply the deploy/examples/csi/nfs/rbac.yaml manifest to deploy the additional resources.
     
    Rook will only deploy the Ceph CSI NFS provisioner and driver components when the ROOK_CSI_ENABLE_NFS config is set to "true" in the rook-ceph-operator-config configmap. Change the value in your manifest, or patch the resource as below.
     
    1
    kubectl --namespace rook-ceph patch configmap rook-ceph-operator-config --type merge --patch '{"data":{"ROOK_CSI_ENABLE_NFS": "true"}}'
 
     
     
    Note
     
    The rook-ceph operator Helm chart will deploy the required RBAC and enable the driver components if csi.nfs.enabled is set to true.
     
     
    Creating NFS exports via PVC
     
    Prerequisites
     
    In order to create NFS exports via the CSI driver, you must first create a CephFilesystem to serve as the underlying storage for the exports, and you must create a CephNFS to run an NFS server that will expose the exports. RGWs cannot be used for the CSI driver.
     
    From the examples, filesystem.yaml creates a CephFilesystem called myfs, and nfs.yaml creates an NFS server called my-nfs.
     
    You may need to enable or disable the Ceph orchestrator.
     
    You must also create a storage class. Ceph CSI is designed to support any arbitrary Ceph cluster, but we are focused here only on Ceph clusters deployed by Rook. Let's take a look at a portion of the example storage class found at deploy/examples/csi/nfs/storageclass.yaml and break down how the values are determined.
     
     1
  2
  3
@@ -43,8 +43,8 @@
 
     
       
    1. provisioner: rook-ceph.nfs.csi.ceph.com because rook-ceph is the namespace where the CephCluster is installed
      2.  
    2. nfsCluster: my-nfs because this is the name of the CephNFS
      3.  
    3. server: rook-ceph-nfs-my-nfs-a because Rook creates this Kubernetes Service for the CephNFS named my-nfs
      4.  
    4. clusterID: rook-ceph because this is the namespace where the CephCluster is installed
      5.  
    5. fsName: myfs because this is the name of the CephFilesystem used to back the NFS exports
      6.  
    6. pool: myfs-replicated because myfs is the name of the CephFilesystem defined in fsName and because replicated is the name of a data pool defined in the CephFilesystem
      7.  
    7. csi.storage.k8s.io/*: note that these values are shared with the Ceph CSI CephFS provisioner
      8.  
     
    Creating a PVC
     
    See deploy/examples/csi/nfs/pvc.yaml for an example of how to create a PVC that will create an NFS export. The export will be created and a PV created for the PVC immediately, even without a Pod to mount the PVC.
     
    Attaching an export to a pod
     
    See deploy/examples/csi/nfs/pod.yaml for an example of how a PVC can be connected to an application pod.
     
    Connecting to an export directly
     
    After a PVC is created successfully, the share parameter set on the resulting PV contains the share path which can be used as the export path when mounting the export manually. In the example below /0001-0009-rook-ceph-0000000000000001-55c910f9-a1af-11ed-9772-1a471870b2f5 is the export path.
     
    1
 2
    $ kubectl get pv pvc-b559f225-de79-451b-a327-3dbec1f95a1c -o jsonpath='{.spec.csi.volumeAttributes}'
 /0001-0009-rook-ceph-0000000000000001-55c910f9-a1af-11ed-9772-1a471870b2f5
-
     
    Taking snapshots of NFS exports
     
    NFS export PVCs can be snapshotted and later restored to new PVCs.
     
    Creating snapshots
     
    First, create a VolumeSnapshotClass as in the example here. The csi.storage.k8s.io/snapshotter-secret-name parameter should reference the name of the secret created for the cephfsplugin here.
     
    1
    kubectl create -f deploy/examples/csi/nfs/snapshotclass.yaml
-
     
    In snapshot, volumeSnapshotClassName should be the name of the VolumeSnapshotClass previously created. The persistentVolumeClaimName should be the name of the PVC which is already created by the NFS CSI driver.
     
    1
    kubectl create -f deploy/examples/csi/nfs/snapshot.yaml
+
     
    Taking snapshots of NFS exports
     
    NFS export PVCs can be snapshotted and later restored to new PVCs.
     
    Creating snapshots
     
    First, create a VolumeSnapshotClass as in the example here. The csi.storage.k8s.io/snapshotter-secret-name parameter should reference the name of the secret created for the cephfsplugin here.
     
    1
    kubectl create -f deploy/examples/csi/nfs/snapshotclass.yaml
+
     
    In snapshot, volumeSnapshotClassName should be the name of the VolumeSnapshotClass previously created. The persistentVolumeClaimName should be the name of the PVC which is already created by the NFS CSI driver.
     
    1
    kubectl create -f deploy/examples/csi/nfs/snapshot.yaml
 
     
    Verifying snapshots
     
    1
 2
 3
    $ kubectl get volumesnapshotclass
@@ -55,7 +55,7 @@
 3
    $ kubectl get volumesnapshot
 NAME                  READYTOUSE   SOURCEPVC   SOURCESNAPSHOTCONTENT  RESTORESIZE   SNAPSHOTCLASS                SNAPSHOTCONTENT                                   CREATIONTIME   AGE
 nfs-pvc-snapshot      true         nfs-pvc                            1Gi           csi-nfsplugin-snapclass      snapcontent-34476204-a14a-4d59-bfbc-2bbba695652c  3h50m          3h51m
-
     
    The snapshot will be ready to restore to a new PVC when READYTOUSE field of the volumesnapshot is set to true.
     
    Restoring snapshot to a new PVC
     
    In pvc-restore, dataSource name should be the name of the VolumeSnapshot previously created. The dataSource kind should be "VolumeSnapshot".
     
    Create a new PVC from the snapshot.
     
    1
    kubectl create -f deploy/examples/csi/nfs/pvc-restore.yaml
+
     
    The snapshot will be ready to restore to a new PVC when READYTOUSE field of the volumesnapshot is set to true.
     
    Restoring snapshot to a new PVC
     
    In pvc-restore, dataSource name should be the name of the VolumeSnapshot previously created. The dataSource kind should be "VolumeSnapshot".
     
    Create a new PVC from the snapshot.
     
    1
    kubectl create -f deploy/examples/csi/nfs/pvc-restore.yaml
 
     
    Verifying restored PVC Creation
     
    1
 2
 3
@@ -68,7 +68,7 @@
 3
    kubectl delete -f deploy/examples/csi/nfs/pvc-restore.yaml
 kubectl delete -f deploy/examples/csi/nfs/snapshot.yaml
 kubectl delete -f deploy/examples/csi/nfs/snapshotclass.yaml
-
     
    Cloning NFS exports
     
    Creating clones
     
    In pvc-clone, dataSource should be the name of the PVC which is already created by NFS CSI driver. The dataSource kind should be "PersistentVolumeClaim" and also storageclass should be same as the source PVC.
     
    Create a new PVC Clone from the PVC as in the example here.
     
    1
    kubectl create -f deploy/examples/csi/nfs/pvc-clone.yaml
+
     
    Cloning NFS exports
     
    Creating clones
     
    In pvc-clone, dataSource should be the name of the PVC which is already created by NFS CSI driver. The dataSource kind should be "PersistentVolumeClaim" and also storageclass should be same as the source PVC.
     
    Create a new PVC Clone from the PVC as in the example here.
     
    1
    kubectl create -f deploy/examples/csi/nfs/pvc-clone.yaml
 
     
    Verifying a cloned PVC
     
    1
    kubectl get pvc
 
     
    1
 2
diff --git a/docs/rook/v1.16/Storage-Configuration/NFS/nfs/index.html b/docs/rook/v1.16/Storage-Configuration/NFS/nfs/index.html
index 34bfde5e7..a13dca14a 100644
--- a/docs/rook/v1.16/Storage-Configuration/NFS/nfs/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/NFS/nfs/index.html
@@ -1,4 +1,4 @@
- NFS Storage Overview - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    NFS Storage Overview
     
    NFS storage can be mounted with read/write permission from multiple pods. NFS storage may be especially useful for leveraging an existing Rook cluster to provide NFS storage for legacy applications that assume an NFS client connection. Such applications may not have been migrated to Kubernetes or might not yet support PVCs. Rook NFS storage can provide access to the same network filesystem storage from within the Kubernetes cluster via PVC while simultaneously providing access via direct client connection from within or outside of the Kubernetes cluster.
     
     
    Warning
     
    Simultaneous access to NFS storage from Pods and from external clients complicates NFS user ID mapping significantly. Client IDs mapped from external clients will not be the same as the IDs associated with the NFS CSI driver, which mount exports for Kubernetes pods.
     
     
     
    Warning
     
    Due to a number of Ceph issues and changes, Rook officially only supports Ceph v16.2.7 or higher for CephNFS. If you are using an earlier version, upgrade your Ceph version following the advice given in Rook's v1.9 NFS docs.
     
     
     
    Note
     
    CephNFSes support NFSv4.1+ access only. Serving earlier protocols inhibits responsiveness after a server restart.
     
     
    Prerequisites
     
    This guide assumes you have created a Rook cluster as explained in the main quickstart guide as well as a Ceph filesystem which will act as the backing storage for NFS.
     
    Many samples reference the CephNFS and CephFilesystem example manifests here and here.
     
    Creating an NFS cluster
     
    Create the NFS cluster by specifying the desired settings documented for the NFS CRD.
     
    Creating Exports
     
    When a CephNFS is first created, all NFS daemons within the CephNFS cluster will share a configuration with no exports defined. When creating an export, it is necessary to specify the CephFilesystem which will act as the backing storage for the NFS export.
     
    RADOS Gateways (RGWs), provided by CephObjectStores, can also be used as backing storage for NFS exports if desired.
     
    Using the Ceph Dashboard
     
    Exports can be created via the Ceph dashboard as well. To enable and use the Ceph dashboard in Rook, see here.
     
    Using the Ceph CLI
     
    The Ceph CLI can be used from the Rook toolbox pod to create and manage NFS exports. To do so, first ensure the necessary Ceph mgr modules are enabled, if necessary, and that the Ceph orchestrator backend is set to Rook.
     
    Enable the Ceph orchestrator (optional)
     
    1
+ NFS Storage Overview - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    NFS Storage Overview
     
    NFS storage can be mounted with read/write permission from multiple pods. NFS storage may be especially useful for leveraging an existing Rook cluster to provide NFS storage for legacy applications that assume an NFS client connection. Such applications may not have been migrated to Kubernetes or might not yet support PVCs. Rook NFS storage can provide access to the same network filesystem storage from within the Kubernetes cluster via PVC while simultaneously providing access via direct client connection from within or outside of the Kubernetes cluster.
     
     
    Warning
     
    Simultaneous access to NFS storage from Pods and from external clients complicates NFS user ID mapping significantly. Client IDs mapped from external clients will not be the same as the IDs associated with the NFS CSI driver, which mount exports for Kubernetes pods.
     
     
     
    Warning
     
    Due to a number of Ceph issues and changes, Rook officially only supports Ceph v16.2.7 or higher for CephNFS. If you are using an earlier version, upgrade your Ceph version following the advice given in Rook's v1.9 NFS docs.
     
     
     
    Note
     
    CephNFSes support NFSv4.1+ access only. Serving earlier protocols inhibits responsiveness after a server restart.
     
     
    Prerequisites
     
    This guide assumes you have created a Rook cluster as explained in the main quickstart guide as well as a Ceph filesystem which will act as the backing storage for NFS.
     
    Many samples reference the CephNFS and CephFilesystem example manifests here and here.
     
    Creating an NFS cluster
     
    Create the NFS cluster by specifying the desired settings documented for the NFS CRD.
     
    Creating Exports
     
    When a CephNFS is first created, all NFS daemons within the CephNFS cluster will share a configuration with no exports defined. When creating an export, it is necessary to specify the CephFilesystem which will act as the backing storage for the NFS export.
     
    RADOS Gateways (RGWs), provided by CephObjectStores, can also be used as backing storage for NFS exports if desired.
     
    Using the Ceph Dashboard
     
    Exports can be created via the Ceph dashboard as well. To enable and use the Ceph dashboard in Rook, see here.
     
    Using the Ceph CLI
     
    The Ceph CLI can be used from the Rook toolbox pod to create and manage NFS exports. To do so, first ensure the necessary Ceph mgr modules are enabled, if necessary, and that the Ceph orchestrator backend is set to Rook.
     
    Enable the Ceph orchestrator (optional)
     
    1
 2
 3
    ceph mgr module enable rook
 ceph mgr module enable nfs
@@ -58,4 +58,4 @@
 2
    ceph orch set backend ""
 ceph mgr module disable rook
 
     
    Mounting exports
     
    Each CephNFS server has a unique Kubernetes Service. This is because NFS clients can't readily handle NFS failover. CephNFS services are named with the pattern rook-ceph-nfs-<cephnfs-name>-<id> <id> is a unique letter ID (e.g., a, b, c, etc.) for a given NFS server. For example, rook-ceph-nfs-my-nfs-a.
     
    For each NFS client, choose an NFS service to use for the connection. With NFS v4, you can mount an export by its path using a mount command like below. You can mount all exports at once by omitting the export path and leaving the directory as just /.
     
    1
    mount -t nfs4 -o proto=tcp <nfs-service-address>:/<export-path> <mount-location>
-
     
    Exposing the NFS server outside of the Kubernetes cluster
     
    Use a LoadBalancer Service to expose an NFS server (and its exports) outside of the Kubernetes cluster. The Service's endpoint can be used as the NFS service address when mounting the export manually. We provide an example Service here: deploy/examples/nfs-load-balancer.yaml.
     
    NFS Security
     
    Security options for NFS are documented here.
     
    Ceph CSI NFS provisioner and NFS CSI driver
     
    The NFS CSI provisioner and driver are documented here
     
    Advanced configuration
     
    Advanced NFS configuration is documented here
     
    Known issues
     
    Known issues are documented on the NFS CRD page.
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
+
     
    Exposing the NFS server outside of the Kubernetes cluster
     
    Use a LoadBalancer Service to expose an NFS server (and its exports) outside of the Kubernetes cluster. The Service's endpoint can be used as the NFS service address when mounting the export manually. We provide an example Service here: deploy/examples/nfs-load-balancer.yaml.
     
    NFS Security
     
    Security options for NFS are documented here.
     
    Ceph CSI NFS provisioner and NFS CSI driver
     
    The NFS CSI provisioner and driver are documented here
     
    Advanced configuration
     
    Advanced NFS configuration is documented here
     
    Known issues
     
    Known issues are documented on the NFS CRD page.
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
diff --git a/docs/rook/v1.16/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite/index.html b/docs/rook/v1.16/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite/index.html
index 709a984f8..284d66f5a 100644
--- a/docs/rook/v1.16/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite/index.html
@@ -1,4 +1,4 @@
- Object Store Multisite - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Object Store Multisite
     
    Multisite is a feature of Ceph that allows object stores to replicate their data over multiple Ceph clusters.
     
    Multisite also allows object stores to be independent and isolated from other object stores in a cluster.
     
    When a ceph-object-store is created without the zone section; a realm, zone group, and zone is created with the same name as the ceph-object-store.
     
    Since it is the only ceph-object-store in the realm, the data in the ceph-object-store remain independent and isolated from others on the same cluster.
     
    When a ceph-object-store is created with the zone section, the ceph-object-store will join a custom created zone, zone group, and realm each with a different names than its own.
     
    This allows the ceph-object-store to replicate its data over multiple Ceph clusters.
     
    To review core multisite concepts please read the ceph-multisite design overview.
     
    Prerequisites
     
    This guide assumes a Rook cluster as explained in the Quickstart.
     
    Creating Object Multisite
     
    If an admin wants to set up multisite on a Rook Ceph cluster, the following resources must be created:
     
       
    1. A realm
      2.  
    2. A zonegroup
      3.  
    3. A zone
      4.  
    4. A ceph object store with the zone section
      5.  
     
    object-multisite.yaml in the examples directory can be used to create the multisite CRDs.
     
    1
    kubectl create -f object-multisite.yaml
+ Object Store Multisite - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Object Store Multisite
     
    Multisite is a feature of Ceph that allows object stores to replicate their data over multiple Ceph clusters.
     
    Multisite also allows object stores to be independent and isolated from other object stores in a cluster.
     
    When a ceph-object-store is created without the zone section; a realm, zone group, and zone is created with the same name as the ceph-object-store.
     
    Since it is the only ceph-object-store in the realm, the data in the ceph-object-store remain independent and isolated from others on the same cluster.
     
    When a ceph-object-store is created with the zone section, the ceph-object-store will join a custom created zone, zone group, and realm each with a different names than its own.
     
    This allows the ceph-object-store to replicate its data over multiple Ceph clusters.
     
    To review core multisite concepts please read the ceph-multisite design overview.
     
    Prerequisites
     
    This guide assumes a Rook cluster as explained in the Quickstart.
     
    Creating Object Multisite
     
    If an admin wants to set up multisite on a Rook Ceph cluster, the following resources must be created:
     
       
    1. A realm
      2.  
    2. A zonegroup
      3.  
    3. A zone
      4.  
    4. A ceph object store with the zone section
      5.  
     
    object-multisite.yaml in the examples directory can be used to create the multisite CRDs.
     
    1
    kubectl create -f object-multisite.yaml
 
     
    The first zone group created in a realm is the master zone group. The first zone created in a zone group is the master zone.
     
    When a non-master zone or non-master zone group is created, the zone group or zone is not in the Ceph Radosgw Multisite Period until an object-store is created in that zone (and zone group).
     
    The zone will create the pools for the object-store(s) that are in the zone to use.
     
    When one of the multisite CRs (realm, zone group, zone) is deleted the underlying ceph realm/zone group/zone is not deleted, neither are the pools created by the zone. See the "Multisite Cleanup" section for more information.
     
    For more information on the multisite CRDs, see the related CRDs:
      
    Pulling a Realm
     
    If an admin wants to sync data from another cluster, the admin needs to pull a realm on a Rook Ceph cluster from another Rook Ceph (or Ceph) cluster.
     
    To begin doing this, the admin needs 2 pieces of information:
     
       
    1. An endpoint from the realm being pulled from
      2.  
    2. The access key and the system key of the system user from the realm being pulled from.
      3.  
     
    Getting the Pull Endpoint
     
    To pull a Ceph realm from a remote Ceph cluster, an endpoint must be added to the CephObjectRealm's pull section in the spec. This endpoint must be from the master zone in the master zone group of that realm.
     
    If an admin does not know of an endpoint that fits this criteria, the admin can find such an endpoint on the remote Ceph cluster (via the tool box if it is a Rook Ceph Cluster) by running:
     
    1
 2
 3
@@ -70,7 +70,7 @@
   namespace: $NEW_ROOK_CLUSTER_NAMESPACE
 type: kubernetes.io/rook
 
     
    Finally, create a kubernetes secret on the pulling Rook Ceph cluster with the new secrets yaml file.
     
    1
    kubectl create -f realm-a-keys.yaml
-
     
    Pulling a Realm on a New Rook Ceph Cluster
     
    Once the admin knows the endpoint and the secret for the keys has been created, the admin should create:
     
       
    1. A CephObjectRealm matching to the realm on the other Ceph cluster, with an endpoint as described above.
      2.  
    2. A CephObjectZoneGroup matching the master zone group name or the master CephObjectZoneGroup from the cluster the realm was pulled from.
      3.  
    3. A CephObjectZone referring to the CephObjectZoneGroup created above.
      4.  
    4. A CephObjectStore referring to the new CephObjectZone resource.
      5.  
     
    object-multisite-pull-realm.yaml (with changes) in the examples directory can be used to create the multisite CRDs.
     
    1
    kubectl create -f object-multisite-pull-realm.yaml
+
     
    Pulling a Realm on a New Rook Ceph Cluster
     
    Once the admin knows the endpoint and the secret for the keys has been created, the admin should create:
     
       
    1. A CephObjectRealm matching to the realm on the other Ceph cluster, with an endpoint as described above.
      2.  
    2. A CephObjectZoneGroup matching the master zone group name or the master CephObjectZoneGroup from the cluster the realm was pulled from.
      3.  
    3. A CephObjectZone referring to the CephObjectZoneGroup created above.
      4.  
    4. A CephObjectStore referring to the new CephObjectZone resource.
      5.  
     
    object-multisite-pull-realm.yaml (with changes) in the examples directory can be used to create the multisite CRDs.
     
    1
    kubectl create -f object-multisite-pull-realm.yaml
 
     
    Scaling a Multisite
     
    Scaling the number of gateways that run the synchronization thread to 2 or more can increase the latency of the replication of each S3 object. The recommended way to scale a multisite configuration is to dissociate the gateway dedicated to the synchronization from gateways that serve clients.
     
    The two types of gateways can be deployed by creating two CephObjectStores associated with the same CephObjectZone. The objectstore that deploys the gateway dedicated to the synchronization must have spec.gateway.instances set to 1, while the objectstore that deploys the client gateways have multiple replicas and should disable the synchronization thread on the gateways by setting spec.gateway.disableMultisiteSyncTraffic to true.
     
     1
  2
  3
diff --git a/docs/rook/v1.16/Storage-Configuration/Object-Storage-RGW/object-storage/index.html b/docs/rook/v1.16/Storage-Configuration/Object-Storage-RGW/object-storage/index.html
index 6ce21134e..81691f27d 100644
--- a/docs/rook/v1.16/Storage-Configuration/Object-Storage-RGW/object-storage/index.html
+++ b/docs/rook/v1.16/Storage-Configuration/Object-Storage-RGW/object-storage/index.html
@@ -655,4 +655,4 @@
     swift:
       # if S3 API is disabled, then SWIFT can be hosted on root path without prefix
       urlPrefix: "/"
-
     
     
    Note
     
    Child resources should refer to the appropriate RGW instance. For example, a CephObjectStoreUser requires the Admin Ops API, so it should refer to an instance where this API is enabled. After the user is created, it can be used for all instances.
     
     
    See the example configuration for more details.
     
    Using Swift and Keystone
     
    It is possible to access an object store using the Swift API. Using Swift requires the use of OpenStack Keystone as an authentication provider.
     
    More information on the use of Swift and Keystone can be found in the document on Object Store with Keystone and Swift.
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
+
     
     
    Note
     
    Child resources should refer to the appropriate RGW instance. For example, a CephObjectStoreUser requires the Admin Ops API, so it should refer to an instance where this API is enabled. After the user is created, it can be used for all instances.
     
     
    See the example configuration for more details.
     
    Using Swift and Keystone
     
    It is possible to access an object store using the Swift API. Using Swift requires the use of OpenStack Keystone as an authentication provider.
     
    More information on the use of Swift and Keystone can be found in the document on Object Store with Keystone and Swift.
     
     
       
      
     
     
     
     
     
      
     © Rook Authors 2022. Documentation distributed under CC-BY-4.0. 
     
     © 2022 The Linux Foundation. All rights reserved. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see our Trademark Usage page. 
     
     Made with  Material for MkDocs  
     
                 
     
     
     
     
     
     
     
        
\ No newline at end of file
diff --git a/docs/rook/v1.16/Troubleshooting/disaster-recovery/index.html b/docs/rook/v1.16/Troubleshooting/disaster-recovery/index.html
index ebfa8293e..6dfe0c839 100644
--- a/docs/rook/v1.16/Troubleshooting/disaster-recovery/index.html
+++ b/docs/rook/v1.16/Troubleshooting/disaster-recovery/index.html
@@ -1,5 +1,5 @@
  Disaster Recovery - Rook Ceph Documentation      
     
      
       
     
     
     
     
     
     
      
     
     
     
     
     
      
     
     
     
     
     
    Disaster Recovery
     
    Under extenuating circumstances, steps may be necessary to recover the cluster health. There are several types of recovery addressed in this document.
     
    Restoring Mon Quorum
     
    Under extenuating circumstances, the mons may lose quorum. If the mons cannot form quorum again, there is a manual procedure to get the quorum going again. The only requirement is that at least one mon is still healthy. The following steps will remove the unhealthy mons from quorum and allow you to form a quorum again with a single mon, then grow the quorum back to the original size.
     
    The Rook kubectl Plugin has a command restore-quorum that will walk you through the mon quorum automated restoration process.
     
    If the name of the healthy mon is c, you would run the command:
     
    1
    kubectl rook-ceph mons restore-quorum c
-
     
    See the restore-quorum documentation for more details.
     
    Restoring CRDs After Deletion
     
    When the Rook CRDs are deleted, the Rook operator will respond to the deletion event to attempt to clean up the cluster resources. If any data appears present in the cluster, Rook will refuse to allow the resources to be deleted since the operator will refuse to remove the finalizer on the CRs until the underlying data is deleted. For more details, see the dependency design doc.
     
    While it is good that the CRs will not be deleted and the underlying Ceph data and daemons continue to be available, the CRs will be stuck indefinitely in a Deleting state in which the operator will not continue to ensure cluster health. Upgrades will be blocked, further updates to the CRs are prevented, and so on. Since Kubernetes does not allow undeleting resources, the following procedure will allow you to restore the CRs to their prior state without even necessarily suffering cluster downtime.
     
     
    Note
     
    In the following commands, the affected CephCluster resource is called rook-ceph. If yours is named differently, the commands will need to be adjusted.
     
     
       
    1.  
      Scale down the operator.
       
      1
      kubectl -n rook-ceph scale --replicas=0 deploy/rook-ceph-operator
+
       
      See the restore-quorum documentation for more details.
       
      Restoring CRDs After Deletion
       
      When the Rook CRDs are deleted, the Rook operator will respond to the deletion event to attempt to clean up the cluster resources. If any data appears present in the cluster, Rook will refuse to allow the resources to be deleted since the operator will refuse to remove the finalizer on the CRs until the underlying data is deleted. For more details, see the dependency design doc.
       
      While it is good that the CRs will not be deleted and the underlying Ceph data and daemons continue to be available, the CRs will be stuck indefinitely in a Deleting state in which the operator will not continue to ensure cluster health. Upgrades will be blocked, further updates to the CRs are prevented, and so on. Since Kubernetes does not allow undeleting resources, the following procedure will allow you to restore the CRs to their prior state without even necessarily suffering cluster downtime.
       
       
      Note
       
      In the following commands, the affected CephCluster resource is called rook-ceph. If yours is named differently, the commands will need to be adjusted.
       
       
         
      1.  
        Scale down the operator.
         
        1
        kubectl -n rook-ceph scale --replicas=0 deploy/rook-ceph-operator
 
         
        2.  
      2.  
        Backup all Rook CRs and critical metadata
         
        1
 2
 3
diff --git a/docs/rook/v1.16/Troubleshooting/openshift-common-issues/index.html b/docs/rook/v1.16/Troubleshooting/openshift-common-issues/index.html
index daa892635..a338d6f0c 100644
--- a/docs/rook/v1.16/Troubleshooting/openshift-common-issues/index.html
+++ b/docs/rook/v1.16/Troubleshooting/openshift-common-issues/index.html
@@ -1,4 +1,4 @@
- OpenShift Common Issues - Rook Ceph Documentation      
         
          
           
         
         
         
         
         
         
          
         
         
         
         
         
          
         
         
         
         
         
        OpenShift Common Issues
         
        Enable Monitoring in the Storage Dashboard
         
        OpenShift Console uses OpenShift Prometheus for monitoring and populating data in Storage Dashboard. Additional configuration is required to monitor the Ceph Cluster from the storage dashboard.
         
           
        1.  
          Change the monitoring namespace to openshift-monitoring
           
          Change the namespace of the RoleBinding rook-ceph-metrics from rook-ceph to openshift-monitoring for the prometheus-k8s ServiceAccount in rbac.yaml.
           
          1
+ OpenShift Common Issues - Rook Ceph Documentation      
           
            
             
           
           
           
           
           
           
            
           
           
           
           
           
            
           
           
           
           
           
          OpenShift Common Issues
           
          Enable Monitoring in the Storage Dashboard
           
          OpenShift Console uses OpenShift Prometheus for monitoring and populating data in Storage Dashboard. Additional configuration is required to monitor the Ceph Cluster from the storage dashboard.
           
             
          1.  
            Change the monitoring namespace to openshift-monitoring
             
            Change the namespace of the RoleBinding rook-ceph-metrics from rook-ceph to openshift-monitoring for the prometheus-k8s ServiceAccount in rbac.yaml.
             
            1
 2
 3
 4
            subjects:
diff --git a/docs/rook/v1.16/sitemap.xml b/docs/rook/v1.16/sitemap.xml
index bca7c7ab4..36ec3b732 100644
--- a/docs/rook/v1.16/sitemap.xml
+++ b/docs/rook/v1.16/sitemap.xml
@@ -2,334 +2,334 @@
 
     
          https://rook.io/v1.16/CRDs/ceph-client-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/ceph-nfs-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/specification/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Block-Storage/ceph-block-pool-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Block-Storage/ceph-block-pool-rados-namespace-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Block-Storage/ceph-rbd-mirror-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/ceph-cluster-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/host-cluster/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/network-providers/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/pvc-cluster/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/stretch-cluster/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/external-cluster/advance-external/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/external-cluster/consumer-import/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/external-cluster/external-cluster/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/external-cluster/provider-export/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/external-cluster/topology-for-external-mode/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Cluster/external-cluster/upgrade-external/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Object-Storage/ceph-object-realm-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Object-Storage/ceph-object-store-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Object-Storage/ceph-object-store-user-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Object-Storage/ceph-object-zone-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Object-Storage/ceph-object-zonegroup-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Shared-Filesystem/ceph-filesystem-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Shared-Filesystem/ceph-fs-mirror-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/CRDs/Shared-Filesystem/ceph-fs-subvolumegroup-crd/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Contributing/ci-configuration/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Contributing/development-environment/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Contributing/development-flow/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Contributing/documentation/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Contributing/rook-test-framework/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/ceph-openshift/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/ceph-teardown/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/example-configurations/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/glossary/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/intro/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/quickstart/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/release-cycle/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/storage-architecture/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/Prerequisites/authenticated-registry/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Getting-Started/Prerequisites/prerequisites/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Helm-Charts/ceph-cluster-chart/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Helm-Charts/helm-charts/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Helm-Charts/operator-chart/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/ceph-teardown/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Advanced/ceph-configuration/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Advanced/ceph-mon-health/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Advanced/ceph-osd-mgmt/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Advanced/configuration/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Advanced/key-management-system/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Block-Storage-RBD/block-storage/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Block-Storage-RBD/rbd-async-disaster-recovery-failover-failback/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Block-Storage-RBD/rbd-mirroring/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Ceph-CSI/ceph-csi-volume-group-snapshot/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Ceph-CSI/custom-images/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Monitoring/ceph-dashboard/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Monitoring/ceph-monitoring/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/NFS/nfs-advanced/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/NFS/nfs-csi-driver/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/NFS/nfs-security/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/NFS/nfs/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Object-Storage-RGW/ceph-object-multisite/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Object-Storage-RGW/ceph-object-swift/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Object-Storage-RGW/cosi/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Object-Storage-RGW/object-storage/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-mirroring/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Troubleshooting/ceph-common-issues/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Troubleshooting/ceph-csi-common-issues/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Troubleshooting/ceph-toolbox/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Troubleshooting/common-issues/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Troubleshooting/direct-tools/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Troubleshooting/disaster-recovery/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Troubleshooting/kubectl-plugin/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Troubleshooting/openshift-common-issues/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Troubleshooting/performance-profiling/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Upgrade/ceph-upgrade/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Upgrade/health-verification/
-         2025-02-20
+         2025-02-24
     
     
          https://rook.io/v1.16/Upgrade/rook-upgrade/
-         2025-02-20
+         2025-02-24
     
 
\ No newline at end of file
diff --git a/docs/rook/v1.16/sitemap.xml.gz b/docs/rook/v1.16/sitemap.xml.gz
index 6e72ddc1d5217262f1ed4fbcf43fbdc271e854e3..f7a96fba3ac2e780be8f0b60cc3f15217edf676c 100644
GIT binary patch
delta 945
zcmV;i15W&^2&xDNABzYGfVR7l2OkdGunmce)kV5J>;p-QJ&`m$D_y)^y}IkYAqZD|
zv!nY(8$$PeQEa!{CFuZGcCw?#qM2O}FVlEgAqgifP$w1S
zKefM2i#-5J>{r0@uHT5q8n6FEl}_fRGX9m
zp84BJbA*u9ARU|Qf8jzNR0Z2>a&3?s17#8Z!jNK1?cyN3N$qFMiI`3I}*Um!42A@i|ynrwZUx~2vmI}NcT$OB@uv=iQ03X7eo5FINm{s2)x)yAE4SV
zb{#lsbsdsY1dzVMtxE*~&(d}V&P|q5(KH5$q~E2IuX(;=D&c?jT5TwHQW1DCAc-f#
zP8moAK~qE`f23Ef4jhm4J{ikz9z5`SiKR~}WFJ~yLSrqDJ+OyHb5g#`gRML3&4lzL
zaK5qTfF#f-wW@Z;S$s;-#
zfLU~+G+N@$lm8ge@WTSHWb9;o43WZ+p+gK|D(djVR;;6bxQd^>&uN-w5mi^dI
z%Js{`e^U_;Cdh*~%Yx?P(v=4ff$F^uyDVtZLY!Cb#pAo*&#L!juQm{O?4`E<2T+IM
zcs?d?&D!3CF~QE7lD0F}G%qJa(%C_s>anJZa
T|NG+~g0T7;p-QK9Mv%D_gu?y}IkYAqZD|
zv!nY(8$$O(QEa!{CFuZGcCdX#R@`3mE&H)a
zU~!cAZ)CfgW*1ByC|zqM1HX$|CBNb88}Ss-%LXO|l;co`XlB>L%QRkANWw`A)JX+7
z(D||wR(p?rcAB;6DS$s1%-Bf=(ISNdL=pf8VT+z8ipN1$yr?Ci=mwQk3luyA)h1h`eGmps)J4O8_Y*l0XsC<>_mz
zggp5Au3>+bXx9H${sAhSyDFT`+gw
z^iYWhZGd)cYj)zQkNs?jC{=9u&p_0d0~&UG^S?WE!RxXQf1FfVmmoYZE%|g0#zRg(!G*+Nd(|zqV}BU#gKk3j&~3u0xx#b2dMUo
zT?dX@U5BI;0i>^R>rz3$v$UOobCac1G>t(b>36B*Yo4!|O8B3>RvU_)R0JLjNaD$`
zQwCB&&=ipff9X}L1IHu1PsZ}Q2M_#SV(F6#*@u>w&{)f35A31QoRshKVC&9$Ga>y1
zoNuf-APMwYt*V`I7N1hIv8|?d0`crqcm>6o6|j%t#riDlO5$F=j`HJFCfd8aB+=m|
zU>2QdIfo8yPE<&hu#FA5FlG4mHn0;`I=*8JviEsWe^R!y4F&P|=4~@dZg6q4
zL(!+XhD;;T3c|FXyXK^VO
zIoVN*Avdn6g290VI>?l5DzD_GKn}dn3T^qG#FOn8bUIA7II6Ca)RrjcD@R?1Wk2Qzb
zS$K;Wjvij2jVZk*>gS7zzD`0qBbtC{N_