refactor: Change recursive_mutex to mutex in DatabaseRotatingImp

src/xrpld/nodestore/DatabaseRotating.h

@@ -44,7 +44,7 @@ class DatabaseRotating : public Database
 
     /** Rotates the backends.
 
-        @param f A function executed before the rotation and under the same lock
+        @param f A function executed before the rotation
     */
     virtual void
     rotateWithLock(std::function<std::unique_ptr<NodeStore::Backend>(

src/xrpld/nodestore/detail/DatabaseRotatingImp.cpp

@@ -45,9 +45,19 @@ DatabaseRotatingImp::rotateWithLock(
     std::function<std::unique_ptr<NodeStore::Backend>(
         std::string const& writableBackendName)> const& f)
 {
+    // backendMutex_ prevents writableBackend_ and archiveBackend_ from changing
+    // while the main mutex_ is released during the callback.
+    std::lock_guard backendLock(backendMutex_);
+
+    auto const writableBackend = [&] {
+        std::lock_guard lock(mutex_);
+        return writableBackend_;
+    }();
+
+    auto newBackend = f(writableBackend->getName());
+
     std::lock_guard lock(mutex_);
 
-    auto newBackend = f(writableBackend_->getName());
     archiveBackend_->setDeletePath();
     archiveBackend_ = std::move(writableBackend_);
     writableBackend_ = std::move(newBackend);

src/xrpld/nodestore/detail/DatabaseRotatingImp.h

@@ -80,15 +80,13 @@ class DatabaseRotatingImp : public DatabaseRotating
     sweep() override;
 
 private:
+    // backendMutex_ is only needed when the *Backend_ members are modified.
+    // Reads are protected by the general mutex_.
+    std::mutex backendMutex_;
     std::shared_ptr<Backend> writableBackend_;
     std::shared_ptr<Backend> archiveBackend_;
-    // This needs to be a recursive mutex because callbacks in `rotateWithLock`
-    // can call function that also lock the mutex. A current example of this is
-    // a callback from SHAMapStoreImp, which calls `clearCaches`. This
-    // `clearCaches` call eventually calls `fetchNodeObject` which tries to
-    // relock the mutex. It would be desirable to rewrite the code so the lock
-    // was not held during a callback.
-    mutable std::recursive_mutex mutex_;
+
+    mutable std::mutex mutex_;
 
     std::shared_ptr<NodeObject>
     fetchNodeObject(