PXB-3220 tolerate file deletion between discovery and file open

storage/innobase/fil/fil0fil.cc

@@ -1610,7 +1610,6 @@ class Fil_system {
   @param[in]  f   Callback
   @return any error returned by the callback function. */
   [[nodiscard]] dberr_t iterate_spaces(Fil_space_iterator::Function &f);
-
 #endif /* XTRABACKUP */
   /** Iterate through all persistent tablespace files
   (FIL_TYPE_TABLESPACE) returning the nodes via callback function cbk.
@@ -4097,7 +4096,6 @@ void fil_node_close_file(fil_node_t *file) {
 
   shard->mutex_release();
 }
-
 #endif /* XTRABACKUP */
 
 /** Iterate through all tablespaces
@@ -11593,7 +11591,6 @@ void Tablespace_dirs::open_ibd(const Const_iter &start, const Const_iter &end,
                                size_t thread_id, bool &result) {
   if (!result) return;
 
-  uint8_t max_attempt_retries = (opt_lock_ddl == LOCK_DDL_REDUCED) ? 10 : 1;
   for (auto it = start; it != end; ++it) {
     const std::string filename = it->second;
     const auto &files = m_dirs[it->first];
@@ -11606,17 +11603,19 @@ void Tablespace_dirs::open_ibd(const Const_iter &start, const Const_iter &end,
     /* cannot use auto [err, space_id] = fil_open_for_xtrabackup() as space_id
     is unused here and we get unused variable error during compilation */
     dberr_t err;
-    uint8_t attempts = 0;
-    while (attempts < max_attempt_retries) {
-      attempts++;
-      std::tie(err, std::ignore) = fil_open_for_xtrabackup(
-          phy_filename, filename.substr(0, filename.length() - 4));
-      /* PXB-2275 - Allow DB_INVALID_ENCRYPTION_META as we will test it in the
-      end of the backup */
-      if (err == DB_SUCCESS || err == DB_INVALID_ENCRYPTION_META) break;
-
-      if (attempts == max_attempt_retries) result = false;
-    }
+    std::tie(err, std::ignore) = fil_open_for_xtrabackup(
+        phy_filename, filename.substr(0, filename.length() - 4));
+
+    /* Allow deleted tables between disovery and file open when
+     LOCK_DDL_REDUCED, they will be handled by ddl_tracker */
+    if (err == DB_CANNOT_OPEN_FILE && opt_lock_ddl == LOCK_DDL_REDUCED) {
+      ddl_tracker->add_missing_table(phy_filename);
+    } else
+      /* PXB-2275 - Allow DB_INVALID_ENCRYPTION_META as we will test it in
+      the end of the backup */
+      if (err != DB_SUCCESS && err != DB_INVALID_ENCRYPTION_META) {
+        result = false;
+      }
   }
 }
 

storage/innobase/xtrabackup/src/ddl_tracker.cc

@@ -121,6 +121,21 @@ void ddl_tracker_t::add_table(const space_id_t &space_id, std::string name) {
   tables_in_backup[space_id] = name;
 }
 
+void ddl_tracker_t::add_missing_table(std::string path) {
+  Fil_path::normalize(path);
+  if (Fil_path::has_prefix(path, Fil_path::DOT_SLASH)) {
+    path.erase(0, 2);
+  }
+  missing_tables.insert(path);
+}
+
+bool ddl_tracker_t::is_missing_table(const std::string &name) {
+  if (missing_tables.count(name)) {
+    return true;
+  }
+  return false;
+}
+
 /* ======== Data copying thread context ======== */
 
 typedef struct {
@@ -226,8 +241,17 @@ void ddl_tracker_t::handle_ddl_operations() {
     if (check_if_skip_table(table.second.first.c_str())) {
       continue;
     }
-    /* renamed new table. update new table entry to renamed table name */
-    if (new_tables.find(table.first) != new_tables.end()) {
+
+    /* renamed new table. update new table entry to renamed table name
+      or if table is missing and renamed, add the renamed table to the new_table
+      list. for example: 1. t1.ibd is discovered
+                   2. t1.ibd renamed to t2.ibd
+                   3. t2.ibd is opened and loaded to cache to copy
+                   4. t1.ibd is missing now
+      so we should add t2.ibd to new_tables and skip .ren file so that we don't
+      try to rename t1.ibd to t2.idb where t1.ibd is missing   */
+    if (new_tables.find(table.first) != new_tables.end() ||
+        is_missing_table(table.second.first)) {
       new_tables[table.first] = table.second.second;
       continue;
     }

storage/innobase/xtrabackup/src/ddl_tracker.h

@@ -35,6 +35,8 @@ class ddl_tracker_t {
   space_id_to_name_t drops;
   /* For DDL operation found in redo log,  */
   std::unordered_map<space_id_t, std::pair<std::string, std::string>> renames;
+  /** Tables that have been deleted between discovery and file open */
+  std::unordered_set<std::string> missing_tables;
 
  public:
   /** Add a new table in the DDL tracker table list.
@@ -52,5 +54,13 @@ class ddl_tracker_t {
                       ulint len, lsn_t start_lsn);
   /** Function responsible to generate files based on DDL operations */
   void handle_ddl_operations();
+
+  /** Note that a table has been deleted between disovery and file open
+  @param[in]  path  missing table name with path. */
+  void add_missing_table(std::string path);
+
+  /** Check if table is in missing list
+  @param[in]  name  tablespace name */
+  bool is_missing_table(const std::string &name);
 };
 #endif  // DDL_TRACKER_H