Merge branch 'PXB-8.4-3269-reduced-lock' into PXB-trunk-3269-reduced-…

…lock

storage/innobase/fsp/fsp0file.cc

@@ -122,9 +122,17 @@ dberr_t Datafile::open_read_only(bool strict) {
   }
 
   set_open_flags(OS_FILE_OPEN);
+
+#ifdef XTRABACKUP
+  m_handle = os_file_create(
+      innodb_data_file_key, m_filepath,
+      OS_FILE_OPEN | OS_FILE_ON_ERROR_SILENT | OS_FILE_ON_ERROR_NO_EXIT,
+      OS_FILE_NORMAL, OS_DATA_FILE, OS_FILE_READ_ONLY, &success);
+#else
   m_handle = os_file_create_simple_no_error_handling(
       innodb_data_file_key, m_filepath, m_open_flags, OS_FILE_READ_ONLY, true,
       &success);
+#endif /* XTRABACKUP */
 
   if (success) {
     m_exists = true;
@@ -666,17 +674,13 @@ dberr_t Datafile::validate_first_page(space_id_t space_id, lsn_t *flush_lsn,
       bool found = false;
 
       if (srv_backup_mode) {
-        mutex_enter(&recv_sys->mutex);
-        if (recv_sys->keys != nullptr) {
-          for (const auto &recv_key : *recv_sys->keys) {
-            if (recv_key.space_id == m_space_id) {
-              memcpy(m_encryption_key, recv_key.ptr, Encryption::KEY_LEN);
-              memcpy(m_encryption_iv, recv_key.iv, Encryption::KEY_LEN);
-              found = true;
-            }
-          }
+        recv_sys_t::Encryption_Key *recv_key = nullptr;
+        std::tie(found, recv_key) = recv_find_encryption_key(m_space_id);
+
+        if (found) {
+          memcpy(m_encryption_key, recv_key->ptr, Encryption::KEY_LEN);
+          memcpy(m_encryption_iv, recv_key->iv, Encryption::KEY_LEN);
         }
-        mutex_exit(&recv_sys->mutex);
       }
 
       if (!found) {

storage/innobase/fsp/fsp0sysspace.cc

@@ -929,6 +929,14 @@ dberr_t SysTablespace::open_or_create(bool is_temp, bool create_new_db,
     }
   }
 
+#ifdef XTRABACKUP
+  // Add system tablespace for tracking purpose. We might have
+  // to recopy it
+  if (ddl_tracker && opt_lock_ddl == LOCK_DDL_REDUCED && !is_server_locked()) {
+    ddl_tracker->add_table_from_ibd_scan(space->id, space->name, space->flags);
+  }
+#endif /* XTRABACKUP */
+
   return (err);
 }
 #endif /* !UNIV_HOTBACKUP */

storage/innobase/include/fil0fil.h

@@ -1385,12 +1385,16 @@ void fil_space_set_imported(space_id_t space_id);
 @param[in]      is_raw          true if a raw device or a raw disk partition
 @param[in]      atomic_write    true if the file has atomic write enabled
 @param[in]      max_pages       maximum number of pages in file
-@return pointer to the file name
-@retval nullptr if error */
-[[nodiscard]] char *fil_node_create(const char *name, page_no_t size,
-                                    fil_space_t *space, bool is_raw,
-                                    bool atomic_write,
-                                    page_no_t max_pages = PAGE_NO_MAX);
+@return DB_SUCCESS if success, else other DB_* for errors */
+[[nodiscard]]
+#ifndef XTRABACKUP
+char *
+#else
+dberr_t
+#endif /* XTRABACKUP */
+fil_node_create(const char *name, page_no_t size, fil_space_t *space,
+                bool is_raw, bool atomic_write,
+                page_no_t max_pages = PAGE_NO_MAX);
 
 /** Create a space memory object and put it to the fil_system hash table.
 The tablespace name is independent from the tablespace file-name.
@@ -2111,27 +2115,25 @@ inline fil_space_t *fil_space_get_sys_space() {
 @param[in]      page_id         Tablespace Id and first page in file
 @param[in]      parsed_bytes    Number of bytes parsed so far
 @param[in]      parse_only      Don't apply, parse only
+@param[in]      record_lsn       LSN of the redo record
 @return pointer to next redo log record
 @retval nullptr if this log record was truncated */
-[[nodiscard]] const byte *fil_tablespace_redo_create(const byte *ptr,
-                                                     const byte *end,
-                                                     const page_id_t &page_id,
-                                                     ulint parsed_bytes,
-                                                     bool parse_only);
+[[nodiscard]] const byte *fil_tablespace_redo_create(
+    const byte *ptr, const byte *end, const page_id_t &page_id,
+    ulint parsed_bytes, bool parse_only IF_XB(, lsn_t record_lsn));
 
 /** Redo a tablespace delete.
 @param[in]      ptr             redo log record
 @param[in]      end             end of the redo log buffer
 @param[in]      page_id         Tablespace Id and first page in file
 @param[in]      parsed_bytes    Number of bytes parsed so far
 @param[in]      parse_only      Don't apply, parse only
+@param[in]      record_lsn       LSN of the redo record
 @return pointer to next redo log record
 @retval nullptr if this log record was truncated */
-[[nodiscard]] const byte *fil_tablespace_redo_delete(const byte *ptr,
-                                                     const byte *end,
-                                                     const page_id_t &page_id,
-                                                     ulint parsed_bytes,
-                                                     bool parse_only);
+[[nodiscard]] const byte *fil_tablespace_redo_delete(
+    const byte *ptr, const byte *end, const page_id_t &page_id,
+    ulint parsed_bytes, bool parse_only IF_XB(, lsn_t record_lsn));
 
 /** Redo a tablespace rename.
 This function doesn't do anything, simply parses the redo log record.
@@ -2140,13 +2142,12 @@ This function doesn't do anything, simply parses the redo log record.
 @param[in]      page_id         Tablespace Id and first page in file
 @param[in]      parsed_bytes    Number of bytes parsed so far
 @param[in]      parse_only      Don't apply, parse only
+@param[in]      record_lsn       LSN of the redo record
 @return pointer to next redo log record
 @retval nullptr if this log record was truncated */
-[[nodiscard]] const byte *fil_tablespace_redo_rename(const byte *ptr,
-                                                     const byte *end,
-                                                     const page_id_t &page_id,
-                                                     ulint parsed_bytes,
-                                                     bool parse_only);
+[[nodiscard]] const byte *fil_tablespace_redo_rename(
+    const byte *ptr, const byte *end, const page_id_t &page_id,
+    ulint parsed_bytes, bool parse_only IF_XB(, lsn_t record_lsn));
 
 /** Redo a tablespace extend
 @param[in]      ptr             redo log record
@@ -2247,16 +2248,26 @@ void fil_set_scan_dir(const std::string &directory, bool is_undo_dir = false);
 void fil_set_scan_dirs(const std::string &directories);
 
 /** Discover tablespaces by reading the header from .ibd files.
-@param[in]  populate_fil_cache Whether to load tablespaces into fil cache
+@param[in] populate_fil_cache  Whether to load tablespaces into fil cache
+@param[in] only_undo           if true, only the undo tablespaces are discovered
 @return DB_SUCCESS if all goes well */
-dberr_t fil_scan_for_tablespaces(bool populate_fil_cache);
+dberr_t fil_scan_for_tablespaces(bool populate_fil_cache IF_XB(,
+                                                               bool only_undo));
 
 /** Open the tablespace and also get the tablespace filenames, space_id must
 already be known.
 @param[in]  space_id  Tablespace ID to lookup
 @return DB_SUCCESS if open was successful */
 [[nodiscard]] dberr_t fil_tablespace_open_for_recovery(space_id_t space_id);
 
+#ifdef XTRABACKUP
+/** This function is a wrapper to fil_tablespace_open_for_recovery(), used when
+ * LOCK_DDL_REDUCED is ON
+@param[in]  path  File path
+@return DB_SUCCESS if open was successful */
+dberr_t fil_open_for_reduced(const std::string &path);
+#endif /* XTRABACKUP */
+
 /** Replay a file rename operation for ddl replay.
 @param[in]      page_id         Space ID and first page number in the file
 @param[in]      old_name        old file name
@@ -2362,6 +2373,12 @@ stage. This is used at rollback phase
 @param[in]	space_id	tablespace id
 @return DB_ERROR_UNSET if no error occured, else DB_* error */
 dberr_t fil_xb_get_tablespace_error(space_id_t space_id);
+
+/** Get the tablespace ID from an .ibd and/or an undo tablespace. If the ID is 0
+on the first page then try finding the ID with Datafile::find_space_id().
+@param[in]      filename        File name to check
+@return s_invalid_space_id if not found, otherwise the space ID */
+space_id_t fil_get_tablespace_id(const std::string &filename);
 #endif /* XTRABACKUP */
 
 dberr_t fil_prepare_file_for_io(space_id_t space_id, page_no_t &page_no,
@@ -2375,4 +2392,14 @@ inline bool fil_node_t::is_offset_valid(os_offset_t byte_offset) const {
   return byte_offset < max_offset;
 }
 
+#ifdef XTRABACKUP
+/** Frees a space object from the tablespace memory cache.
+Closes a tablespaces files but does not delete them.
+There must not be any pending i/o's or flushes on the files.
+@param[in]      space_id        Tablespace ID
+@param[in]      x_latched       Whether the caller holds X-mode space->latch
+@return true if success */
+bool fil_space_free(space_id_t space_id, bool x_latched);
+#endif /* XTRABACKUP */
+
 #endif /* fil0fil_h */

storage/innobase/include/log0recv.h

@@ -796,6 +796,9 @@ bool recv_sys_resize_buf();
 hash table to wait merging to file pages. */
 void recv_parse_log_recs();
 
+std::tuple<bool, recv_sys_t::Encryption_Key *> recv_find_encryption_key(
+    space_id_t space_id);
+
 #include "log0recv.ic"
 
 #endif

storage/innobase/include/os0file.h

@@ -1844,10 +1844,14 @@ class Dir_Walker {
  public:
   using Path = std::string;
 
+#ifdef XTRABACKUP
+  static std::tuple<bool, os_file_type_t> is_directory(const Path &path);
+#else
   /** Check if the path is a directory. The file/directory must exist.
   @param[in]    path            The path to check
   @return true if it is a directory */
   static bool is_directory(const Path &path);
+#endif /* XTRABACKUP */
 
   /** Depth first traversal of the directory starting from basedir
   @param[in]  basedir    Start scanning from this directory

storage/innobase/include/xb0xb.h

@@ -20,21 +20,25 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 
 #ifndef xb0xb_h
 #define xb0xb_h
+#include "storage/innobase/xtrabackup/src/ddl_tracker.h"
 
 extern bool innodb_log_checksums_specified;
 extern bool innodb_checksum_algorithm_specified;
 
 extern bool opt_lock_ddl_per_table;
-extern bool opt_lock_ddl;
 extern bool redo_catchup_completed;
+extern bool xtrabackup_prepare;
 extern bool opt_page_tracking;
 extern char *xtrabackup_incremental;
 extern lsn_t incremental_start_checkpoint_lsn;
 extern lsn_t xtrabackup_start_checkpoint;
 extern bool use_dumped_tablespace_keys;
 extern unsigned long xb_backup_version;
 extern bool xb_generated_redo;
+enum lock_ddl_type_t { LOCK_DDL_OFF, LOCK_DDL_ON, LOCK_DDL_REDUCED };
+extern lock_ddl_type_t opt_lock_ddl;
 
+extern ddl_tracker_t *ddl_tracker;
 extern std::vector<ulint> invalid_encrypted_tablespace_ids;
 
 /** Fetch tablespace key from "xtrabackup_keys".
@@ -103,6 +107,21 @@ const std::string KEYRING_NOT_LOADED =
 @param[in]	name	sync point name */
 void debug_sync_point(const char *name);
 
+#ifdef UNIV_DEBUG
+/** Pause xtrabackup thread and wait for resume.
+Thread can be resumed by deleting the sync_point filename
+@param[in]	name	sync point name */
+void debug_sync_thread(const char *name);
+#else
+#define debug_sync_thread(A)
+#endif /* UNIV_DEBUG */
+
 extern char *xtrabackup_debug_sync;
 
+/** @return true if xtrabackup has locked Server with LOCK INSTANCE FOR BACKP or
+LOCK TABLES FOR BACKUP */
+bool is_server_locked();
+
+bool xb_check_and_set_open_files_limit(size_t num_files);
+
 #endif

storage/innobase/log/log0recv.cc

@@ -1669,7 +1669,8 @@ static const byte *recv_parse_or_apply_log_rec_body(
 #ifdef XTRABACKUP
       /* error out backup if undo truncation happens during backup */
       if (srv_backup_mode && fsp_is_undo_tablespace(space_id) &&
-          backup_redo_log_flushed_lsn < recv_sys->recovered_lsn) {
+          backup_redo_log_flushed_lsn < recv_sys->recovered_lsn &&
+          ddl_tracker == nullptr) {
         xb::info() << "Last flushed lsn: " << backup_redo_log_flushed_lsn
                    << " undo_delete lsn " << recv_sys->recovered_lsn;
 
@@ -1688,24 +1689,32 @@ static const byte *recv_parse_or_apply_log_rec_body(
       return fil_tablespace_redo_delete(
           ptr, end_ptr, page_id_t(space_id, page_no), parsed_bytes,
           recv_sys->bytes_to_ignore_before_checkpoint !=
-              0 IF_XB(|| recv_sys->recovered_lsn + parsed_bytes <
-                             backup_redo_log_flushed_lsn));
+              0 IF_XB(||
+                      recv_sys->recovered_lsn + parsed_bytes <
+                          backup_redo_log_flushed_lsn ||
+                      opt_lock_ddl == LOCK_DDL_REDUCED),
+          start_lsn);
 
     case MLOG_FILE_CREATE:
 
       return fil_tablespace_redo_create(
           ptr, end_ptr, page_id_t(space_id, page_no), parsed_bytes,
           recv_sys->bytes_to_ignore_before_checkpoint !=
-              0 IF_XB(|| recv_sys->recovered_lsn + parsed_bytes <
-                             backup_redo_log_flushed_lsn));
+              0 IF_XB(||
+                      recv_sys->recovered_lsn + parsed_bytes <
+                          backup_redo_log_flushed_lsn ||
+                      opt_lock_ddl == LOCK_DDL_REDUCED),
+          start_lsn);
 
     case MLOG_FILE_RENAME:
-
       return fil_tablespace_redo_rename(
           ptr, end_ptr, page_id_t(space_id, page_no), parsed_bytes,
           recv_sys->bytes_to_ignore_before_checkpoint !=
-              0 IF_XB(|| recv_sys->recovered_lsn + parsed_bytes <
-                             backup_redo_log_flushed_lsn));
+              0 IF_XB(||
+                      recv_sys->recovered_lsn + parsed_bytes <
+                          backup_redo_log_flushed_lsn ||
+                      opt_lock_ddl == LOCK_DDL_REDUCED),
+          start_lsn);
 
     case MLOG_FILE_EXTEND:
 
@@ -1749,27 +1758,32 @@ static const byte *recv_parse_or_apply_log_rec_body(
       if (!recv_recovery_on) {
         if (redo_catchup_completed) {
           if (backup_redo_log_flushed_lsn < recv_sys->recovered_lsn) {
-            xb::info() << "Last flushed lsn: " << backup_redo_log_flushed_lsn
-                       << " load_index lsn " << recv_sys->recovered_lsn;
-
-            if (backup_redo_log_flushed_lsn == 0) {
-              xb::error(ER_IB_MSG_715) << "PXB was not able"
-                                       << " to determine the"
-                                       << " InnoDB Engine"
-                                       << " Status";
+            if (ddl_tracker) {
+              ddl_tracker->backup_file_op(space_id, MLOG_INDEX_LOAD, nullptr, 0,
+                                          start_lsn);
+            } else {
+              xb::info() << "Last flushed lsn: " << backup_redo_log_flushed_lsn
+                         << " load_index lsn " << recv_sys->recovered_lsn;
+
+              if (backup_redo_log_flushed_lsn == 0) {
+                xb::error(ER_IB_MSG_715) << "PXB was not able"
+                                         << " to determine the"
+                                         << " InnoDB Engine"
+                                         << " Status";
+              }
+
+              xb::error(ER_IB_MSG_716) << "An optimized (without"
+                                       << " redo logging) DDL"
+                                       << " operation has been"
+                                       << " performed. All modified"
+                                       << " pages may not have been"
+                                       << " flushed to the disk yet.\n"
+                                       << "    PXB will not be able to"
+                                       << " take a consistent backup."
+                                       << " Retry the backup"
+                                       << " operation";
+              exit(EXIT_FAILURE);
             }
-
-            xb::error(ER_IB_MSG_716) << "An optimized (without"
-                                     << " redo logging) DDL"
-                                     << " operation has been"
-                                     << " performed. All modified"
-                                     << " pages may not have been"
-                                     << " flushed to the disk yet.\n"
-                                     << "    PXB will not be able to"
-                                     << " take a consistent backup."
-                                     << " Retry the backup"
-                                     << " operation";
-            exit(EXIT_FAILURE);
           }
           /** else the index is flushed to disk before
           backup started hence no error */
@@ -1832,6 +1846,10 @@ static const byte *recv_parse_or_apply_log_rec_body(
               recv_sys->recovered_lsn > incremental_start_checkpoint_lsn) {
             full_scan_tables.insert(space_id);
           }
+
+          if (ddl_tracker && redo_catchup_completed)
+            ddl_tracker->backup_file_op(space_id, MLOG_WRITE_STRING, nullptr, 0,
+                                        start_lsn);
 #endif /* XTRABACKUP */
 
           ut_ad(LSN_MAX != start_lsn);
@@ -4702,3 +4720,20 @@ const char *get_mlog_string(mlog_id_t type) {
   return nullptr;
 }
 #endif /* UNIV_DEBUG || UNIV_HOTBACKUP */
+
+std::tuple<bool, recv_sys_t::Encryption_Key *> recv_find_encryption_key(
+    space_id_t space_id) {
+  if (recv_sys == nullptr || recv_sys->keys == nullptr) {
+    return {false, nullptr};
+  }
+
+  mutex_enter(&recv_sys->mutex);
+  for (auto &recv_key : *recv_sys->keys) {
+    if (recv_key.space_id == space_id) {
+      mutex_exit(&recv_sys->mutex);
+      return {true, &recv_key};
+    }
+  }
+  mutex_exit(&recv_sys->mutex);
+  return {false, nullptr};
+}