MDEV-34062: Implement innodb_log_file_mmap on 64-bit systems

When using the default innodb_log_buffer_size=2m, mariadb-backup --backup
would spend a lot of time re-reading and re-parsing the log. For reads,
it would be beneficial to memory-map the entire ib_logfile0 to the
address space (typically 48 bits or 256 TiB) and read it from there,
both during --backup and --prepare. That is what we will be doing by
default.

We can also enable memory-mapped log writes in case the new parameter
innodb_log_file_mmap is set to ON. This could speed up I/O and allow
the log data to be shared between mariadbd and mariadb-backup --backup
in the RAM buffer.

Memory-mapped regular files differ from log_sys.is_pmem() in the way
that an open file handle to ib_logfile0 will be retained. That allows
log_t::set_mmap() to enable or disable the interface with fewer
operations.

On Linux, on log checkpoint we will invoke fallocate() with
FALLOC_FL_ZERO_RANGE and madvise() with MADV_DONTNEED in order to
reduce the memory pressure. Without the fallocate() call, using
MADV_DONTNEED could lead to reads of old garbage contents of the
circular log file when a page fault occurs while writing a record.

Most references to HAVE_PMEM or log_sys.is_pmem() are replaced with
HAVE_INNODB_MMAP or log_sys.is_mmap(). The main difference is that
PMEM skips the use of write_lock and flush_lock and uses pmem_persist(),
while the memory-mapped interface will use a combination of msync()
and fdatasync().

extra/mariabackup/xtrabackup.cc

@@ -205,6 +205,8 @@ lsn_t checkpoint_lsn_start;
 lsn_t checkpoint_no_start;
 /** whether log_copying_thread() is active; protected by recv_sys.mutex */
 static bool log_copying_running;
+/** for --backup, target LSN to copy the log to; protected by recv_sys.mutex */
+lsn_t metadata_to_lsn;
 
 uint xtrabackup_parallel;
 
@@ -236,7 +238,6 @@ my_bool opt_encrypted_backup;
 #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints"
 char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/
 static lsn_t metadata_from_lsn;
-lsn_t metadata_to_lsn;
 static lsn_t metadata_last_lsn;
 
 static ds_file_t*	dst_log_file;
@@ -282,9 +283,6 @@ my_bool xtrabackup_incremental_force_scan = FALSE;
  */
 ulong xtrabackup_innodb_force_recovery = 0;
 
-/* The flushed lsn which is read from data files */
-lsn_t	flushed_lsn= 0;
-
 ulong xb_open_files_limit= 0;
 char *xb_plugin_dir;
 char *xb_plugin_load;
@@ -1329,6 +1327,9 @@ enum options_xtrabackup
   OPT_INNODB_BUFFER_POOL_FILENAME,
   OPT_INNODB_LOCK_WAIT_TIMEOUT,
   OPT_INNODB_LOG_BUFFER_SIZE,
+#ifdef HAVE_INNODB_MMAP
+  OPT_INNODB_LOG_FILE_MMAP,
+#endif
 #if defined __linux__ || defined _WIN32
   OPT_INNODB_LOG_FILE_BUFFERING,
 #endif
@@ -1890,6 +1891,13 @@ struct my_option xb_server_options[] =
    (G_PTR*) &log_sys.buf_size, (G_PTR*) &log_sys.buf_size, 0,
    GET_UINT, REQUIRED_ARG, 2U << 20,
    2U << 20, log_sys.buf_size_max, 0, 4096, 0},
+#ifdef HAVE_INNODB_MMAP
+  {"innodb_log_file_mmap", OPT_INNODB_LOG_FILE_SIZE,
+   "Whether ib_logfile0 should be memory-mapped",
+   (G_PTR*) &log_sys.log_mmap,
+   (G_PTR*) &log_sys.log_mmap, 0, GET_BOOL, NO_ARG,
+   TRUE, 0, 0, 0, 0, 0},
+#endif
 #if defined __linux__ || defined _WIN32
   {"innodb_log_file_buffering", OPT_INNODB_LOG_FILE_BUFFERING,
    "Whether the file system cache for ib_logfile0 is enabled during --backup",
@@ -3360,25 +3368,126 @@ static my_bool xtrabackup_copy_datafile(ds_ctxt *ds_data,
 	return(FALSE);
 }
 
+#ifdef HAVE_INNODB_MMAP
+static int
+xtrabackup_copy_mmap_snippet(ds_file_t *ds, const byte *start, const byte *end)
+{
+  if (UNIV_UNLIKELY(start > end))
+  {
+    if (int r= ds_write(ds, start, log_sys.buf + log_sys.file_size - start))
+      return r;
+    start= log_sys.buf + log_sys.START_OFFSET;
+  }
+  return ds_write(ds, start, end - start);
+}
+
+/** Copy memory-mapped log until the end of the log is reached
+or the log_copying_stop signal is received
+@return whether the operation failed */
+static bool xtrabackup_copy_mmap_logfile()
+{
+  mysql_mutex_assert_owner(&recv_sys.mutex);
+  recv_sys.offset= size_t(log_sys.calc_lsn_offset(recv_sys.lsn));
+  recv_sys.len= size_t(log_sys.file_size);
+  const size_t seq_offset{log_sys.is_encrypted() ? 8U + 5U : 5U};
+  const char one{'\1'};
+
+  for (unsigned retry_count{0};;)
+  {
+    recv_sys_t::parse_mtr_result r;
+    const byte *start= &log_sys.buf[recv_sys.offset];
+
+    if (recv_sys.parse_mmap<false>(false) == recv_sys_t::OK)
+    {
+      const byte *end;
+
+      do
+      {
+        /* Set the sequence bit (the backed-up log will not wrap around) */
+        size_t seqo= recv_sys.offset - seq_offset;
+        if (seqo < log_sys.START_OFFSET)
+          seqo+= log_sys.file_size - log_sys.START_OFFSET;
+        const byte *seq= &log_sys.buf[seqo];
+        ut_ad(*seq == log_sys.get_sequence_bit(recv_sys.lsn - seq_offset));
+        if (!*seq)
+        {
+          if (xtrabackup_copy_mmap_snippet(dst_log_file, start, seq) ||
+              ds_write(dst_log_file, &one, 1))
+            goto write_error;
+          start = seq + 1;
+        }
+      }
+      while ((r= recv_sys.parse_mmap<false>(false)) == recv_sys_t::OK);
+
+      end= &log_sys.buf[recv_sys.offset];
+
+      if (xtrabackup_copy_mmap_snippet(dst_log_file, start, end))
+      {
+      write_error:
+        msg("Error: write to ib_logfile0 failed");
+        return true;
+      }
+
+      start= end;
+
+      pthread_cond_broadcast(&scanned_lsn_cond);
+
+      if (r == recv_sys_t::GOT_EOF)
+        break;
+
+      retry_count= 0;
+    }
+    else
+    {
+      if (metadata_to_lsn)
+      {
+        if (metadata_to_lsn <= recv_sys.lsn)
+          return false;
+      }
+      else if (xtrabackup_throttle && io_ticket-- < 0)
+        mysql_cond_wait(&wait_throttle, &recv_sys.mutex);
+
+      if (!retry_count++)
+        msg("Retrying read of log at LSN=" LSN_PF, recv_sys.lsn);
+      else if (retry_count == 100)
+        break;
+      else
+      {
+        timespec abstime;
+        set_timespec_nsec(abstime, 1000000ULL /* 1 ms */);
+        if (!mysql_cond_timedwait(&log_copying_stop, &recv_sys.mutex,
+                                  &abstime))
+          return true;
+      }
+    }
+  }
+
+  if (verbose)
+    msg(">> log scanned up to (" LSN_PF ")", recv_sys.lsn);
+  return false;
+}
+#endif
+
 /** Copy redo log until the current end of the log is reached
-@return	whether the operation failed */
+@return whether the operation failed */
 static bool xtrabackup_copy_logfile()
 {
   mysql_mutex_assert_owner(&recv_sys.mutex);
   DBUG_EXECUTE_IF("log_checksum_mismatch", return false;);
 
   ut_a(dst_log_file);
   ut_ad(recv_sys.is_initialised());
+
+#ifdef HAVE_INNODB_MMAP
+  if (log_sys.is_mmap())
+    return xtrabackup_copy_mmap_logfile();
+#endif
   const size_t sequence_offset{log_sys.is_encrypted() ? 8U + 5U : 5U};
   const size_t block_size_1{log_sys.get_block_size() - 1};
 
-  ut_ad(!log_sys.is_pmem());
-
-  {
-    recv_sys.offset= size_t(recv_sys.lsn - log_sys.get_first_lsn()) &
-      block_size_1;
-    recv_sys.len= 0;
-  }
+  recv_sys.offset= size_t(recv_sys.lsn - log_sys.get_first_lsn()) &
+    block_size_1;
+  recv_sys.len= 0;
 
   for (unsigned retry_count{0};;)
   {
@@ -5299,9 +5408,8 @@ static bool xtrabackup_backup_func()
 		goto fail;
 	}
 
-	if (!log_sys.create()) {
-		goto fail;
-	}
+	log_sys.create();
+
 	/* get current checkpoint_lsn */
 	{
 		log_sys.latch.wr_lock(SRW_LOCK_CALL);
@@ -6658,9 +6766,7 @@ static bool xtrabackup_prepare_func(char** argv)
 		}
 
 		recv_sys.create();
-		if (!log_sys.create()) {
-			goto error;
-		}
+		log_sys.create();
 		recv_sys.recovery_on = true;
 
 		xb_fil_io_init();

mysql-test/suite/innodb/r/log_file_size_online.result

@@ -20,6 +20,14 @@ Variable_name	Value
 innodb_log_file_size	4194304
 FOUND 1 /InnoDB: Resized log to 4\.000MiB/ in mysqld.1.err
 UPDATE t SET b='' WHERE a<10;
+SET @save=@@GLOBAL.innodb_log_file_buffering;
+SET GLOBAL innodb_log_file_buffering=OFF;
+SET GLOBAL innodb_log_file_buffering=ON;
+SET GLOBAL innodb_log_file_buffering=@save;
+SET @save=@@GLOBAL.innodb_log_file_mmap;
+SET GLOBAL innodb_log_file_mmap=OFF;
+SET GLOBAL innodb_log_file_mmap=ON;
+SET GLOBAL innodb_log_file_mmap=@save;
 SET GLOBAL innodb_log_file_size=5242880;
 SHOW VARIABLES LIKE 'innodb_log_file_size';
 Variable_name	Value

mysql-test/suite/innodb/t/log_file_size_online.test

@@ -27,6 +27,23 @@ let SEARCH_PATTERN = InnoDB: Resized log to 4\\.000MiB;
 
 UPDATE t SET b='' WHERE a<10;
 
+--error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET @save=@@GLOBAL.innodb_log_file_buffering;
+--error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET GLOBAL innodb_log_file_buffering=OFF;
+--error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET GLOBAL innodb_log_file_buffering=ON;
+--error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET GLOBAL innodb_log_file_buffering=@save;
+--error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET @save=@@GLOBAL.innodb_log_file_mmap;
+--error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET GLOBAL innodb_log_file_mmap=OFF;
+--error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET GLOBAL innodb_log_file_mmap=ON;
+--error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET GLOBAL innodb_log_file_mmap=@save;
+
 SET GLOBAL innodb_log_file_size=5242880;
 SHOW VARIABLES LIKE 'innodb_log_file_size';
 SELECT global_value FROM information_schema.system_variables

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -4,6 +4,7 @@ variable_name not in (
 'innodb_numa_interleave',           # only available WITH_NUMA
 'innodb_evict_tables_on_commit_debug', # one may want to override this
 'innodb_use_native_aio',            # default value depends on OS
+'innodb_log_file_mmap',             # only available on 64-bit
 'innodb_log_file_buffering',        # only available on Linux and Windows
 'innodb_buffer_pool_load_pages_abort')            # debug build only, and is only for testing
 order by variable_name;

mysql-test/suite/sys_vars/t/sysvars_innodb.test

@@ -11,6 +11,7 @@ select VARIABLE_NAME, SESSION_VALUE, DEFAULT_VALUE, VARIABLE_SCOPE, VARIABLE_TYP
     'innodb_numa_interleave',           # only available WITH_NUMA
     'innodb_evict_tables_on_commit_debug', # one may want to override this
     'innodb_use_native_aio',            # default value depends on OS
+    'innodb_log_file_mmap',             # only available on 64-bit
     'innodb_log_file_buffering',        # only available on Linux and Windows
     'innodb_buffer_pool_load_pages_abort')            # debug build only, and is only for testing
   order by variable_name;