PS-10595 [trunk]: Add innodb_large_page_populate to control huge page pre-population

mysql-test/suite/sys_vars/r/all_vars.result

@@ -48,6 +48,8 @@ init_slave
 init_slave
 innodb_dedicated_server
 innodb_dedicated_server
+innodb_large_page_populate
+innodb_large_page_populate
 innodb_log_spin_cpu_abs_lwm
 innodb_log_spin_cpu_abs_lwm
 innodb_log_spin_cpu_pct_hwm

storage/innobase/handler/ha_innodb.cc

@@ -24170,6 +24170,15 @@ static MYSQL_SYSVAR_BOOL(
     nullptr, nullptr, true);
 #endif /* HAVE_LIBNUMA */
 
+#ifdef UNIV_LINUX
+static MYSQL_SYSVAR_BOOL(
+    large_page_populate, srv_large_page_populate, PLUGIN_VAR_NOCMDARG,
+    "Pre-populate huge / large pages used by the InnoDB buffer pool and page "
+    "allocator at allocation time. When OFF, population is skipped to reduce "
+    "startup time and NUMA overhead on large systems.",
+    nullptr, nullptr, false);
+#endif /* UNIV_LINUX */
+
 static MYSQL_SYSVAR_BOOL(
     api_enable_binlog, ib_binlog_enabled,
     PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY,
@@ -24626,6 +24635,9 @@ static SYS_VAR *innobase_system_variables[] = {
 #ifdef HAVE_LIBNUMA
     MYSQL_SYSVAR(numa_interleave),
 #endif /* HAVE_LIBNUMA */
+#ifdef UNIV_LINUX
+    MYSQL_SYSVAR(large_page_populate),
+#endif /* UNIV_LINUX */
     MYSQL_SYSVAR(change_buffering),
     MYSQL_SYSVAR(change_buffer_max_size),
 #if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG

storage/innobase/include/detail/ut/large_page_alloc.h

@@ -48,6 +48,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #include "storage/innobase/include/detail/ut/helper.h"
 #include "storage/innobase/include/detail/ut/page_metadata.h"
 #include "storage/innobase/include/detail/ut/pfs.h"
+#include "storage/innobase/include/os0populate.h"
 
 extern const size_t large_page_default_size;
 
@@ -112,7 +113,8 @@ struct Large_page_alloc : public allocator_traits<false> {
   static inline void *alloc(std::size_t size, bool populate) {
     auto total_len = round_to_next_multiple(
         size + page_allocation_metadata::len, large_page_default_size);
-    auto mem = large_page_aligned_alloc(total_len, populate);
+    bool do_populate = populate && srv_large_page_populate;
+    auto mem = large_page_aligned_alloc(total_len, do_populate);
     if (unlikely(!mem)) return nullptr;
     page_allocation_metadata::datalen(mem, total_len);
     page_allocation_metadata::page_type(mem, Page_type::large_page);
@@ -249,7 +251,8 @@ struct Large_page_alloc_pfs : public allocator_traits<true> {
       page_allocation_metadata::pfs_metadata::pfs_memory_key_t key) {
     auto total_len = round_to_next_multiple(
         size + page_allocation_metadata::len, large_page_default_size);
-    auto mem = large_page_aligned_alloc(total_len, populate);
+    bool do_populate = populate && srv_large_page_populate;
+    auto mem = large_page_aligned_alloc(total_len, do_populate);
     if (unlikely(!mem)) return nullptr;
 
 #ifdef HAVE_PSI_MEMORY_INTERFACE

storage/innobase/include/detail/ut/page_alloc.h

@@ -66,6 +66,7 @@ namespace detail {
     @return Pointer to the allocated storage. nullptr if allocation failed.
 */
 inline void *page_aligned_alloc(size_t n_bytes, bool populate) {
+  const bool do_populate = populate && srv_large_page_populate;
 #ifdef _WIN32
   // With lpAddress set to nullptr, VirtualAlloc will internally round n_bytes
   // to the multiple of system page size if it is not already
@@ -83,7 +84,7 @@ inline void *page_aligned_alloc(size_t n_bytes, bool populate) {
   // multiple of system page size if it is not already
   void *ptr =
       mmap(nullptr, n_bytes, PROT_READ | PROT_WRITE,
-           MAP_PRIVATE | MAP_ANON | (populate ? OS_MAP_POPULATE : 0), -1, 0);
+           MAP_PRIVATE | MAP_ANON | (do_populate ? OS_MAP_POPULATE : 0), -1, 0);
   if (unlikely(ptr == (void *)-1)) {
     ib::log_warn(ER_IB_MSG_856) << "page_aligned_alloc mmap(" << n_bytes
                                 << " bytes) failed;"
@@ -93,7 +94,7 @@ inline void *page_aligned_alloc(size_t n_bytes, bool populate) {
   }
 #endif
 
-  if (populate) prefault_if_not_map_populate(ptr, n_bytes);
+  if (do_populate) prefault_if_not_map_populate(ptr, n_bytes);
 
   return ptr;
 }

storage/innobase/include/os0populate.h

@@ -17,4 +17,14 @@
 void prefault_if_not_map_populate(void *ptr [[maybe_unused]],
                                   size_t n_bytes [[maybe_unused]]);
 
+#ifdef UNIV_LINUX
+/** When false, huge-page / page-aligned allocations skip pre-population
+(MAP_POPULATE and the explicit prefault step) even when the caller requests it.
+Backed by the innodb_large_page_populate system variable. */
+extern bool srv_large_page_populate;
+#else
+/* On non-Linux platforms population is always honored as requested. */
+constexpr bool srv_large_page_populate = true;
+#endif
+
 #endif

storage/innobase/srv/srv0srv.cc

@@ -225,6 +225,12 @@ bool srv_use_native_aio = false;
 
 bool srv_numa_interleave = false;
 
+#ifdef UNIV_LINUX
+/** Whether huge / page-aligned allocations honor a caller's populate request.
+Exposed as the innodb_large_page_populate system variable. */
+bool srv_large_page_populate = false;
+#endif /* UNIV_LINUX */
+
 #ifdef UNIV_DEBUG
 /** Force all user tables to use page compression. */
 ulong srv_debug_compress;

storage/innobase/srv/srv0start.cc

@@ -110,6 +110,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #include "lock0lock.h"
 #include "log0meb.h"
 #include "os0event.h"
+#include "os0populate.h"
 #include "os0proc.h"
 #include "pars0pars.h"
 #include "que0que.h"
@@ -1626,6 +1627,18 @@ dberr_t srv_start(bool create_new_db) {
 
   ib::info(ER_IB_MSG_1132);
 
+  if (srv_numa_interleave && os_use_large_pages && !srv_large_page_populate) {
+    ib::warn() << "innodb_numa_interleave is enabled together with large "
+                  "pages, but innodb_large_page_populate is OFF. Large pages "
+                  "are not subject to NUMA rebalancing and the interleave "
+                  "policy is only a hint: if a node is temporarily out of "
+                  "free memory when a page is first touched, the page can "
+                  "land on the wrong node and stay there permanently. To "
+                  "avoid this, drop the page cache before starting MySQL and "
+                  "enable innodb_large_page_populate when both "
+                  "innodb_numa_interleave and large_pages are ON.";
+  }
+
 #ifdef UNIV_DEBUG
   /* We have observed deadlocks with a 5MB buffer pool but
   the actual lower limit could very well be a little higher. */