PS-10595 [8.4]: Reduce rw_lock_list_mutex contention during buffer pool initialization

storage/innobase/buf/buf0buf.cc

@@ -869,6 +869,9 @@ static void buf_block_init(
 
   mutex_create(LATCH_ID_BUF_BLOCK_MUTEX, &block->mutex);
 
+  /* Initialize the block rw-locks WITHOUT registering them in rw_lock_list.
+  Registration is done in bulk once the whole chunk has been initialized,
+  to avoid taking rw_lock_list_mutex once per block. */
 #if defined PFS_SKIP_BUFFER_MUTEX_RWLOCK || defined PFS_GROUP_BUFFER_SYNC
   /* If PFS_SKIP_BUFFER_MUTEX_RWLOCK is defined, skip registration
   of buffer block rwlock with performance schema.
@@ -877,17 +880,19 @@ static void buf_block_init(
   since buffer block rwlock will be registered later in
   pfs_register_buffer_block(). */
 
-  rw_lock_create(PFS_NOT_INSTRUMENTED, &block->lock, LATCH_ID_BUF_BLOCK_LOCK);
+  rw_lock_init_only_inst(PFS_NOT_INSTRUMENTED, &block->lock,
+                         LATCH_ID_BUF_BLOCK_LOCK);
 
-  ut_d(rw_lock_create(PFS_NOT_INSTRUMENTED, &block->debug_latch,
-                      LATCH_ID_BUF_BLOCK_DEBUG));
+  ut_d(rw_lock_init_only_inst(PFS_NOT_INSTRUMENTED, &block->debug_latch,
+                              LATCH_ID_BUF_BLOCK_DEBUG));
 
 #else /* PFS_SKIP_BUFFER_MUTEX_RWLOCK || PFS_GROUP_BUFFER_SYNC */
 
-  rw_lock_create(buf_block_lock_key, &block->lock, LATCH_ID_BUF_BLOCK_LOCK);
+  rw_lock_init_only_inst(buf_block_lock_key, &block->lock,
+                         LATCH_ID_BUF_BLOCK_LOCK);
 
-  ut_d(rw_lock_create(buf_block_debug_latch_key, &block->debug_latch,
-                      LATCH_ID_BUF_BLOCK_DEBUG));
+  ut_d(rw_lock_init_only_inst(buf_block_debug_latch_key, &block->debug_latch,
+                              LATCH_ID_BUF_BLOCK_DEBUG));
 
 #endif /* PFS_SKIP_BUFFER_MUTEX_RWLOCK || PFS_GROUP_BUFFER_SYNC */
 
@@ -1157,11 +1162,26 @@ static buf_chunk_t *buf_chunk_init(
     frame += UNIV_PAGE_SIZE;
   }
 
+  /* Build this chunk's block rw-lock list WITHOUT holding any mutex. The
+  chunk's blocks are owned exclusively by this thread until the chunk is
+  published, so this O(n) work is lock-free. */
+  rw_lock_list_t chunk_locks;
+  UT_LIST_INIT(chunk_locks);
+  {
+    buf_block_t *lock_block = chunk->blocks;
+    for (ulint j = 0; j < chunk->size; ++j, ++lock_block) {
+      UT_LIST_ADD_LAST(chunk_locks, &lock_block->lock);
+      ut_d(UT_LIST_ADD_LAST(chunk_locks, &lock_block->debug_latch));
+    }
+  }
+
+  /* Register the chunk and rw-locks. */
   if (mutex != nullptr) {
     mutex->lock();
   }
 
   buf_pool_register_chunk(chunk);
+  rw_lock_list_register_bulk(chunk_locks); /* O(1), uses rw_lock_list_mutex */
 
   if (mutex != nullptr) {
     mutex->unlock();

storage/innobase/include/sync0rw.h

@@ -126,6 +126,18 @@ extern ib_mutex_t rw_lock_list_mutex;
  @param[in] clocation location where created */
 void rw_lock_create_func(rw_lock_t *lock,
                          IF_DEBUG(latch_id_t id, ) ut::Location clocation);
+
+/** Initializes an rw-lock object but does NOT register it in the global
+ rw_lock_list. The caller is responsible for registering it later,
+ e.g. in bulk via rw_lock_list_register_bulk(). This lets callers that
+ create many rw-locks (such as buffer pool blocks) avoid taking
+ rw_lock_list_mutex once per lock.
+ @param[in] lock pointer to memory
+ @param[in] id latch_id (debug builds only)
+ @param[in] clocation location where created */
+void rw_lock_init_only(rw_lock_t *lock,
+                       IF_DEBUG(latch_id_t id, ) ut::Location clocation);
+
 /** Calling this function is obligatory only if the memory buffer containing
  the rw-lock is freed. Removes an rw-lock object from the global list. The
  rw-lock is checked to be in the non-locked state. */
@@ -517,6 +529,19 @@ static inline void pfs_rw_lock_create_func(mysql_pfs_key_t key, rw_lock_t *lock,
                                            IF_DEBUG(latch_id_t id, )
                                                ut::Location clocation);
 
+/** Performance schema instrumented wrap function for rw_lock_init_only().
+Behaves like pfs_rw_lock_create_func() but defers registration of the lock in
+the global rw_lock_list to the caller. NOTE! Please use the corresponding macro
+rw_lock_init_only_inst(), not directly this function!
+@param[in]      key             key registered with performance schema
+@param[in]      lock            rw lock
+@param[in]      id              latch_id
+@param[in]      clocation       location where created */
+static inline void pfs_rw_lock_init_only_func(mysql_pfs_key_t key,
+                                              rw_lock_t *lock,
+                                              IF_DEBUG(latch_id_t id, )
+                                                  ut::Location clocation);
+
 /** Performance schema instrumented wrap function for rw_lock_x_lock_func()
 NOTE! Please use the corresponding macro rw_lock_x_lock(), not directly this
 function!
@@ -631,8 +656,12 @@ static inline void pfs_rw_lock_free_func(rw_lock_t *lock); /*!< in: rw-lock */
 #ifdef UNIV_DEBUG
 #define rw_lock_create(K, L, ID) \
   rw_lock_create_func((L), (ID), UT_LOCATION_HERE)
+#define rw_lock_init_only_inst(K, L, ID) \
+  rw_lock_init_only((L), (ID), UT_LOCATION_HERE)
 #else /* UNIV_DEBUG */
 #define rw_lock_create(K, L, ID) rw_lock_create_func((L), UT_LOCATION_HERE)
+#define rw_lock_init_only_inst(K, L, ID) \
+  rw_lock_init_only((L), UT_LOCATION_HERE)
 #endif /* UNIV_DEBUG */
 
 /** NOTE! The following macros should be used in rw locking and
@@ -719,9 +748,13 @@ static inline void rw_lock_x_unlock_gen(rw_lock_t *L, ulint P) {
 #ifdef UNIV_DEBUG
 #define rw_lock_create(K, L, ID) \
   pfs_rw_lock_create_func((K), (L), (ID), UT_LOCATION_HERE)
+#define rw_lock_init_only_inst(K, L, ID) \
+  pfs_rw_lock_init_only_func((K), (L), (ID), UT_LOCATION_HERE)
 #else /* UNIV_DEBUG */
 #define rw_lock_create(K, L, ID) \
   pfs_rw_lock_create_func((K), (L), UT_LOCATION_HERE)
+#define rw_lock_init_only_inst(K, L, ID) \
+  pfs_rw_lock_init_only_func((K), (L), UT_LOCATION_HERE)
 #endif /* UNIV_DEBUG */
 
 /******************************************************************
@@ -821,4 +854,13 @@ typedef UT_LIST_BASE_NODE_T(rw_lock_t, list) rw_lock_list_t;
 
 extern rw_lock_list_t rw_lock_list;
 
+#ifndef UNIV_HOTBACKUP
+/** Registers all rw-locks contained in the given list into the global
+ rw_lock_list under a single rw_lock_list_mutex critical section, in O(1) time.
+ The locks must have been prepared with rw_lock_init_only().
+ @param[in,out] locks list of locks to register; emptied on return as its nodes
+                      are spliced into rw_lock_list */
+void rw_lock_list_register_bulk(rw_lock_list_t &locks);
+#endif /* !UNIV_HOTBACKUP */
+
 #endif /* sync0rw.h */

storage/innobase/include/sync0rw.ic

@@ -476,6 +476,20 @@ static inline void pfs_rw_lock_create_func(mysql_pfs_key_t key, rw_lock_t *lock,
   /* The actual function to initialize an rwlock */
   rw_lock_create_func(lock, IF_DEBUG(id, ) clocation);
 }
+
+static inline void pfs_rw_lock_init_only_func(mysql_pfs_key_t key,
+                                              rw_lock_t *lock,
+                                              IF_DEBUG(latch_id_t id, )
+                                                  ut::Location clocation) {
+  new (lock) rw_lock_t{};
+
+  /* Initialize the rwlock for performance schema */
+  lock->pfs_psi = PSI_RWLOCK_CALL(init_rwlock)(key.m_value, lock);
+
+  /* Initialize the rwlock without registering it in rw_lock_list; the caller
+  registers it later (e.g. in bulk via rw_lock_list_register_bulk()). */
+  rw_lock_init_only(lock, IF_DEBUG(id, ) clocation);
+}
 /** Performance schema instrumented wrap function for rw_lock_x_lock_func()
  NOTE! Please use the corresponding macro rw_lock_x_lock(), not directly
  this function!

storage/innobase/include/ut0lst.h

@@ -353,6 +353,42 @@ void ut_list_append(List &list, typename List::elem_type *elem) {
  @param ELEM the element to add */
 #define UT_LIST_ADD_LAST(LIST, ELEM) ut_list_append(LIST, ELEM)
 
+/** Appends all elements of src to the end of dst in O(1) time, leaving src
+ empty. Both lists must be of the same type, i.e. chained through the same node
+ member, and an element must not be present in both lists at once.
+ @param[in,out] dst list that receives the elements
+ @param[in,out] src list whose elements are moved to dst; emptied on return */
+template <typename List>
+void ut_list_concatenate(List &dst, List &src) {
+  ut_ad(UT_LIST_IS_INITIALISED(dst));
+  ut_ad(UT_LIST_IS_INITIALISED(src));
+
+  if (src.first_element == nullptr) {
+    /* Nothing to move. */
+    return;
+  }
+
+  if (dst.last_element != nullptr) {
+    /* Link the two lists at the junction. */
+    List::get_node(*dst.last_element).next = src.first_element;
+    List::get_node(*src.first_element).prev = dst.last_element;
+  } else {
+    /* dst is empty, so it simply becomes src. */
+    dst.first_element = src.first_element;
+  }
+
+  dst.last_element = src.last_element;
+  dst.update_length(static_cast<int>(src.get_length()));
+
+  src.clear();
+}
+
+/** Appends all elements of SRC to the end of DST in O(1) time, leaving SRC
+ empty.
+ @param DST destination list base node (not a pointer to it)
+ @param SRC source list base node (not a pointer to it) */
+#define UT_LIST_CONCATENATE(DST, SRC) ut_list_concatenate(DST, SRC)
+
 /** Inserts a ELEM2 after ELEM1 in a list.
  @param list the base node
  @param elem1 node after which ELEM2 is inserted

storage/innobase/sync/sync0rw.cc

@@ -188,8 +188,8 @@ static rw_lock_debug_t *rw_lock_debug_create(void) {
 static void rw_lock_debug_free(rw_lock_debug_t *info) { ut::free(info); }
 #endif /* UNIV_DEBUG */
 
-void rw_lock_create_func(rw_lock_t *lock,
-                         IF_DEBUG(latch_id_t id, ) ut::Location clocation) {
+void rw_lock_init_only(rw_lock_t *lock,
+                       IF_DEBUG(latch_id_t id, ) ut::Location clocation) {
 #if !defined(UNIV_PFS_RWLOCK)
   /* It should have been created in pfs_rw_lock_create_func() */
   new (lock) rw_lock_t();
@@ -231,6 +231,11 @@ void rw_lock_create_func(rw_lock_t *lock,
   lock->wait_ex_event = os_event_create();
 
   lock->is_block_lock = false;
+}
+
+void rw_lock_create_func(rw_lock_t *lock,
+                         IF_DEBUG(latch_id_t id, ) ut::Location clocation) {
+  rw_lock_init_only(lock, IF_DEBUG(id, ) clocation);
 
   mutex_enter(&rw_lock_list_mutex);
 
@@ -242,6 +247,23 @@ void rw_lock_create_func(rw_lock_t *lock,
   mutex_exit(&rw_lock_list_mutex);
 }
 
+void rw_lock_list_register_bulk(rw_lock_list_t &locks) {
+  if (UT_LIST_GET_LEN(locks) == 0) {
+    return;
+  }
+
+  mutex_enter(&rw_lock_list_mutex);
+
+  ut_ad(UT_LIST_GET_FIRST(rw_lock_list) == nullptr ||
+        UT_LIST_GET_FIRST(rw_lock_list)->magic_n == rw_lock_t::MAGIC_N);
+
+  /* O(1) splice of the prepared list onto the global list. The per-lock work
+  (initialization and list linkage) was already done outside this mutex. */
+  UT_LIST_CONCATENATE(rw_lock_list, locks);
+
+  mutex_exit(&rw_lock_list_mutex);
+}
+
 /** Calling this function is obligatory only if the memory buffer containing
  the rw-lock is freed. Removes an rw-lock object from the global list. The
  rw-lock is checked to be in the non-locked state. */