From 3462a86aabe7342db131c973815f1bb070b4527a Mon Sep 17 00:00:00 2001 From: Alexey Bychko Date: Thu, 25 Jun 2026 12:04:06 +0200 Subject: [PATCH] PS-10595 [8.4]: Reduce rw_lock_list contention during buffer pool initialization https://perconadev.atlassian.net/browse/PS-10595 This is a contribution from Alexey Bychko , with a modification on top of the original patch: instead of inserting each chunk's rw-locks into the global rw_lock_list one by one while holding rw_lock_list_mutex, the per-chunk list is prepared outside the mutex and then concatenated onto rw_lock_list in O(1) under the mutex. During buffer pool initialization every buffer block rw-lock (block->lock and, in debug builds, block->debug_latch) was registered in the global rw_lock_list under rw_lock_list_mutex, one lock at a time. With large buffer pools and parallel initialization this serializes heavily on that single mutex. Initialize block rw-locks without registering them in rw_lock_list, then register a whole chunk's locks at once: [+] rw_lock_init_only() initializes an rw_lock_t exactly like rw_lock_create_func() but does not add it to rw_lock_list. A PFS-aware wrapper pfs_rw_lock_init_only_func() and the rw_lock_init_only_inst() macro preserve placement-new, pfs_psi and (debug) latch id handling. [+] ut_list_concatenate() / UT_LIST_CONCATENATE() splice one UT_LIST onto another in O(1). [+] rw_lock_list_register_bulk() only splices a caller-prepared list onto rw_lock_list in O(1) under a single rw_lock_list_mutex section. [*] buf_block_init() uses rw_lock_init_only_inst(); buf_chunk_init() builds the chunk's rw-lock list and registers it in bulk, exactly once per chunk creation. buf_pool_register_chunk() only inserts into the chunk map, since buf_pool_resize() re-invokes it on carried-over chunks and must not re-register (double-register) their locks in rw_lock_list. rw_lock_create_func() keeps its existing signature and behavior, so all other callers and diagnostics based on rw_lock_list are unaffected; the complete rw_lock_list is available once initialization finishes. --- storage/innobase/buf/buf0buf.cc | 32 +++++++++++++++++----- storage/innobase/include/sync0rw.h | 42 +++++++++++++++++++++++++++++ storage/innobase/include/sync0rw.ic | 14 ++++++++++ storage/innobase/include/ut0lst.h | 36 +++++++++++++++++++++++++ storage/innobase/sync/sync0rw.cc | 26 ++++++++++++++++-- 5 files changed, 142 insertions(+), 8 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index b1c3b5214590..c43ffd85e5fe 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -869,6 +869,9 @@ static void buf_block_init( mutex_create(LATCH_ID_BUF_BLOCK_MUTEX, &block->mutex); + /* Initialize the block rw-locks WITHOUT registering them in rw_lock_list. + Registration is done in bulk once the whole chunk has been initialized, + to avoid taking rw_lock_list_mutex once per block. */ #if defined PFS_SKIP_BUFFER_MUTEX_RWLOCK || defined PFS_GROUP_BUFFER_SYNC /* If PFS_SKIP_BUFFER_MUTEX_RWLOCK is defined, skip registration of buffer block rwlock with performance schema. @@ -877,17 +880,19 @@ static void buf_block_init( since buffer block rwlock will be registered later in pfs_register_buffer_block(). */ - rw_lock_create(PFS_NOT_INSTRUMENTED, &block->lock, LATCH_ID_BUF_BLOCK_LOCK); + rw_lock_init_only_inst(PFS_NOT_INSTRUMENTED, &block->lock, + LATCH_ID_BUF_BLOCK_LOCK); - ut_d(rw_lock_create(PFS_NOT_INSTRUMENTED, &block->debug_latch, - LATCH_ID_BUF_BLOCK_DEBUG)); + ut_d(rw_lock_init_only_inst(PFS_NOT_INSTRUMENTED, &block->debug_latch, + LATCH_ID_BUF_BLOCK_DEBUG)); #else /* PFS_SKIP_BUFFER_MUTEX_RWLOCK || PFS_GROUP_BUFFER_SYNC */ - rw_lock_create(buf_block_lock_key, &block->lock, LATCH_ID_BUF_BLOCK_LOCK); + rw_lock_init_only_inst(buf_block_lock_key, &block->lock, + LATCH_ID_BUF_BLOCK_LOCK); - ut_d(rw_lock_create(buf_block_debug_latch_key, &block->debug_latch, - LATCH_ID_BUF_BLOCK_DEBUG)); + ut_d(rw_lock_init_only_inst(buf_block_debug_latch_key, &block->debug_latch, + LATCH_ID_BUF_BLOCK_DEBUG)); #endif /* PFS_SKIP_BUFFER_MUTEX_RWLOCK || PFS_GROUP_BUFFER_SYNC */ @@ -1157,11 +1162,26 @@ static buf_chunk_t *buf_chunk_init( frame += UNIV_PAGE_SIZE; } + /* Build this chunk's block rw-lock list WITHOUT holding any mutex. The + chunk's blocks are owned exclusively by this thread until the chunk is + published, so this O(n) work is lock-free. */ + rw_lock_list_t chunk_locks; + UT_LIST_INIT(chunk_locks); + { + buf_block_t *lock_block = chunk->blocks; + for (ulint j = 0; j < chunk->size; ++j, ++lock_block) { + UT_LIST_ADD_LAST(chunk_locks, &lock_block->lock); + ut_d(UT_LIST_ADD_LAST(chunk_locks, &lock_block->debug_latch)); + } + } + + /* Register the chunk and rw-locks. */ if (mutex != nullptr) { mutex->lock(); } buf_pool_register_chunk(chunk); + rw_lock_list_register_bulk(chunk_locks); /* O(1), uses rw_lock_list_mutex */ if (mutex != nullptr) { mutex->unlock(); diff --git a/storage/innobase/include/sync0rw.h b/storage/innobase/include/sync0rw.h index 02286a654e5c..9916646d3a1d 100644 --- a/storage/innobase/include/sync0rw.h +++ b/storage/innobase/include/sync0rw.h @@ -126,6 +126,18 @@ extern ib_mutex_t rw_lock_list_mutex; @param[in] clocation location where created */ void rw_lock_create_func(rw_lock_t *lock, IF_DEBUG(latch_id_t id, ) ut::Location clocation); + +/** Initializes an rw-lock object but does NOT register it in the global + rw_lock_list. The caller is responsible for registering it later, + e.g. in bulk via rw_lock_list_register_bulk(). This lets callers that + create many rw-locks (such as buffer pool blocks) avoid taking + rw_lock_list_mutex once per lock. + @param[in] lock pointer to memory + @param[in] id latch_id (debug builds only) + @param[in] clocation location where created */ +void rw_lock_init_only(rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) ut::Location clocation); + /** Calling this function is obligatory only if the memory buffer containing the rw-lock is freed. Removes an rw-lock object from the global list. The rw-lock is checked to be in the non-locked state. */ @@ -517,6 +529,19 @@ static inline void pfs_rw_lock_create_func(mysql_pfs_key_t key, rw_lock_t *lock, IF_DEBUG(latch_id_t id, ) ut::Location clocation); +/** Performance schema instrumented wrap function for rw_lock_init_only(). +Behaves like pfs_rw_lock_create_func() but defers registration of the lock in +the global rw_lock_list to the caller. NOTE! Please use the corresponding macro +rw_lock_init_only_inst(), not directly this function! +@param[in] key key registered with performance schema +@param[in] lock rw lock +@param[in] id latch_id +@param[in] clocation location where created */ +static inline void pfs_rw_lock_init_only_func(mysql_pfs_key_t key, + rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) + ut::Location clocation); + /** Performance schema instrumented wrap function for rw_lock_x_lock_func() NOTE! Please use the corresponding macro rw_lock_x_lock(), not directly this function! @@ -631,8 +656,12 @@ static inline void pfs_rw_lock_free_func(rw_lock_t *lock); /*!< in: rw-lock */ #ifdef UNIV_DEBUG #define rw_lock_create(K, L, ID) \ rw_lock_create_func((L), (ID), UT_LOCATION_HERE) +#define rw_lock_init_only_inst(K, L, ID) \ + rw_lock_init_only((L), (ID), UT_LOCATION_HERE) #else /* UNIV_DEBUG */ #define rw_lock_create(K, L, ID) rw_lock_create_func((L), UT_LOCATION_HERE) +#define rw_lock_init_only_inst(K, L, ID) \ + rw_lock_init_only((L), UT_LOCATION_HERE) #endif /* UNIV_DEBUG */ /** NOTE! The following macros should be used in rw locking and @@ -719,9 +748,13 @@ static inline void rw_lock_x_unlock_gen(rw_lock_t *L, ulint P) { #ifdef UNIV_DEBUG #define rw_lock_create(K, L, ID) \ pfs_rw_lock_create_func((K), (L), (ID), UT_LOCATION_HERE) +#define rw_lock_init_only_inst(K, L, ID) \ + pfs_rw_lock_init_only_func((K), (L), (ID), UT_LOCATION_HERE) #else /* UNIV_DEBUG */ #define rw_lock_create(K, L, ID) \ pfs_rw_lock_create_func((K), (L), UT_LOCATION_HERE) +#define rw_lock_init_only_inst(K, L, ID) \ + pfs_rw_lock_init_only_func((K), (L), UT_LOCATION_HERE) #endif /* UNIV_DEBUG */ /****************************************************************** @@ -821,4 +854,13 @@ typedef UT_LIST_BASE_NODE_T(rw_lock_t, list) rw_lock_list_t; extern rw_lock_list_t rw_lock_list; +#ifndef UNIV_HOTBACKUP +/** Registers all rw-locks contained in the given list into the global + rw_lock_list under a single rw_lock_list_mutex critical section, in O(1) time. + The locks must have been prepared with rw_lock_init_only(). + @param[in,out] locks list of locks to register; emptied on return as its nodes + are spliced into rw_lock_list */ +void rw_lock_list_register_bulk(rw_lock_list_t &locks); +#endif /* !UNIV_HOTBACKUP */ + #endif /* sync0rw.h */ diff --git a/storage/innobase/include/sync0rw.ic b/storage/innobase/include/sync0rw.ic index 3dcd85180030..098986f69bf5 100644 --- a/storage/innobase/include/sync0rw.ic +++ b/storage/innobase/include/sync0rw.ic @@ -476,6 +476,20 @@ static inline void pfs_rw_lock_create_func(mysql_pfs_key_t key, rw_lock_t *lock, /* The actual function to initialize an rwlock */ rw_lock_create_func(lock, IF_DEBUG(id, ) clocation); } + +static inline void pfs_rw_lock_init_only_func(mysql_pfs_key_t key, + rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) + ut::Location clocation) { + new (lock) rw_lock_t{}; + + /* Initialize the rwlock for performance schema */ + lock->pfs_psi = PSI_RWLOCK_CALL(init_rwlock)(key.m_value, lock); + + /* Initialize the rwlock without registering it in rw_lock_list; the caller + registers it later (e.g. in bulk via rw_lock_list_register_bulk()). */ + rw_lock_init_only(lock, IF_DEBUG(id, ) clocation); +} /** Performance schema instrumented wrap function for rw_lock_x_lock_func() NOTE! Please use the corresponding macro rw_lock_x_lock(), not directly this function! diff --git a/storage/innobase/include/ut0lst.h b/storage/innobase/include/ut0lst.h index ae804d26ba13..df626b518471 100644 --- a/storage/innobase/include/ut0lst.h +++ b/storage/innobase/include/ut0lst.h @@ -353,6 +353,42 @@ void ut_list_append(List &list, typename List::elem_type *elem) { @param ELEM the element to add */ #define UT_LIST_ADD_LAST(LIST, ELEM) ut_list_append(LIST, ELEM) +/** Appends all elements of src to the end of dst in O(1) time, leaving src + empty. Both lists must be of the same type, i.e. chained through the same node + member, and an element must not be present in both lists at once. + @param[in,out] dst list that receives the elements + @param[in,out] src list whose elements are moved to dst; emptied on return */ +template +void ut_list_concatenate(List &dst, List &src) { + ut_ad(UT_LIST_IS_INITIALISED(dst)); + ut_ad(UT_LIST_IS_INITIALISED(src)); + + if (src.first_element == nullptr) { + /* Nothing to move. */ + return; + } + + if (dst.last_element != nullptr) { + /* Link the two lists at the junction. */ + List::get_node(*dst.last_element).next = src.first_element; + List::get_node(*src.first_element).prev = dst.last_element; + } else { + /* dst is empty, so it simply becomes src. */ + dst.first_element = src.first_element; + } + + dst.last_element = src.last_element; + dst.update_length(static_cast(src.get_length())); + + src.clear(); +} + +/** Appends all elements of SRC to the end of DST in O(1) time, leaving SRC + empty. + @param DST destination list base node (not a pointer to it) + @param SRC source list base node (not a pointer to it) */ +#define UT_LIST_CONCATENATE(DST, SRC) ut_list_concatenate(DST, SRC) + /** Inserts a ELEM2 after ELEM1 in a list. @param list the base node @param elem1 node after which ELEM2 is inserted diff --git a/storage/innobase/sync/sync0rw.cc b/storage/innobase/sync/sync0rw.cc index 8d28cf845ab8..8a094383b1b2 100644 --- a/storage/innobase/sync/sync0rw.cc +++ b/storage/innobase/sync/sync0rw.cc @@ -188,8 +188,8 @@ static rw_lock_debug_t *rw_lock_debug_create(void) { static void rw_lock_debug_free(rw_lock_debug_t *info) { ut::free(info); } #endif /* UNIV_DEBUG */ -void rw_lock_create_func(rw_lock_t *lock, - IF_DEBUG(latch_id_t id, ) ut::Location clocation) { +void rw_lock_init_only(rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) ut::Location clocation) { #if !defined(UNIV_PFS_RWLOCK) /* It should have been created in pfs_rw_lock_create_func() */ new (lock) rw_lock_t(); @@ -231,6 +231,11 @@ void rw_lock_create_func(rw_lock_t *lock, lock->wait_ex_event = os_event_create(); lock->is_block_lock = false; +} + +void rw_lock_create_func(rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) ut::Location clocation) { + rw_lock_init_only(lock, IF_DEBUG(id, ) clocation); mutex_enter(&rw_lock_list_mutex); @@ -242,6 +247,23 @@ void rw_lock_create_func(rw_lock_t *lock, mutex_exit(&rw_lock_list_mutex); } +void rw_lock_list_register_bulk(rw_lock_list_t &locks) { + if (UT_LIST_GET_LEN(locks) == 0) { + return; + } + + mutex_enter(&rw_lock_list_mutex); + + ut_ad(UT_LIST_GET_FIRST(rw_lock_list) == nullptr || + UT_LIST_GET_FIRST(rw_lock_list)->magic_n == rw_lock_t::MAGIC_N); + + /* O(1) splice of the prepared list onto the global list. The per-lock work + (initialization and list linkage) was already done outside this mutex. */ + UT_LIST_CONCATENATE(rw_lock_list, locks); + + mutex_exit(&rw_lock_list_mutex); +} + /** Calling this function is obligatory only if the memory buffer containing the rw-lock is freed. Removes an rw-lock object from the global list. The rw-lock is checked to be in the non-locked state. */