From 35c66d00c4e42be446748b8629e315f14396045a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Olchawa?= Date: Tue, 14 Jul 2026 18:31:47 +0200 Subject: [PATCH] PS-10595 [9.7]: combined lazy latch init + rw_lock_list bulk registration https://perconadev.atlassian.net/browse/PS-10595 Combines two buffer-pool initialization speedups: 1. Reduce rw_lock_list contention during buffer pool initialization by registering block rw-locks in bulk via an O(1) list splice under rw_lock_list_mutex instead of taking it once per block. 2. Lazy per-block latch initialization - defer creation of each block's mutex and rw-locks until the block is first used. The second optimization is deferring the cost (paid later in runtime). Therefore it is disabled by default and a new sysvar has to be used to enable it: innodb_buffer_pool_lazy_latch_init. While it is disabled, the first optimization helps a little bit. These changes are based on a contribution we got from: Alexey Bychko --- .../r/buffer_pool_lazy_latch_init.result | 51 +++ .../t/buffer_pool_lazy_latch_init-master.opt | 1 + .../innodb/t/buffer_pool_lazy_latch_init.test | 97 ++++++ ...b_buffer_pool_lazy_latch_init_basic.result | 9 + ...odb_buffer_pool_lazy_latch_init_basic.test | 18 ++ storage/innobase/buf/buf0buf.cc | 298 +++++++++++++----- storage/innobase/buf/buf0lru.cc | 12 + storage/innobase/handler/ha_innodb.cc | 10 + storage/innobase/handler/i_s.cc | 24 +- storage/innobase/include/buf0buf.h | 37 ++- storage/innobase/include/buf0buf.ic | 13 +- storage/innobase/include/mtr0mtr.ic | 2 + storage/innobase/include/srv0srv.h | 5 + storage/innobase/include/sync0rw.h | 46 +++ storage/innobase/include/sync0rw.ic | 13 + storage/innobase/include/ut0lst.h | 36 +++ storage/innobase/srv/srv0srv.cc | 3 + storage/innobase/sync/sync0rw.cc | 41 ++- 18 files changed, 620 insertions(+), 96 deletions(-) create mode 100644 mysql-test/suite/innodb/r/buffer_pool_lazy_latch_init.result create mode 100644 mysql-test/suite/innodb/t/buffer_pool_lazy_latch_init-master.opt create mode 100644 mysql-test/suite/innodb/t/buffer_pool_lazy_latch_init.test create mode 100644 mysql-test/suite/sys_vars/r/innodb_buffer_pool_lazy_latch_init_basic.result create mode 100644 mysql-test/suite/sys_vars/t/innodb_buffer_pool_lazy_latch_init_basic.test diff --git a/mysql-test/suite/innodb/r/buffer_pool_lazy_latch_init.result b/mysql-test/suite/innodb/r/buffer_pool_lazy_latch_init.result new file mode 100644 index 000000000000..cfeac1e2af5b --- /dev/null +++ b/mysql-test/suite/innodb/r/buffer_pool_lazy_latch_init.result @@ -0,0 +1,51 @@ +SELECT @@global.innodb_buffer_pool_lazy_latch_init; +@@global.innodb_buffer_pool_lazy_latch_init +1 +# Scan I_S.INNODB_BUFFER_PAGE while the pool is mostly unused. +# Without the lazy-init guard this enters an unconstructed mutex. +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page +WHERE page_state = 'NOT_USED' AND page_type = 'UNKNOWN'; +ok +1 +# Fault index/data pages into the pool. +CREATE TABLE t1 (id INT PRIMARY KEY, val INT) ENGINE=InnoDB; +# Some INDEX pages are now cached. +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page +WHERE page_type = 'INDEX'; +ok +1 +# Create compressed pages, evict their uncompressed frames, then scan +# I_S.INNODB_BUFFER_PAGE_LRU for standalone compressed descriptors. +CREATE TABLE t_zip ( +id INT PRIMARY KEY, +val BLOB, +KEY(val(10)) +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +SELECT COUNT(*) FROM t_zip; +COUNT(*) +100 +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page_lru; +ok +1 +# Expand the buffer pool (new lazy chunks), then re-scan. +SET GLOBAL innodb_buffer_pool_size = 25165824; +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page; +ok +1 +# Shrink the buffer pool (chunks freed), then re-scan. +SET GLOBAL innodb_buffer_pool_size = 8388608; +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page; +ok +1 +# Restart: the pool comes back up in lazy mode (opt reapplied). +# restart +SELECT @@global.innodb_buffer_pool_lazy_latch_init; +@@global.innodb_buffer_pool_lazy_latch_init +1 +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page; +ok +1 +SELECT COUNT(*) FROM t1; +COUNT(*) +2000 +DROP TABLE t1, t_zip; diff --git a/mysql-test/suite/innodb/t/buffer_pool_lazy_latch_init-master.opt b/mysql-test/suite/innodb/t/buffer_pool_lazy_latch_init-master.opt new file mode 100644 index 000000000000..e65ee78b8bc1 --- /dev/null +++ b/mysql-test/suite/innodb/t/buffer_pool_lazy_latch_init-master.opt @@ -0,0 +1 @@ +--innodb-buffer-pool-lazy-latch-init=ON --innodb-buffer-pool-size=16M --innodb-buffer-pool-chunk-size=2M diff --git a/mysql-test/suite/innodb/t/buffer_pool_lazy_latch_init.test b/mysql-test/suite/innodb/t/buffer_pool_lazy_latch_init.test new file mode 100644 index 000000000000..330a5ca9d2a8 --- /dev/null +++ b/mysql-test/suite/innodb/t/buffer_pool_lazy_latch_init.test @@ -0,0 +1,97 @@ +# +# PS-10595: exercise innodb_buffer_pool_lazy_latch_init=ON end to end. +# +# With lazy latch init, a never-used buffer block has no constructed latches. +# This test drives the paths that touch every chunk block regardless of state: +# - INFORMATION_SCHEMA.INNODB_BUFFER_PAGE scans all blocks (including unused +# ones) - it must report unused blocks as free without entering their +# (unconstructed) mutex, +# - online buffer pool resize creates fresh lazy chunks and later frees them, +# - a restart brings the pool back up in lazy mode. +# On a debug build this also activates all the block->latches_initialized +# assertions added by this feature. +# + +--source include/have_innodb_max_16k.inc + +# The feature is opt-in and read only; confirm the server started with it ON. +SELECT @@global.innodb_buffer_pool_lazy_latch_init; + +let $wait_timeout = 180; +let $wait_condition = + SELECT SUBSTR(variable_value, 1, 34) = 'Completed resizing buffer pool at ' + FROM performance_schema.global_status + WHERE LOWER(variable_name) = 'innodb_buffer_pool_resize_status'; + +--echo # Scan I_S.INNODB_BUFFER_PAGE while the pool is mostly unused. +--echo # Without the lazy-init guard this enters an unconstructed mutex. +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page + WHERE page_state = 'NOT_USED' AND page_type = 'UNKNOWN'; + +--echo # Fault index/data pages into the pool. +CREATE TABLE t1 (id INT PRIMARY KEY, val INT) ENGINE=InnoDB; +--disable_query_log +BEGIN; +let $i = 2000; +while ($i) { + eval INSERT INTO t1 VALUES ($i, $i); + dec $i; +} +COMMIT; +--enable_query_log + +--echo # Some INDEX pages are now cached. +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page + WHERE page_type = 'INDEX'; + +--echo # Create compressed pages, evict their uncompressed frames, then scan +--echo # I_S.INNODB_BUFFER_PAGE_LRU for standalone compressed descriptors. +CREATE TABLE t_zip ( + id INT PRIMARY KEY, + val BLOB, + KEY(val(10)) +) ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +--disable_query_log +BEGIN; +let $i = 100; +while ($i) { + eval INSERT INTO t_zip VALUES ($i, REPEAT('a', 2000)); + dec $i; +} +COMMIT; +--enable_query_log +SELECT COUNT(*) FROM t_zip; + +let $have_debug = `SELECT VERSION() LIKE '%debug%'`; +if ($have_debug) { + --disable_query_log + SET GLOBAL innodb_buffer_pool_evict = 'uncompressed'; + --enable_query_log + + let $compressed_only_pages = `SELECT COUNT(*) + FROM information_schema.innodb_buffer_page_lru + WHERE table_name LIKE '%t_zip%' AND compressed = 'YES'`; + if (!$compressed_only_pages) { + --die Expected standalone compressed descriptors for t_zip + } +} + +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page_lru; + +--echo # Expand the buffer pool (new lazy chunks), then re-scan. +SET GLOBAL innodb_buffer_pool_size = 25165824; +--source include/wait_condition.inc +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page; + +--echo # Shrink the buffer pool (chunks freed), then re-scan. +SET GLOBAL innodb_buffer_pool_size = 8388608; +--source include/wait_condition.inc +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page; + +--echo # Restart: the pool comes back up in lazy mode (opt reapplied). +--source include/restart_mysqld.inc +SELECT @@global.innodb_buffer_pool_lazy_latch_init; +SELECT COUNT(*) > 0 AS ok FROM information_schema.innodb_buffer_page; +SELECT COUNT(*) FROM t1; + +DROP TABLE t1, t_zip; diff --git a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_lazy_latch_init_basic.result b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_lazy_latch_init_basic.result new file mode 100644 index 000000000000..1fdc0ac6efa9 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_lazy_latch_init_basic.result @@ -0,0 +1,9 @@ +SELECT @@global.innodb_buffer_pool_lazy_latch_init; +@@global.innodb_buffer_pool_lazy_latch_init +0 +SET @@global.innodb_buffer_pool_lazy_latch_init = ON; +ERROR HY000: Variable 'innodb_buffer_pool_lazy_latch_init' is a read only variable +SET @@global.innodb_buffer_pool_lazy_latch_init = OFF; +ERROR HY000: Variable 'innodb_buffer_pool_lazy_latch_init' is a read only variable +SELECT @@session.innodb_buffer_pool_lazy_latch_init; +ERROR HY000: Variable 'innodb_buffer_pool_lazy_latch_init' is a GLOBAL variable diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_lazy_latch_init_basic.test b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_lazy_latch_init_basic.test new file mode 100644 index 000000000000..b35298cb28e8 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_lazy_latch_init_basic.test @@ -0,0 +1,18 @@ +# +# Basic test for innodb_buffer_pool_lazy_latch_init +# +# It is a read-only, global-only boolean that defaults to OFF. +# + +# Check the default value. +SELECT @@global.innodb_buffer_pool_lazy_latch_init; + +# It is read only: it cannot be changed at runtime. +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SET @@global.innodb_buffer_pool_lazy_latch_init = ON; +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SET @@global.innodb_buffer_pool_lazy_latch_init = OFF; + +# It has only a global scope. +--error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.innodb_buffer_pool_lazy_latch_init; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 747e146424d8..c9d05ba83bb1 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -67,6 +67,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include #include #include +#include #include #include #include @@ -82,7 +83,11 @@ this program; if not, write to the Free Software Foundation, Inc., #include "srv0srv.h" #include "srv0start.h" #include "sync0sync.h" +#include "sync0types.h" #include "trx0trx.h" +#include "ut0dbg.h" +#include "ut0lst.h" +#include "ut0mutex.h" #include "ut0new.h" #include "scope_guard.h" @@ -767,6 +772,50 @@ void buf_page_print(const byte *read_buf, const page_size_t &page_size, extern mysql_pfs_key_t buffer_block_mutex_key; #endif /* !PFS_SKIP_BUFFER_MUTEX_RWLOCK */ +/** Registers one buffer block's mutex and rw-locks with performance schema. + The latch objects must already be constructed. Used by + pfs_register_buffer_block() when latches are created eagerly, and by + buf_block_lazy_init_latches() when they are created lazily on first use. */ +static void pfs_register_buffer_block_latches( + buf_block_t *block) /*!< in/out: buffer block */ +{ +#ifdef UNIV_PFS_MUTEX + BPageMutex *mutex; + + mutex = &block->mutex; + +#ifndef PFS_SKIP_BUFFER_MUTEX_RWLOCK + mutex->pfs_add(buffer_block_mutex_key); +#endif /* !PFS_SKIP_BUFFER_MUTEX_RWLOCK */ + +#endif /* UNIV_PFS_MUTEX */ + + rw_lock_t *rwlock; + +#ifdef UNIV_PFS_RWLOCK + rwlock = &block->lock; + ut_a(!rwlock->pfs_psi); + +#ifndef PFS_SKIP_BUFFER_MUTEX_RWLOCK + rwlock->pfs_psi = + (PSI_server) ? PSI_server->init_rwlock(buf_block_lock_key, rwlock) : NULL; +#else + rwlock->pfs_psi = (PSI_server) + ? PSI_server->init_rwlock(PFS_NOT_INSTRUMENTED, rwlock) + : NULL; +#endif /* !PFS_SKIP_BUFFER_MUTEX_RWLOCK */ + +#ifdef UNIV_DEBUG + rwlock = &block->debug_latch; + ut_a(!rwlock->pfs_psi); + rwlock->pfs_psi = + (PSI_server) ? PSI_server->init_rwlock(buf_block_debug_latch_key, rwlock) + : NULL; +#endif /* UNIV_DEBUG */ + +#endif /* UNIV_PFS_RWLOCK */ +} + /** This function registers mutexes and rwlocks in buffer blocks with performance schema. If PFS_MAX_BUFFER_MUTEX_LOCK_REGISTER is defined to be a value less than chunk->size, then only mutexes @@ -783,57 +832,67 @@ static void pfs_register_buffer_block( num_to_register = std::min(chunk->size, PFS_MAX_BUFFER_MUTEX_LOCK_REGISTER); for (ulint i = 0; i < num_to_register; i++) { -#ifdef UNIV_PFS_MUTEX - BPageMutex *mutex; + pfs_register_buffer_block_latches(block); + block++; + } +} +#endif /* PFS_GROUP_BUFFER_SYNC */ - mutex = &block->mutex; +/** Initialize latches for a buffer block. */ +void buf_block_lazy_init_latches(buf_block_t *block) { + ut_a(!block->latches_initialized.load(std::memory_order_relaxed)); -#ifndef PFS_SKIP_BUFFER_MUTEX_RWLOCK - mutex->pfs_add(buffer_block_mutex_key); -#endif /* !PFS_SKIP_BUFFER_MUTEX_RWLOCK */ + /* This runs when lazy_init is being used, on the first use of a block: + the block has just been taken from the free list and is owned + exclusively by this thread (BUF_BLOCK_READY_FOR_USE), so it + is not yet reachable by any other thread. */ + ut_ad(buf_block_get_state(block) == BUF_BLOCK_READY_FOR_USE); -#endif /* UNIV_PFS_MUTEX */ + mutex_create(LATCH_ID_BUF_BLOCK_MUTEX, &block->mutex); - rw_lock_t *rwlock; +#if defined PFS_SKIP_BUFFER_MUTEX_RWLOCK || defined PFS_GROUP_BUFFER_SYNC + rw_lock_create(PFS_NOT_INSTRUMENTED, &block->lock, LATCH_ID_BUF_BLOCK_LOCK); + ut_d(rw_lock_create(PFS_NOT_INSTRUMENTED, &block->debug_latch, + LATCH_ID_BUF_BLOCK_DEBUG)); +#else + rw_lock_create(buf_block_lock_key, &block->lock, LATCH_ID_BUF_BLOCK_LOCK); + ut_d(rw_lock_create(buf_block_debug_latch_key, &block->debug_latch, + LATCH_ID_BUF_BLOCK_DEBUG)); +#endif -#ifdef UNIV_PFS_RWLOCK - rwlock = &block->lock; - ut_a(!rwlock->pfs_psi); +#ifdef PFS_GROUP_BUFFER_SYNC + /* With eager initialization the block latches are registered with + performance schema in bulk by pfs_register_buffer_block() when the chunk is + created. In lazy mode that call is skipped (the latch objects do not exist at + chunk creation), so instrument this block's freshly created latches here. + The rw-locks above were created with PFS_NOT_INSTRUMENTED, i.e. with a null + pfs_psi, exactly like in the eager flow before its bulk registration. + + Note the intentional divergence: eager pfs_register_buffer_block() only + registers the first PFS_MAX_BUFFER_MUTEX_LOCK_REGISTER latches of each chunk, + whereas here every block is registered on its first use. The cap defaults to + ULINT_MAX so there is no practical difference today; should it ever be + lowered, lazy mode would instrument more latches than eager mode. */ + pfs_register_buffer_block_latches(block); +#endif /* PFS_GROUP_BUFFER_SYNC */ -#ifndef PFS_SKIP_BUFFER_MUTEX_RWLOCK - rwlock->pfs_psi = (PSI_server) - ? PSI_server->init_rwlock(buf_block_lock_key, rwlock) - : NULL; -#else - rwlock->pfs_psi = - (PSI_server) ? PSI_server->init_rwlock(PFS_NOT_INSTRUMENTED, rwlock) - : NULL; -#endif /* !PFS_SKIP_BUFFER_MUTEX_RWLOCK */ + block->lock.is_block_lock = true; -#ifdef UNIV_DEBUG - rwlock = &block->debug_latch; - ut_a(!rwlock->pfs_psi); - rwlock->pfs_psi = (PSI_server) ? PSI_server->init_rwlock( - buf_block_debug_latch_key, rwlock) - : NULL; -#endif /* UNIV_DEBUG */ + ut_ad(rw_lock_validate(&block->lock)); -#endif /* UNIV_PFS_RWLOCK */ - block++; - } + /* Publish with release so the acquire load in the lock-free + INFORMATION_SCHEMA.INNODB_BUFFER_PAGE scanner sees fully constructed latch + objects once it observes the flag set. */ + block->latches_initialized.store(true, std::memory_order_release); } -#endif /* PFS_GROUP_BUFFER_SYNC */ -/** Initializes a buffer control block when the buf_pool is created. */ -static void buf_block_init( - buf_pool_t *buf_pool, /*!< in: buffer pool instance */ - buf_block_t *block, /*!< in: pointer to control block */ - byte *frame) /*!< in: pointer to buffer frame */ -{ +/** Lightweight initialization of a buffer control block: no latches created. */ +static void buf_block_init_without_latches(buf_pool_t *buf_pool, + buf_block_t *block, byte *frame) { UNIV_MEM_DESC(frame, UNIV_PAGE_SIZE); - /* This function should only be executed at database startup or by - buf_pool_resize(). Either way, adaptive hash index must not exist. */ + /* buf_block_init_without_latches() is only executed at database startup + or by buf_pool_resize(). Either way, adaptive hash index must not exist. */ block->ahi.assert_empty_on_init(); block->frame = frame; @@ -863,34 +922,20 @@ static void buf_block_init( page_zip_des_init(&block->page.zip); - mutex_create(LATCH_ID_BUF_BLOCK_MUTEX, &block->mutex); - -#if defined PFS_SKIP_BUFFER_MUTEX_RWLOCK || defined PFS_GROUP_BUFFER_SYNC - /* If PFS_SKIP_BUFFER_MUTEX_RWLOCK is defined, skip registration - of buffer block rwlock with performance schema. - - If PFS_GROUP_BUFFER_SYNC is defined, skip the registration - since buffer block rwlock will be registered later in - pfs_register_buffer_block(). */ - - rw_lock_create(PFS_NOT_INSTRUMENTED, &block->lock, LATCH_ID_BUF_BLOCK_LOCK); - - ut_d(rw_lock_create(PFS_NOT_INSTRUMENTED, &block->debug_latch, - LATCH_ID_BUF_BLOCK_DEBUG)); - -#else /* PFS_SKIP_BUFFER_MUTEX_RWLOCK || PFS_GROUP_BUFFER_SYNC */ - - rw_lock_create(buf_block_lock_key, &block->lock, LATCH_ID_BUF_BLOCK_LOCK); - - ut_d(rw_lock_create(buf_block_debug_latch_key, &block->debug_latch, - LATCH_ID_BUF_BLOCK_DEBUG)); - -#endif /* PFS_SKIP_BUFFER_MUTEX_RWLOCK || PFS_GROUP_BUFFER_SYNC */ - - block->lock.is_block_lock = true; - - ut_ad(rw_lock_validate(&(block->lock))); + /* Latches are NOT initialized here. Block creation happens either at + startup (no concurrency on this instance yet) or, for resize, under the + instance's free_list_mutex. */ + ut_ad(srv_is_being_started || mutex_own(&buf_pool->free_list_mutex)); + /* Relaxed is safe: false is the sentinel the lock-free I_S scanner treats as + "unused, do not touch the latch", so observing it (or the default-constructed + false) never causes a dereference and needs no ordering. This flag is only + ever written false here, at chunk creation, before the block is reachable as + a normal free-list block; it is never flipped true->false for a live block + (latches, once created, live until the chunk is freed). The only value whose + visibility must be ordered is the true published after latch construction. */ + block->latches_initialized.store(false, std::memory_order_relaxed); } + /* We maintain our private view of innobase_should_madvise_buf_pool() which we initialize at the beginning of buf_pool_init() and then update when the @@global.innodb_buffer_pool_in_core_file changes. @@ -1077,7 +1122,8 @@ static buf_chunk_t *buf_chunk_init( buf_chunk_t *chunk, /*!< out: chunk of buffers */ ulonglong mem_size, /*!< in: requested size in bytes */ bool populate, /*!< in: virtual page preallocation */ - std::mutex *mutex) /*!< in,out: Mutex protecting chunk map. */ + std::mutex *mutex) /*!< in,out: mutex protecting the chunk map, or + nullptr when no concurrency is possible */ { buf_block_t *block; byte *frame; @@ -1139,7 +1185,38 @@ static buf_chunk_t *buf_chunk_init( block = chunk->blocks; for (i = chunk->size; i--;) { - buf_block_init(buf_pool, block, frame); + buf_block_init_without_latches(buf_pool, block, frame); + + /* When lazy latch initialization is disabled, create the latches now + (eager initialization, the original behavior). When enabled, they are + created on first use in buf_LRU_get_free_only(). */ + if (!srv_buf_pool_lazy_latch_init) { + /* Use rw_lock_create_unregistered so all rw-locks from this chunk are + spliced onto rw_lock_list in one O(1) operation below, instead of + taking rw_lock_list_mutex once per block. */ + mutex_create(LATCH_ID_BUF_BLOCK_MUTEX, &block->mutex); +#if defined PFS_SKIP_BUFFER_MUTEX_RWLOCK || defined PFS_GROUP_BUFFER_SYNC + rw_lock_create_unregistered(PFS_NOT_INSTRUMENTED, &block->lock, + LATCH_ID_BUF_BLOCK_LOCK); + ut_d(rw_lock_create_unregistered( + PFS_NOT_INSTRUMENTED, &block->debug_latch, LATCH_ID_BUF_BLOCK_DEBUG)); +#else + rw_lock_create_unregistered(buf_block_lock_key, &block->lock, + LATCH_ID_BUF_BLOCK_LOCK); + ut_d(rw_lock_create_unregistered(buf_block_debug_latch_key, + &block->debug_latch, + LATCH_ID_BUF_BLOCK_DEBUG)); +#endif + block->lock.is_block_lock = true; + ut_ad(rw_lock_validate(&block->lock)); + /* Publish with release, same as the lazy path: at startup this is + redundant (single thread), but buf_chunk_init() also runs for chunks + added by an online resize, whose blocks become visible to the lock-free + INFORMATION_SCHEMA.INNODB_BUFFER_PAGE scanner. Release guarantees that a + scanner observing the flag set also sees the constructed latches. */ + block->latches_initialized.store(true, std::memory_order_release); + } + UNIV_MEM_INVALID(block->frame, UNIV_PAGE_SIZE); /* Add the block to the free list */ @@ -1153,19 +1230,44 @@ static buf_chunk_t *buf_chunk_init( frame += UNIV_PAGE_SIZE; } + /* Build this chunk's rw-lock list lock-free (blocks are exclusively owned + until published). This is an O(n) pass over the chunk's blocks, but it + acquires no mutex, so many threads can run it concurrently (each on its own + chunk) without contending. Only needed for eager init; lazy init registers + each lock individually when the block is first used. */ + rw_lock_list_t chunk_locks; + UT_LIST_INIT(chunk_locks); + if (!srv_buf_pool_lazy_latch_init) { + buf_block_t *lock_block = chunk->blocks; + for (ulint j = 0; j < chunk->size; ++j, ++lock_block) { + UT_LIST_ADD_LAST(chunk_locks, &lock_block->lock); + ut_d(UT_LIST_ADD_LAST(chunk_locks, &lock_block->debug_latch)); + } + } + + /* buf_pool instances are created in parallel during buf_pool_init(), so the + caller passes a mutex to serialize inserts into the shared chunk map. During + resize there is no concurrency and mutex is nullptr. */ if (mutex != nullptr) { mutex->lock(); } buf_pool_register_chunk(chunk); + /* O(1) splice under rw_lock_list_mutex; consumes (empties) chunk_locks. */ + rw_lock_list_register_bulk(std::move(chunk_locks)); if (mutex != nullptr) { mutex->unlock(); } #ifdef PFS_GROUP_BUFFER_SYNC - pfs_register_buffer_block(chunk); + /* In lazy mode the latch objects are not constructed yet; they are + instrumented one by one in buf_block_lazy_init_latches() instead. */ + if (!srv_buf_pool_lazy_latch_init) { + pfs_register_buffer_block(chunk); + } #endif /* PFS_GROUP_BUFFER_SYNC */ + return (chunk); } @@ -1276,7 +1378,8 @@ static void buf_pool_set_sizes(void) { @param[in] buf_pool buffer pool instance @param[in] buf_pool_size size in bytes @param[in] instance_no id of the instance -@param[in,out] mutex Mutex to protect common data structures +@param[in,out] mutex mutex protecting the shared chunk map while + instances are created in parallel @param[out] err DB_SUCCESS if all goes well @param[in] populate virtual page preallocation */ static void buf_pool_create(buf_pool_t *buf_pool, ulint buf_pool_size, @@ -1357,14 +1460,17 @@ static void buf_pool_create(buf_pool_t *buf_pool, ulint buf_pool_size, do { if (!buf_chunk_init(buf_pool, chunk, chunk_size, populate, mutex)) { + /* Failure cleanup at startup, under chunks_mutex. */ + ut_ad(mutex_own(&buf_pool->chunks_mutex)); while (--chunk >= buf_pool->chunks) { buf_block_t *block = chunk->blocks; for (i = chunk->size; i--; block++) { - mutex_free(&block->mutex); - rw_lock_free(&block->lock); - - ut_d(rw_lock_free(&block->debug_latch)); + if (block->latches_initialized) { + mutex_free(&block->mutex); + rw_lock_free(&block->lock); + ut_d(rw_lock_free(&block->debug_latch)); + } } buf_pool->deallocate_chunk(chunk); } @@ -1493,14 +1599,17 @@ static void buf_pool_free_instance(buf_pool_t *buf_pool) { chunks = buf_pool->chunks; chunk = chunks + buf_pool->n_chunks; + ut_ad(mutex_own(&buf_pool->chunks_mutex)); + while (--chunk >= chunks) { buf_block_t *block = chunk->blocks; for (ulint i = chunk->size; i--; block++) { - mutex_free(&block->mutex); - rw_lock_free(&block->lock); - - ut_d(rw_lock_free(&block->debug_latch)); + if (block->latches_initialized) { + mutex_free(&block->mutex); + rw_lock_free(&block->lock); + ut_d(rw_lock_free(&block->debug_latch)); + } } buf_pool->deallocate_chunk(chunk); @@ -2477,20 +2586,24 @@ static void buf_pool_resize() { ulint sum_freed = 0; + /* Resize holds the instance's free_list_mutex (and runs with + buf_pool_resizing set), so reading latches_initialized here is safe. */ + ut_ad(buf_pool_resizing); + ut_ad(mutex_own(&buf_pool->free_list_mutex)); + while (chunk < echunk) { buf_block_t *block = chunk->blocks; for (ulint j = chunk->size; j--; block++) { - mutex_free(&block->mutex); - rw_lock_free(&block->lock); - - ut_d(rw_lock_free(&block->debug_latch)); + if (block->latches_initialized) { + mutex_free(&block->mutex); + rw_lock_free(&block->lock); + ut_d(rw_lock_free(&block->debug_latch)); + } } buf_pool->deallocate_chunk(chunk); - sum_freed += chunk->size; - ++chunk; } @@ -3583,8 +3696,8 @@ buf_block_t *buf_block_from_ahi(const byte *ptr) { buf_block_t *block = &chunk->blocks[offs]; - /* The function buf_chunk_init() invokes buf_block_init() so that - block[n].frame == block->frame + n * UNIV_PAGE_SIZE. Check it. */ + /* The function buf_chunk_init() invokes buf_block_init_without_latches() so + that block[n].frame == block->frame + n * UNIV_PAGE_SIZE. Check it. */ ut_ad(block->frame == page_align(ptr)); /* Read the state of the block without holding a mutex. A state transition from BUF_BLOCK_FILE_PAGE to @@ -3634,6 +3747,7 @@ static void buf_wait_for_read(buf_block_t *block, trx_t *trx) { The repeated reads of io_fix will not be optimized out because it's an atomic variable.*/ + ut_ad(block->latches_initialized); std::chrono::steady_clock::time_point start_time; while (block->page.was_io_fix_read()) { if (start_time == std::chrono::steady_clock::time_point{}) @@ -4058,6 +4172,7 @@ dberr_t Buf_fetch::zip_page_handler(buf_block_t *&fix_block) { buf_block_set_io_fix(block, BUF_IO_READ); ut::Location loc{m_file, m_line}; + ut_ad(block->latches_initialized); rw_lock_x_lock_gen(&block->lock, 0, loc); rw_lock_x_unlock(m_hash_lock); @@ -4199,6 +4314,8 @@ void Buf_fetch::read_page() { template void Buf_fetch::mtr_add_page(buf_block_t *block) { + ut_ad(block->latches_initialized); + mtr_memo_type_t fix_type; ut::Location loc{m_file, m_line}; @@ -4586,6 +4703,8 @@ bool buf_page_optimistic_get(ulint rw_latch, buf_block_t *block, ut_ad(!ibuf_inside(mtr) || ibuf_page(block->page.id, block->page.size, UT_LOCATION_HERE, nullptr)); + ut_ad(block->latches_initialized); + bool success; mtr_memo_type_t fix_type; @@ -4711,6 +4830,8 @@ bool buf_page_get_known_nowait(ulint rw_latch, buf_block_t *block, ut_ad(!ibuf_inside(mtr) || hint == Cache_hint::KEEP_OLD); + ut_ad(block->latches_initialized); + bool success; mtr_memo_type_t fix_type; @@ -4802,6 +4923,8 @@ const buf_block_t *buf_page_try_get(const page_id_t &page_id, buf_block_buf_fix_inc(block, location); buf_page_mutex_exit(block); + ut_ad(block->latches_initialized); + mtr_memo_type_t fix_type = MTR_MEMO_PAGE_S_FIX; auto success = rw_lock_s_lock_nowait(&block->lock, location); @@ -5066,6 +5189,7 @@ buf_page_t *buf_page_init_for_read(ulint mode, const page_id_t &page_id, read is completed. The x-lock is cleared by the io-handler thread. */ + ut_ad(block->latches_initialized); rw_lock_x_lock_gen(&block->lock, BUF_IO_READ, UT_LOCATION_HERE); rw_lock_x_unlock(hash_lock); @@ -5232,6 +5356,8 @@ buf_block_t *buf_page_create(const page_id_t &page_id, size limit. */ mtr_memo_type_t mtr_latch_type; + ut_ad(block->latches_initialized); + if (rw_latch == RW_X_LATCH) { rw_lock_x_lock(&block->lock, UT_LOCATION_HERE); mtr_latch_type = MTR_MEMO_PAGE_X_FIX; diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index ad092edbffef..040d4f87804b 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1235,6 +1235,18 @@ buf_block_t *buf_LRU_get_free_only(buf_pool_t *buf_pool) { ut_ad(buf_pool_from_block(block) == buf_pool); + /* Initialize latches on first use. The block has just been removed from + the free list and is owned exclusively by this thread, so this access is + race-free; the prior thread's initialization (if any) is visible through + the free_list_mutex handoff. No mutex protects this access - exclusive + ownership does, hence a relaxed load. (The flag is std::atomic only for + the lock-free I_S buffer-page scanner; see buf0buf.h.) */ + ut_ad(buf_block_get_state(block) == BUF_BLOCK_READY_FOR_USE); + ut_ad(!block->page.in_free_list); + if (!block->latches_initialized.load(std::memory_order_relaxed)) { + buf_block_lazy_init_latches(block); + } + return (block); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 69a2c99df38e..3437f19fcfdb 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -24170,6 +24170,15 @@ static MYSQL_SYSVAR_BOOL( nullptr, nullptr, true); #endif /* HAVE_LIBNUMA */ +static MYSQL_SYSVAR_BOOL( + buffer_pool_lazy_latch_init, srv_buf_pool_lazy_latch_init, + PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, + "Create buffer pool block latches (mutex, rw-locks) lazily on first use" + " instead of eagerly while the buffer pool is built. Speeds up buffer" + " pool initialization for large pools at the cost of slightly slower first" + " use of each page, which then pays the latch construction.", + nullptr, nullptr, false); + static MYSQL_SYSVAR_BOOL( api_enable_binlog, ib_binlog_enabled, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, @@ -24626,6 +24635,7 @@ static SYS_VAR *innobase_system_variables[] = { #ifdef HAVE_LIBNUMA MYSQL_SYSVAR(numa_interleave), #endif /* HAVE_LIBNUMA */ + MYSQL_SYSVAR(buffer_pool_lazy_latch_init), MYSQL_SYSVAR(change_buffering), MYSQL_SYSVAR(change_buffer_max_size), #if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 748f5d89d277..4f45bd9a30f9 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -4623,14 +4623,14 @@ static void i_s_innodb_buffer_page_get_info( out: structure filled with scanned info */ { - BPageMutex *mutex = buf_page_get_mutex(bpage); - ut_ad(pool_id < MAX_BUFFER_POOLS); page_info->pool_id = pool_id; page_info->block_id = pos; + BPageMutex *mutex = buf_page_get_mutex(bpage); + mutex_enter(mutex); page_info->page_state = buf_page_get_state(bpage); @@ -4753,8 +4753,24 @@ static int i_s_innodb_fill_buffer_pool( /* GO through each block in the chunk */ for (n_blocks = num_to_process; n_blocks--; block++) { - i_s_innodb_buffer_page_get_info(&block->page, pool_id, block_id, - info_buffer + num_page); + buf_page_info_t *page_info = info_buffer + num_page; + + /* With innodb_buffer_pool_lazy_latch_init=ON a never-used block has + no constructed mutex; A false result means the block is still + BUF_BLOCK_NOT_USED: report it as free without touching its latch. + Only this chunk scanner can meet such blocks: the LRU scanner only + sees pages past buf_LRU_get_free_only() (latches built), and its + compressed-only pages are standalone buf_page_t descriptors with no + surrounding buf_block_t at all. */ + if (!block->latches_initialized.load(std::memory_order_acquire)) { + page_info->pool_id = pool_id; + page_info->block_id = block_id; + page_info->page_state = BUF_BLOCK_NOT_USED; + page_info->page_type = I_S_PAGE_TYPE_UNKNOWN; + } else { + i_s_innodb_buffer_page_get_info(&block->page, pool_id, block_id, + page_info); + } block_id++; num_page++; } diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 2c365f1f4c37..6c07516e33cf 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1907,6 +1907,32 @@ struct buf_block_t { single thread. */ bool made_dirty_with_no_latch; + /** Whether this block's latches (mutex, lock, debug_latch) have been + created (eagerly, or lazily if innodb_buffer_pool_lazy_latch_init is + enabled). + + The false->true transition always happens while the block is owned + exclusively by a single thread (just removed from the free list, not yet in + the page hash or LRU). For a thread that later reuses the block after the + normal free_list_mutex / page-hash / LRU-mutex handoff, that handoff already + publishes the flag, so those owner-side reads (the first-use trigger in + buf_LRU_get_free_only()) and the teardown reads (under chunks_mutex / + free_list_mutex, no concurrent writer) can use relaxed/plain semantics. + + It is std::atomic only so that the lock-free I_S.INNODB_BUFFER_PAGE scanner, + which walks every chunk block without owning it, can safely probe a + possibly-never-initialized block: the flag is published with a release store + at the end of buf_block_lazy_init_latches() and read there with an acquire + load. A true result means the embedded latch objects are fully constructed and + safe to lock; a false result means the block is still unused and its latches + must not be used. + + Describes this block's own embedded latch objects, so it is intentionally + never copied along with page contents (e.g. by the buf_page_t copy + constructor or buf_relocate()): each block keeps the flag matching the state + of its own latches. */ + std::atomic latches_initialized{false}; + #ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG /** @name Debug fields */ @@ -2051,6 +2077,10 @@ static inline uint64_t buf_pool_hash_zip_frame(void *ptr) { static inline uint64_t buf_pool_hash_zip(buf_block_t *b) { return buf_pool_hash_zip_frame(b->frame); } + +/* Lazy latch initialization for buffer block. */ +void buf_block_lazy_init_latches(buf_block_t *block); + /** @} */ /** A "Hazard Pointer" class used to iterate over page lists @@ -2620,9 +2650,10 @@ Use these instead of accessing buffer pool mutexes directly. */ mutex_exit(&(b)->flush_list_mutex); \ } while (0) /** Acquire the block->mutex. */ -#define buf_page_mutex_enter(b) \ - do { \ - mutex_enter(&(b)->mutex); \ +#define buf_page_mutex_enter(b) \ + do { \ + ut_ad((b)->latches_initialized); \ + mutex_enter(&(b)->mutex); \ } while (0) /** Release the block->mutex. */ diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index 02011ae83266..ca8d7612a137 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -256,7 +256,14 @@ static inline bool buf_page_is_private(const buf_page_t *bpage, ut_a(!bpage->in_free_list); ut_a(!bpage->in_LRU_list); if (!hold_block_mutex) { - ut_a(!mutex_own(buf_page_get_mutex(bpage))); + /* A never-used block can reach this validation before lazy construction + of its embedded mutex. Reused blocks retain their latches, so keep checking + ownership once latches_initialized has been published. + Note: If mutex was not initialized, then it is also not an owned mutex. */ + ut_a((buf_page_get_state(bpage) == BUF_BLOCK_NOT_USED && + !reinterpret_cast(bpage) + ->latches_initialized.load(std::memory_order_relaxed)) || + !mutex_own(buf_page_get_mutex(bpage))); } ut_a(!mutex_own(&buf_pool->free_list_mutex)); if (!hold_zip_free_mutex) { @@ -777,6 +784,10 @@ static inline ulint buf_block_fix(buf_block_t *block) { static inline void buf_block_buf_fix_inc_func(IF_DEBUG(ut::Location location, ) buf_block_t *block) { #ifdef UNIV_DEBUG + /* Any block being buffer-fixed must have left the free list, and thus have + had its latches created (see buf_LRU_get_free_only()). */ + ut_ad(block->latches_initialized); + /* No debug latch is acquired if block belongs to system temporary. Debug latch is not of much help if access to block is single threaded. */ diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic index 6402f4dda119..ff27c54984c1 100644 --- a/storage/innobase/include/mtr0mtr.ic +++ b/storage/innobase/include/mtr0mtr.ic @@ -96,6 +96,7 @@ void mtr_t::sx_latch_at_savepoint(ulint savepoint, buf_block_t *block) { /* == RW_NO_LATCH */ ut_a(slot->type == MTR_MEMO_BUF_FIX); + ut_ad(block->latches_initialized); rw_lock_sx_lock(&block->lock, UT_LOCATION_HERE); slot->type = MTR_MEMO_PAGE_SX_FIX; @@ -121,6 +122,7 @@ void mtr_t::x_latch_at_savepoint(ulint savepoint, buf_block_t *block) { /* == RW_NO_LATCH */ ut_a(slot->type == MTR_MEMO_BUF_FIX); + ut_ad(block->latches_initialized); rw_lock_x_lock(&block->lock, UT_LOCATION_HERE); slot->type = MTR_MEMO_PAGE_X_FIX; diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 284e5c35783b..bd551cbc1c01 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -414,6 +414,11 @@ Currently we support native aio on windows and linux */ extern bool srv_use_native_aio; extern bool srv_numa_interleave; +/** When true, buffer pool blocks are created with lightweight initialization +and their latches (mutex, rw-locks) are created lazily on first use. When +false (default), latches are created eagerly while the buffer pool is built. */ +extern bool srv_buf_pool_lazy_latch_init; + /* The innodb_directories variable value. This a list of directories deliminated by ';', i.e the FIL_PATH_SEPARATOR. */ extern char *srv_innodb_directories; diff --git a/storage/innobase/include/sync0rw.h b/storage/innobase/include/sync0rw.h index df7f3e9d2ee2..43399129abdf 100644 --- a/storage/innobase/include/sync0rw.h +++ b/storage/innobase/include/sync0rw.h @@ -125,6 +125,19 @@ extern ib_mutex_t rw_lock_list_mutex; @param[in] clocation location where created */ void rw_lock_create_func(rw_lock_t *lock, IF_DEBUG(latch_id_t id, ) ut::Location clocation); + +/** Initializes an rw-lock object but does NOT register it in the global + rw_lock_list. The caller is responsible for registering it later, + e.g. in bulk via rw_lock_list_register_bulk(). This lets callers that + create many rw-locks (such as buffer pool blocks) avoid taking + rw_lock_list_mutex once per lock. + @param[in] lock pointer to memory + @param[in] id latch_id (debug builds only) + @param[in] clocation location where created */ +void rw_lock_create_unregistered_func(rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) + ut::Location clocation); + /** Calling this function is obligatory only if the memory buffer containing the rw-lock is freed. Removes an rw-lock object from the global list. The rw-lock is checked to be in the non-locked state. */ @@ -516,6 +529,19 @@ static inline void pfs_rw_lock_create_func(mysql_pfs_key_t key, rw_lock_t *lock, IF_DEBUG(latch_id_t id, ) ut::Location clocation); +/** Performance schema instrumented wrap function for +rw_lock_create_unregistered_func(). Behaves like pfs_rw_lock_create_func() but +defers registration of the lock in the global rw_lock_list to the caller. NOTE! +Please use the corresponding macro rw_lock_create_unregistered(), not directly +this function! +@param[in] key key registered with performance schema +@param[in] lock rw lock +@param[in] id latch_id +@param[in] clocation location where created */ +static inline void pfs_rw_lock_create_unregistered_func( + mysql_pfs_key_t key, rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) ut::Location clocation); + /** Performance schema instrumented wrap function for rw_lock_x_lock_func() NOTE! Please use the corresponding macro rw_lock_x_lock(), not directly this function! @@ -630,8 +656,12 @@ static inline void pfs_rw_lock_free_func(rw_lock_t *lock); /*!< in: rw-lock */ #ifdef UNIV_DEBUG #define rw_lock_create(K, L, ID) \ rw_lock_create_func((L), (ID), UT_LOCATION_HERE) +#define rw_lock_create_unregistered(K, L, ID) \ + rw_lock_create_unregistered_func((L), (ID), UT_LOCATION_HERE) #else /* UNIV_DEBUG */ #define rw_lock_create(K, L, ID) rw_lock_create_func((L), UT_LOCATION_HERE) +#define rw_lock_create_unregistered(K, L, ID) \ + rw_lock_create_unregistered_func((L), UT_LOCATION_HERE) #endif /* UNIV_DEBUG */ /** NOTE! The following macros should be used in rw locking and @@ -718,9 +748,13 @@ static inline void rw_lock_x_unlock_gen(rw_lock_t *L, ulint P) { #ifdef UNIV_DEBUG #define rw_lock_create(K, L, ID) \ pfs_rw_lock_create_func((K), (L), (ID), UT_LOCATION_HERE) +#define rw_lock_create_unregistered(K, L, ID) \ + pfs_rw_lock_create_unregistered_func((K), (L), (ID), UT_LOCATION_HERE) #else /* UNIV_DEBUG */ #define rw_lock_create(K, L, ID) \ pfs_rw_lock_create_func((K), (L), UT_LOCATION_HERE) +#define rw_lock_create_unregistered(K, L, ID) \ + pfs_rw_lock_create_unregistered_func((K), (L), UT_LOCATION_HERE) #endif /* UNIV_DEBUG */ /****************************************************************** @@ -820,4 +854,16 @@ typedef UT_LIST_BASE_NODE_T(rw_lock_t, list) rw_lock_list_t; extern rw_lock_list_t rw_lock_list; +#ifndef UNIV_HOTBACKUP +/** Registers all rw-locks contained in the given list into the global + rw_lock_list under a single rw_lock_list_mutex critical section, in O(1) time. + The locks must have been prepared with rw_lock_create_unregistered_func(). + Takes ownership of the list: the base node is consumed (emptied) as its + elements are spliced into rw_lock_list. The rw_lock_t objects themselves are + not owned by the list base node (they are embedded elsewhere, e.g. in buffer + blocks) and are only re-linked - they remain valid and are not destroyed. + @param[in] locks list of locks to register; consumed (emptied) by this call */ +void rw_lock_list_register_bulk(rw_lock_list_t &&locks); +#endif /* !UNIV_HOTBACKUP */ + #endif /* sync0rw.h */ diff --git a/storage/innobase/include/sync0rw.ic b/storage/innobase/include/sync0rw.ic index 3dcd85180030..d1fe8c7f4fb3 100644 --- a/storage/innobase/include/sync0rw.ic +++ b/storage/innobase/include/sync0rw.ic @@ -476,6 +476,19 @@ static inline void pfs_rw_lock_create_func(mysql_pfs_key_t key, rw_lock_t *lock, /* The actual function to initialize an rwlock */ rw_lock_create_func(lock, IF_DEBUG(id, ) clocation); } + +static inline void pfs_rw_lock_create_unregistered_func( + mysql_pfs_key_t key, rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) ut::Location clocation) { + new (lock) rw_lock_t{}; + + /* Initialize the rwlock for performance schema */ + lock->pfs_psi = PSI_RWLOCK_CALL(init_rwlock)(key.m_value, lock); + + /* Initialize the rwlock without registering it in rw_lock_list; the caller + registers it later (e.g. in bulk via rw_lock_list_register_bulk()). */ + rw_lock_create_unregistered_func(lock, IF_DEBUG(id, ) clocation); +} /** Performance schema instrumented wrap function for rw_lock_x_lock_func() NOTE! Please use the corresponding macro rw_lock_x_lock(), not directly this function! diff --git a/storage/innobase/include/ut0lst.h b/storage/innobase/include/ut0lst.h index ae804d26ba13..df626b518471 100644 --- a/storage/innobase/include/ut0lst.h +++ b/storage/innobase/include/ut0lst.h @@ -353,6 +353,42 @@ void ut_list_append(List &list, typename List::elem_type *elem) { @param ELEM the element to add */ #define UT_LIST_ADD_LAST(LIST, ELEM) ut_list_append(LIST, ELEM) +/** Appends all elements of src to the end of dst in O(1) time, leaving src + empty. Both lists must be of the same type, i.e. chained through the same node + member, and an element must not be present in both lists at once. + @param[in,out] dst list that receives the elements + @param[in,out] src list whose elements are moved to dst; emptied on return */ +template +void ut_list_concatenate(List &dst, List &src) { + ut_ad(UT_LIST_IS_INITIALISED(dst)); + ut_ad(UT_LIST_IS_INITIALISED(src)); + + if (src.first_element == nullptr) { + /* Nothing to move. */ + return; + } + + if (dst.last_element != nullptr) { + /* Link the two lists at the junction. */ + List::get_node(*dst.last_element).next = src.first_element; + List::get_node(*src.first_element).prev = dst.last_element; + } else { + /* dst is empty, so it simply becomes src. */ + dst.first_element = src.first_element; + } + + dst.last_element = src.last_element; + dst.update_length(static_cast(src.get_length())); + + src.clear(); +} + +/** Appends all elements of SRC to the end of DST in O(1) time, leaving SRC + empty. + @param DST destination list base node (not a pointer to it) + @param SRC source list base node (not a pointer to it) */ +#define UT_LIST_CONCATENATE(DST, SRC) ut_list_concatenate(DST, SRC) + /** Inserts a ELEM2 after ELEM1 in a list. @param list the base node @param elem1 node after which ELEM2 is inserted diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index e7e56501ce0f..fa0a4a78b8e7 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -225,6 +225,9 @@ bool srv_use_native_aio = false; bool srv_numa_interleave = false; +/** See srv0srv.h. Default off: latches are created eagerly. */ +bool srv_buf_pool_lazy_latch_init = false; + #ifdef UNIV_DEBUG /** Force all user tables to use page compression. */ ulong srv_debug_compress; diff --git a/storage/innobase/sync/sync0rw.cc b/storage/innobase/sync/sync0rw.cc index 8d28cf845ab8..3520334da16a 100644 --- a/storage/innobase/sync/sync0rw.cc +++ b/storage/innobase/sync/sync0rw.cc @@ -40,6 +40,8 @@ this program; if not, write to the Free Software Foundation, Inc., #include "sync0rw.h" +#include "univ.i" + #include #include @@ -50,6 +52,9 @@ this program; if not, write to the Free Software Foundation, Inc., #include "srv0mon.h" #include "srv0srv.h" #include "sync0debug.h" +#include "sync0types.h" +#include "ut0core.h" +#include "ut0lst.h" /* IMPLEMENTATION OF THE RW_LOCK @@ -188,8 +193,9 @@ static rw_lock_debug_t *rw_lock_debug_create(void) { static void rw_lock_debug_free(rw_lock_debug_t *info) { ut::free(info); } #endif /* UNIV_DEBUG */ -void rw_lock_create_func(rw_lock_t *lock, - IF_DEBUG(latch_id_t id, ) ut::Location clocation) { +void rw_lock_create_unregistered_func(rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) + ut::Location clocation) { #if !defined(UNIV_PFS_RWLOCK) /* It should have been created in pfs_rw_lock_create_func() */ new (lock) rw_lock_t(); @@ -231,6 +237,11 @@ void rw_lock_create_func(rw_lock_t *lock, lock->wait_ex_event = os_event_create(); lock->is_block_lock = false; +} + +void rw_lock_create_func(rw_lock_t *lock, + IF_DEBUG(latch_id_t id, ) ut::Location clocation) { + rw_lock_create_unregistered_func(lock, IF_DEBUG(id, ) clocation); mutex_enter(&rw_lock_list_mutex); @@ -242,6 +253,32 @@ void rw_lock_create_func(rw_lock_t *lock, mutex_exit(&rw_lock_list_mutex); } +void rw_lock_list_register_bulk(rw_lock_list_t &&locks) { + if (UT_LIST_GET_LEN(locks) == 0) { + return; + } + + /* Sanity-check an incoming lock: it must have been fully prepared through + rw_lock_create_unregistered_func() (the list is non-empty here, guaranteed by + the early return above). */ + ut_ad(UT_LIST_GET_FIRST(locks)->magic_n == rw_lock_t::MAGIC_N); + + mutex_enter(&rw_lock_list_mutex); + + ut_ad(UT_LIST_GET_FIRST(rw_lock_list) == nullptr || + UT_LIST_GET_FIRST(rw_lock_list)->magic_n == rw_lock_t::MAGIC_N); + + /* O(1) splice of the prepared list onto the global list. The per-lock work + (initialization and list linkage) was already done outside this mutex. */ + UT_LIST_CONCATENATE(rw_lock_list, locks); + + mutex_exit(&rw_lock_list_mutex); + + /* The moved-from list base node is left empty: the locks now live on + rw_lock_list only. */ + ut_ad(UT_LIST_GET_LEN(locks) == 0); +} + /** Calling this function is obligatory only if the memory buffer containing the rw-lock is freed. Removes an rw-lock object from the global list. The rw-lock is checked to be in the non-locked state. */