diff --git a/include/my_sys.h b/include/my_sys.h index 43f263519beb1..b4aff773a60ea 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -95,6 +95,10 @@ C_MODE_START #define MY_ROOT_USE_VMEM 0x20000U /* init_alloc_root: use my_virtual_mem_commit */ /* Tree that should delete things automatically */ #define MY_TREE_WITH_DELETE 0x40000U +#define MY_TRY_LARGE_PAGES 0x80000U /* my_large_malloc(): attempt to use + large pages; the caller must only + pass this when my_use_large_pages + is set */ #define MY_CHECK_ERROR 1U /* Params to my_end; Check open-close */ #define MY_GIVE_INFO 2U /* Give time info about process*/ @@ -178,12 +182,20 @@ extern char *my_strdup(PSI_memory_key key, const char *from,myf MyFlags); extern char *my_strndup(PSI_memory_key key, const char *from, size_t length, myf MyFlags); extern my_bool my_use_large_pages; +/** @return the myf flags to request large pages from my_large_malloc(), +my_large_virtual_alloc(), or the my_virtual_mem_*() functions, if +--large-pages is enabled */ +static inline myf my_large_pages_flag(void) +{ + return MYF(my_use_large_pages ? MY_TRY_LARGE_PAGES : 0); +} + int my_init_large_pages(void); uchar *my_large_malloc(size_t *size, myf my_flags); #ifdef _WIN32 /* On Windows, use my_virtual_mem_reserve() and my_virtual_mem_commit(). */ #else -char *my_large_virtual_alloc(size_t *size); +char *my_large_virtual_alloc(size_t *size, myf my_flags); #endif void my_large_free(void *ptr, size_t size); void my_large_page_truncate(size_t *size); diff --git a/include/my_virtual_mem.h b/include/my_virtual_mem.h index 689c75d5258e3..7be229c5bb7f0 100644 --- a/include/my_virtual_mem.h +++ b/include/my_virtual_mem.h @@ -19,6 +19,7 @@ (reserve, commit, decommit, release) */ #include /*size_t*/ +#include /*myf*/ #ifdef __cplusplus extern "C" { @@ -26,10 +27,10 @@ extern "C" { enum my_vmem_prot { MY_VMEM_READONLY= 0, MY_VMEM_READWRITE }; -char *my_virtual_mem_reserve(size_t *size); -char *my_virtual_mem_commit(char *ptr, size_t size); -void my_virtual_mem_decommit(char *ptr, size_t size); -void my_virtual_mem_release(char *ptr, size_t size); +char *my_virtual_mem_reserve(size_t *size, myf my_flags); +char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags); +void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags); +void my_virtual_mem_release(char *ptr, size_t size, myf my_flags); void my_virtual_mem_protect(void *ptr, size_t size, enum my_vmem_prot prot); #ifdef __cplusplus diff --git a/mysql-test/main/large_pages.result b/mysql-test/main/large_pages.result index c5e73f044a9a7..b447ccbecd94c 100644 --- a/mysql-test/main/large_pages.result +++ b/mysql-test/main/large_pages.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*"); +call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*"); call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*"); create table t1 ( a int not null auto_increment, diff --git a/mysql-test/main/large_pages.test b/mysql-test/main/large_pages.test index 7c0f497c6d31a..e043dc3eb1fd9 100644 --- a/mysql-test/main/large_pages.test +++ b/mysql-test/main/large_pages.test @@ -2,7 +2,7 @@ --source include/have_innodb.inc -call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*"); +call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*"); call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*"); create table t1 ( a int not null auto_increment, diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c index 5c57360d9fb76..44d0750013fa9 100644 --- a/mysys/mf_keycache.c +++ b/mysys/mf_keycache.c @@ -549,7 +549,7 @@ int init_simple_key_cache(void *keycache_, blocks--; keycache->allocated_mem_size= blocks * keycache->key_cache_block_size; if ((keycache->block_mem= my_large_malloc(&keycache->allocated_mem_size, - MYF(0)))) + my_large_pages_flag()))) { /* Allocate memory for blocks, hash_links and hash entries; diff --git a/mysys/my_alloc.c b/mysys/my_alloc.c index b8cb910b20704..281992a230cb7 100644 --- a/mysys/my_alloc.c +++ b/mysys/my_alloc.c @@ -52,7 +52,7 @@ static void *root_alloc(MEM_ROOT *root, size_t size, size_t *alloced_size, { void *ptr; *alloced_size= MY_ALIGN(size, my_system_page_size); - if ((ptr= my_virtual_mem_commit(NULL, *alloced_size))) + if ((ptr= my_virtual_mem_commit(NULL, *alloced_size, MYF(0)))) update_malloc_size(*alloced_size, MY_TEST(root->flags & ROOT_FLAG_THREAD_SPECIFIC)); return ptr; @@ -67,7 +67,7 @@ static void root_free(MEM_ROOT *root, void *ptr, size_t size) { update_malloc_size(-(longlong) size, MY_TEST(root->flags & ROOT_FLAG_THREAD_SPECIFIC)); - my_virtual_mem_release(ptr, size); + my_virtual_mem_release(ptr, size, MYF(0)); } else my_free(ptr); diff --git a/mysys/my_largepage.c b/mysys/my_largepage.c index 731f41908d1d5..9571aa84d8c6b 100644 --- a/mysys/my_largepage.c +++ b/mysys/my_largepage.c @@ -295,13 +295,19 @@ MAP_ANON but MAP_ANONYMOUS is marked "for compatibility" */ uchar *my_large_malloc(size_t *size, myf my_flags) { uchar *ptr= NULL; + /* Only actually attempt large pages if the caller passed + MY_TRY_LARGE_PAGES (the caller is expected to only do so when + my_use_large_pages is set); otherwise always do a plain allocation of + the exact requested size, so *size is never rounded up to the large + page granularity. */ + const my_bool use_large_pages= (my_flags & MY_TRY_LARGE_PAGES) != 0; #ifdef _WIN32 DWORD alloc_type= MEM_COMMIT | MEM_RESERVE; size_t orig_size= *size; DBUG_ENTER("my_large_malloc"); - if (my_use_large_pages) + if (use_large_pages) { alloc_type|= MEM_LARGE_PAGES; /* Align block size to my_large_page_size */ @@ -312,7 +318,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags) { if (my_flags & MY_WME) { - if (my_use_large_pages) + if (use_large_pages) { my_printf_error(EE_OUTOFMEMORY, "Couldn't allocate %zu bytes (MEM_LARGE_PAGES page " @@ -325,7 +331,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags) my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_ERROR_LOG), *size); } } - if (my_use_large_pages) + if (use_large_pages) { *size= orig_size; ptr= VirtualAlloc(NULL, *size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); @@ -345,7 +351,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags) while (1) { mapflag= MAP_PRIVATE | OS_MAP_ANON; - if (my_use_large_pages) + if (use_large_pages) { large_page_size= my_next_large_page_size(*size, &page_i); /* this might be 0, in which case we do a standard mmap */ @@ -432,13 +438,21 @@ uchar *my_large_malloc(size_t *size, myf my_flags) Special large pages allocator, with possibility to commit to allocating more memory later. Every implementation returns a zero filled buffer here. + Initial protection of returned buffer is readwrite, if MY_TRY_LARGE_PAGES + is set in my_flags or on AIX(ask Marko why), but no access otherwise. + The caller is expected to call my_virtual_mem_commit() before using memory. */ -char *my_large_virtual_alloc(size_t *size) +char *my_large_virtual_alloc(size_t *size, myf my_flags) { char *ptr; + int prot; DBUG_ENTER("my_large_virtual_alloc"); - - if (my_use_large_pages) +#ifdef _AIX + prot= PROT_READ | PROT_WRITE; +#else + prot= (my_flags & MY_TRY_LARGE_PAGES) ? PROT_READ|PROT_WRITE : PROT_NONE; +#endif + if (my_flags & MY_TRY_LARGE_PAGES) { size_t large_page_size; int page_i= 0; @@ -469,7 +483,7 @@ char *my_large_virtual_alloc(size_t *size) OS_MAP_ANON; size_t aligned_size= MY_ALIGN(*size, (size_t) large_page_size); - ptr= mmap(NULL, aligned_size, PROT_READ | PROT_WRITE, mapflag, -1, 0); + ptr= mmap(NULL, aligned_size, prot, mapflag, -1, 0); if (ptr == MAP_FAILED) { ptr= NULL; @@ -490,24 +504,20 @@ char *my_large_virtual_alloc(size_t *size) DBUG_RETURN(ptr); } } - - my_use_large_pages= FALSE; } -# ifdef _AIX - /* On IBM AIX, my_virtual_mem_commit() relies on mprotect(2) rather than - a subsequent mmap(2) with MAP_FIXED. */ - ptr= mmap(NULL, *size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | OS_MAP_ANON, -1, 0); -# else /* Illumos important to have MAP_NORESERVE otherwise reserves all swap. On innodb_buffer_pool_size_max overallocation. Linux is controlled on sysctl vm.overcommit_memory. + + MAP_NORESERVE only applies to the PROT_NONE, reserve-only case: like the + large page loop above, prot == PROT_READ|PROT_WRITE (MY_TRY_LARGE_PAGES + fallback, or AIX) needs memory that is guaranteed to be usable right away. */ - ptr= mmap(NULL, *size, PROT_NONE, MAP_PRIVATE | OS_MAP_ANON | MAP_NORESERVE, + ptr= mmap(NULL, *size, prot, + MAP_PRIVATE | OS_MAP_ANON | (prot == PROT_NONE ? MAP_NORESERVE : 0), -1, 0); -# endif if (ptr == MAP_FAILED) ptr= NULL; diff --git a/mysys/my_virtual_mem.c b/mysys/my_virtual_mem.c index acd63a9b0a81c..3fd92a78c08bb 100644 --- a/mysys/my_virtual_mem.c +++ b/mysys/my_virtual_mem.c @@ -37,12 +37,14 @@ InnoDB, the only user of this functionality), but it's the established terminology. - We try to respect use_large_pages setting, both on Windows and Linux + The caller requests large pages by passing MY_TRY_LARGE_PAGES in + my_flags (only when my_use_large_pages is set), consistently across + the reserve/commit/decommit/release calls for a given allocation. */ -char *my_virtual_mem_reserve(size_t *size) +char *my_virtual_mem_reserve(size_t *size, myf my_flags) { #ifdef _WIN32 - DWORD flags= my_use_large_pages + DWORD flags= (my_flags & MY_TRY_LARGE_PAGES) ? MEM_LARGE_PAGES | MEM_RESERVE | MEM_COMMIT : MEM_RESERVE; char *ptr= VirtualAlloc(NULL, *size, flags, PAGE_READWRITE); @@ -55,7 +57,7 @@ char *my_virtual_mem_reserve(size_t *size) } return ptr; #else - return my_large_virtual_alloc(size); + return my_large_virtual_alloc(size, my_flags); #endif } @@ -76,12 +78,12 @@ static my_bool is_memory_committed(char *ptr, size_t size) This is compatible with the mmap / VirtualAlloc semantics. */ -char *my_virtual_mem_commit(char *ptr, size_t size) +char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags) { #ifdef _WIN32 if (!ptr) return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - if (my_use_large_pages) + if (my_flags & MY_TRY_LARGE_PAGES) { DBUG_ASSERT(is_memory_committed(ptr, size)); } @@ -102,7 +104,7 @@ char *my_virtual_mem_commit(char *ptr, size_t size) MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); return p == MAP_FAILED ? NULL : p; } - if (my_use_large_pages) + if (my_flags & MY_TRY_LARGE_PAGES) /* my_large_virtual_alloc() already created a read/write mapping. */; else { @@ -142,11 +144,11 @@ char *my_virtual_mem_commit(char *ptr, size_t size) return ptr; } -void my_virtual_mem_decommit(char *ptr, size_t size) +void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags) { #ifdef _WIN32 DBUG_ASSERT(is_memory_committed(ptr, size)); - if (!my_use_large_pages) + if (!(my_flags & MY_TRY_LARGE_PAGES)) { if (!VirtualFree(ptr, size, MEM_DECOMMIT)) { @@ -183,8 +185,9 @@ void my_virtual_mem_decommit(char *ptr, size_t size) update_malloc_size(-(longlong) size, 0); } -void my_virtual_mem_release(char *ptr, size_t size) +void my_virtual_mem_release(char *ptr, size_t size, myf my_flags) { + (void) my_flags; #ifdef _WIN32 if (!VirtualFree(ptr, 0, MEM_RELEASE)) { diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 0b32a7beca954..d63b25ea72059 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1339,7 +1339,7 @@ bool buf_pool_t::create() noexcept retry: { NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE; - memory_unaligned= my_virtual_mem_reserve(&size); + memory_unaligned= my_virtual_mem_reserve(&size, my_large_pages_flag()); if (memory_unaligned); #if defined __aarch64__ || defined __riscv || defined __mips__ || defined __loongarch64 else if (size_in_bytes_max_default != 0 && @@ -1373,7 +1373,7 @@ bool buf_pool_t::create() noexcept if (size < size_in_bytes_max + alignment_waste) { - my_virtual_mem_release(memory_unaligned, size); + my_virtual_mem_release(memory_unaligned, size, my_large_pages_flag()); size+= 1 + (~size_t(memory_unaligned) & (innodb_buffer_pool_extent_size - 1)); goto retry; @@ -1398,9 +1398,10 @@ bool buf_pool_t::create() noexcept PSI_MEMORY_CALL(memory_alloc)(mem_key_buf_buf_pool, actual_size, &owner); #endif #ifndef _AIX - if (!my_virtual_mem_commit(memory, actual_size)) + if (!my_virtual_mem_commit(memory, actual_size, my_large_pages_flag())) { - my_virtual_mem_release(memory_unaligned, size_unaligned); + my_virtual_mem_release(memory_unaligned, size_unaligned, + my_large_pages_flag()); memory= nullptr; memory_unaligned= nullptr; goto oom; @@ -1575,8 +1576,9 @@ void buf_pool_t::close() noexcept owner= nullptr; #endif os_total_large_mem_allocated-= size; - my_virtual_mem_decommit(memory, size); - my_virtual_mem_release(memory_unaligned, size_unaligned); + my_virtual_mem_decommit(memory, size, my_large_pages_flag()); + my_virtual_mem_release(memory_unaligned, size_unaligned, + my_large_pages_flag()); memory= nullptr; memory_unaligned= nullptr; } @@ -1892,7 +1894,7 @@ inline void buf_pool_t::shrunk(size_t size, size_t reduced) noexcept guess before we invoke my_virtual_mem_decommit() below. */ latch.unlock(); } - my_virtual_mem_decommit(memory + size, reduced); + my_virtual_mem_decommit(memory + size, reduced, my_large_pages_flag()); #ifdef UNIV_PFS_MEMORY PSI_MEMORY_CALL(memory_free)(mem_key_buf_buf_pool, reduced, owner); #endif @@ -1950,7 +1952,8 @@ ATTRIBUTE_COLD void buf_pool_t::resize(size_t size, THD *thd) noexcept if (n_blocks_removed <= 0) { - if (!my_virtual_mem_commit(memory + old_size, size - old_size)) + if (!my_virtual_mem_commit(memory + old_size, size - old_size, + my_large_pages_flag())) { mysql_mutex_unlock(&mutex); sql_print_error("InnoDB: Cannot commit innodb_buffer_pool_size=%zum;" diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c index c925bfe54f1be..8fdaad25def89 100644 --- a/storage/maria/ma_pagecache.c +++ b/storage/maria/ma_pagecache.c @@ -855,7 +855,8 @@ size_t init_pagecache(PAGECACHE *pagecache, size_t use_mem, /* Allocate memory for cache page buffers */ pagecache->mem_size= blocks * pagecache->block_size; if ((pagecache->block_mem= - my_large_malloc(&pagecache->mem_size, MYF(MY_WME)))) + my_large_malloc(&pagecache->mem_size, + MYF(MY_WME) | my_large_pages_flag()))) { /* Allocate memory for blocks, hash_links and hash entries;