MDEV-38918 Make large pages an explicit per-caller opt-in

include/my_sys.h

@@ -95,6 +95,10 @@ C_MODE_START
 #define MY_ROOT_USE_VMEM 0x20000U /* init_alloc_root: use my_virtual_mem_commit */
 /* Tree that should delete things automatically */
 #define MY_TREE_WITH_DELETE 0x40000U
+#define MY_TRY_LARGE_PAGES 0x80000U   /* my_large_malloc(): attempt to use
+                                         large pages; the caller must only
+                                         pass this when my_use_large_pages
+                                         is set */
 
 #define MY_CHECK_ERROR	1U	/* Params to my_end; Check open-close */
 #define MY_GIVE_INFO	2U	/* Give time info about process*/
@@ -178,12 +182,20 @@ extern char *my_strdup(PSI_memory_key key, const char *from,myf MyFlags);
 extern char *my_strndup(PSI_memory_key key, const char *from, size_t length, myf MyFlags);
 extern my_bool my_use_large_pages;
 
+/** @return the myf flags to request large pages from my_large_malloc(),
+my_large_virtual_alloc(), or the my_virtual_mem_*() functions, if
+--large-pages is enabled */
+static inline myf my_large_pages_flag(void)
+{
+  return MYF(my_use_large_pages ? MY_TRY_LARGE_PAGES : 0);
+}
+
 int my_init_large_pages(void);
 uchar *my_large_malloc(size_t *size, myf my_flags);
 #ifdef _WIN32
 /* On Windows, use my_virtual_mem_reserve() and my_virtual_mem_commit(). */
 #else
-char *my_large_virtual_alloc(size_t *size);
+char *my_large_virtual_alloc(size_t *size, myf my_flags);
 #endif
 void my_large_free(void *ptr, size_t size);
 void my_large_page_truncate(size_t *size);

include/my_virtual_mem.h

@@ -19,17 +19,18 @@
   (reserve, commit, decommit, release)
 */
 #include <stddef.h> /*size_t*/
+#include <my_global.h> /*myf*/
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 enum my_vmem_prot { MY_VMEM_READONLY= 0, MY_VMEM_READWRITE };
 
-char *my_virtual_mem_reserve(size_t *size);
-char *my_virtual_mem_commit(char *ptr, size_t size);
-void my_virtual_mem_decommit(char *ptr, size_t size);
-void my_virtual_mem_release(char *ptr, size_t size);
+char *my_virtual_mem_reserve(size_t *size, myf my_flags);
+char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags);
+void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags);
+void my_virtual_mem_release(char *ptr, size_t size, myf my_flags);
 void my_virtual_mem_protect(void *ptr, size_t size, enum my_vmem_prot prot);
 
 #ifdef __cplusplus

mysql-test/main/large_pages.result

@@ -1,4 +1,4 @@
-call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*");
+call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*");
 call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*");
 create table t1 (
 a int not null auto_increment,

mysql-test/main/large_pages.test

@@ -2,7 +2,7 @@
 
 --source include/have_innodb.inc
 
-call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEMLOCK) page size [0-9]+\\).*");
+call mtr.add_suppression("\\[Warning\\] (mysqld|mariadbd): Couldn't allocate [0-9]+ bytes \\((Large/HugeTLB memory|MEM_LARGE_PAGES|MEMLOCK) page size [0-9]+\\).*");
 call mtr.add_suppression("\\[ERROR\\]*Lock Pages in memory access rights required.*");
 create table t1 (
   a int not null auto_increment,

mysys/mf_keycache.c

@@ -549,7 +549,7 @@ int init_simple_key_cache(void *keycache_,
         blocks--;
       keycache->allocated_mem_size= blocks * keycache->key_cache_block_size;
       if ((keycache->block_mem= my_large_malloc(&keycache->allocated_mem_size,
-                                                MYF(0))))
+                                                my_large_pages_flag())))
       {
         /*
 	  Allocate memory for blocks, hash_links and hash entries;

mysys/my_alloc.c

@@ -52,7 +52,7 @@ static void *root_alloc(MEM_ROOT *root, size_t size, size_t *alloced_size,
   {
     void *ptr;
     *alloced_size= MY_ALIGN(size, my_system_page_size);
-    if ((ptr= my_virtual_mem_commit(NULL, *alloced_size)))
+    if ((ptr= my_virtual_mem_commit(NULL, *alloced_size, MYF(0))))
       update_malloc_size(*alloced_size,
                          MY_TEST(root->flags & ROOT_FLAG_THREAD_SPECIFIC));
     return ptr;
@@ -67,7 +67,7 @@ static void root_free(MEM_ROOT *root, void *ptr, size_t size)
   {
     update_malloc_size(-(longlong) size,
                         MY_TEST(root->flags & ROOT_FLAG_THREAD_SPECIFIC));
-    my_virtual_mem_release(ptr, size);
+    my_virtual_mem_release(ptr, size, MYF(0));
   }
   else
     my_free(ptr);

mysys/my_largepage.c

@@ -295,13 +295,19 @@ MAP_ANON but MAP_ANONYMOUS is marked "for compatibility" */
 uchar *my_large_malloc(size_t *size, myf my_flags)
 {
   uchar *ptr= NULL;
+  /* Only actually attempt large pages if the caller passed
+  MY_TRY_LARGE_PAGES (the caller is expected to only do so when
+  my_use_large_pages is set); otherwise always do a plain allocation of
+  the exact requested size, so *size is never rounded up to the large
+  page granularity. */
+  const my_bool use_large_pages= (my_flags & MY_TRY_LARGE_PAGES) != 0;
 
 #ifdef _WIN32
   DWORD alloc_type= MEM_COMMIT | MEM_RESERVE;
   size_t orig_size= *size;
   DBUG_ENTER("my_large_malloc");
 
-  if (my_use_large_pages)
+  if (use_large_pages)
   {
     alloc_type|= MEM_LARGE_PAGES;
     /* Align block size to my_large_page_size */
@@ -312,7 +318,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags)
   {
     if (my_flags & MY_WME)
     {
-      if (my_use_large_pages)
+      if (use_large_pages)
       {
         my_printf_error(EE_OUTOFMEMORY,
                         "Couldn't allocate %zu bytes (MEM_LARGE_PAGES page "
@@ -325,7 +331,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags)
         my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_ERROR_LOG), *size);
       }
     }
-    if (my_use_large_pages)
+    if (use_large_pages)
     {
       *size= orig_size;
       ptr= VirtualAlloc(NULL, *size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
@@ -345,7 +351,7 @@ uchar *my_large_malloc(size_t *size, myf my_flags)
   while (1)
   {
     mapflag= MAP_PRIVATE | OS_MAP_ANON;
-    if (my_use_large_pages)
+    if (use_large_pages)
     {
       large_page_size= my_next_large_page_size(*size, &page_i);
       /* this might be 0, in which case we do a standard mmap */
@@ -432,13 +438,21 @@ uchar *my_large_malloc(size_t *size, myf my_flags)
   Special large pages allocator, with possibility to commit to allocating
   more memory later.
   Every implementation returns a zero filled buffer here.
+  Initial protection of returned buffer is readwrite, if MY_TRY_LARGE_PAGES
+  is set in my_flags or on AIX(ask Marko why), but no access otherwise.
+  The caller is expected to call my_virtual_mem_commit() before using memory.
 */
-char *my_large_virtual_alloc(size_t *size)
+char *my_large_virtual_alloc(size_t *size, myf my_flags)
 {
   char *ptr;
+  int prot;
   DBUG_ENTER("my_large_virtual_alloc");
-
-  if (my_use_large_pages)
+#ifdef _AIX
+  prot= PROT_READ | PROT_WRITE;
+#else
+  prot= (my_flags & MY_TRY_LARGE_PAGES) ? PROT_READ|PROT_WRITE : PROT_NONE;
+#endif
+  if (my_flags & MY_TRY_LARGE_PAGES)
   {
     size_t large_page_size;
     int page_i= 0;
@@ -469,7 +483,7 @@ char *my_large_virtual_alloc(size_t *size)
         OS_MAP_ANON;
 
       size_t aligned_size= MY_ALIGN(*size, (size_t) large_page_size);
-      ptr= mmap(NULL, aligned_size, PROT_READ | PROT_WRITE, mapflag, -1, 0);
+      ptr= mmap(NULL, aligned_size, prot, mapflag, -1, 0);
       if (ptr == MAP_FAILED)
       {
         ptr= NULL;
@@ -490,24 +504,20 @@ char *my_large_virtual_alloc(size_t *size)
         DBUG_RETURN(ptr);
       }
     }
-
-    my_use_large_pages= FALSE;
   }
 
-# ifdef _AIX
-  /* On IBM AIX, my_virtual_mem_commit() relies on mprotect(2) rather than
-  a subsequent mmap(2) with MAP_FIXED. */
-  ptr= mmap(NULL, *size, PROT_READ | PROT_WRITE,
-            MAP_PRIVATE | OS_MAP_ANON, -1, 0);
-# else
 /*
   Illumos important to have MAP_NORESERVE otherwise reserves all swap. On
   innodb_buffer_pool_size_max overallocation.
   Linux is controlled on sysctl vm.overcommit_memory.
+
+  MAP_NORESERVE only applies to the PROT_NONE, reserve-only case: like the
+  large page loop above, prot == PROT_READ|PROT_WRITE (MY_TRY_LARGE_PAGES
+  fallback, or AIX) needs memory that is guaranteed to be usable right away.
 */
-  ptr= mmap(NULL, *size, PROT_NONE, MAP_PRIVATE | OS_MAP_ANON | MAP_NORESERVE,
+  ptr= mmap(NULL, *size, prot,
+            MAP_PRIVATE | OS_MAP_ANON | (prot == PROT_NONE ? MAP_NORESERVE : 0),
             -1, 0);
-# endif
   if (ptr == MAP_FAILED)
     ptr= NULL;
 

mysys/my_virtual_mem.c

@@ -37,12 +37,14 @@
   InnoDB, the only user of this functionality), but it's the established
   terminology.
 
-  We try to respect use_large_pages setting, both on Windows and Linux
+  The caller requests large pages by passing MY_TRY_LARGE_PAGES in
+  my_flags (only when my_use_large_pages is set), consistently across
+  the reserve/commit/decommit/release calls for a given allocation.
 */
-char *my_virtual_mem_reserve(size_t *size)
+char *my_virtual_mem_reserve(size_t *size, myf my_flags)
 {
 #ifdef _WIN32
-  DWORD flags= my_use_large_pages
+  DWORD flags= (my_flags & MY_TRY_LARGE_PAGES)
     ? MEM_LARGE_PAGES | MEM_RESERVE | MEM_COMMIT
     : MEM_RESERVE;
   char *ptr= VirtualAlloc(NULL, *size, flags, PAGE_READWRITE);
@@ -55,7 +57,7 @@ char *my_virtual_mem_reserve(size_t *size)
   }
   return ptr;
 #else
-  return my_large_virtual_alloc(size);
+  return my_large_virtual_alloc(size, my_flags);
 #endif
 }
 
@@ -76,12 +78,12 @@ static my_bool is_memory_committed(char *ptr, size_t size)
 
   This is compatible with the mmap / VirtualAlloc semantics.
 */
-char *my_virtual_mem_commit(char *ptr, size_t size)
+char *my_virtual_mem_commit(char *ptr, size_t size, myf my_flags)
 {
 #ifdef _WIN32
   if (!ptr)
     return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
-  if (my_use_large_pages)
+  if (my_flags & MY_TRY_LARGE_PAGES)
   {
     DBUG_ASSERT(is_memory_committed(ptr, size));
   }
@@ -102,7 +104,7 @@ char *my_virtual_mem_commit(char *ptr, size_t size)
                   MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
     return p == MAP_FAILED ? NULL : p;
   }
-  if (my_use_large_pages)
+  if (my_flags & MY_TRY_LARGE_PAGES)
     /* my_large_virtual_alloc() already created a read/write mapping. */;
   else
   {
@@ -142,11 +144,11 @@ char *my_virtual_mem_commit(char *ptr, size_t size)
   return ptr;
 }
 
-void my_virtual_mem_decommit(char *ptr, size_t size)
+void my_virtual_mem_decommit(char *ptr, size_t size, myf my_flags)
 {
 #ifdef _WIN32
   DBUG_ASSERT(is_memory_committed(ptr, size));
-  if (!my_use_large_pages)
+  if (!(my_flags & MY_TRY_LARGE_PAGES))
   {
     if (!VirtualFree(ptr, size, MEM_DECOMMIT))
     {
@@ -183,8 +185,9 @@ void my_virtual_mem_decommit(char *ptr, size_t size)
   update_malloc_size(-(longlong) size, 0);
 }
 
-void my_virtual_mem_release(char *ptr, size_t size)
+void my_virtual_mem_release(char *ptr, size_t size, myf my_flags)
 {
+  (void) my_flags;
 #ifdef _WIN32
   if (!VirtualFree(ptr, 0, MEM_RELEASE))
   {

storage/innobase/buf/buf0buf.cc

@@ -1339,7 +1339,7 @@ bool buf_pool_t::create() noexcept
  retry:
   {
     NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE;
-    memory_unaligned= my_virtual_mem_reserve(&size);
+    memory_unaligned= my_virtual_mem_reserve(&size, my_large_pages_flag());
     if (memory_unaligned);
 #if defined __aarch64__ || defined __riscv || defined __mips__ || defined __loongarch64
     else if (size_in_bytes_max_default != 0 &&
@@ -1373,7 +1373,7 @@ bool buf_pool_t::create() noexcept
 
   if (size < size_in_bytes_max + alignment_waste)
   {
-    my_virtual_mem_release(memory_unaligned, size);
+    my_virtual_mem_release(memory_unaligned, size, my_large_pages_flag());
     size+= 1 +
       (~size_t(memory_unaligned) & (innodb_buffer_pool_extent_size - 1));
     goto retry;
@@ -1398,9 +1398,10 @@ bool buf_pool_t::create() noexcept
   PSI_MEMORY_CALL(memory_alloc)(mem_key_buf_buf_pool, actual_size, &owner);
 #endif
 #ifndef _AIX
-  if (!my_virtual_mem_commit(memory, actual_size))
+  if (!my_virtual_mem_commit(memory, actual_size, my_large_pages_flag()))
   {
-    my_virtual_mem_release(memory_unaligned, size_unaligned);
+    my_virtual_mem_release(memory_unaligned, size_unaligned,
+                           my_large_pages_flag());
     memory= nullptr;
     memory_unaligned= nullptr;
     goto oom;
@@ -1575,8 +1576,9 @@ void buf_pool_t::close() noexcept
     owner= nullptr;
 #endif
     os_total_large_mem_allocated-= size;
-    my_virtual_mem_decommit(memory, size);
-    my_virtual_mem_release(memory_unaligned, size_unaligned);
+    my_virtual_mem_decommit(memory, size, my_large_pages_flag());
+    my_virtual_mem_release(memory_unaligned, size_unaligned,
+                           my_large_pages_flag());
     memory= nullptr;
     memory_unaligned= nullptr;
   }
@@ -1892,7 +1894,7 @@ inline void buf_pool_t::shrunk(size_t size, size_t reduced) noexcept
     guess before we invoke my_virtual_mem_decommit() below. */
     latch.unlock();
   }
-  my_virtual_mem_decommit(memory + size, reduced);
+  my_virtual_mem_decommit(memory + size, reduced, my_large_pages_flag());
 #ifdef UNIV_PFS_MEMORY
   PSI_MEMORY_CALL(memory_free)(mem_key_buf_buf_pool, reduced, owner);
 #endif
@@ -1950,7 +1952,8 @@ ATTRIBUTE_COLD void buf_pool_t::resize(size_t size, THD *thd) noexcept
 
   if (n_blocks_removed <= 0)
   {
-    if (!my_virtual_mem_commit(memory + old_size, size - old_size))
+    if (!my_virtual_mem_commit(memory + old_size, size - old_size,
+                               my_large_pages_flag()))
     {
       mysql_mutex_unlock(&mutex);
       sql_print_error("InnoDB: Cannot commit innodb_buffer_pool_size=%zum;"

storage/maria/ma_pagecache.c

@@ -855,7 +855,8 @@ size_t init_pagecache(PAGECACHE *pagecache, size_t use_mem,
     /* Allocate memory for cache page buffers */
     pagecache->mem_size= blocks * pagecache->block_size;
     if ((pagecache->block_mem=
-      my_large_malloc(&pagecache->mem_size, MYF(MY_WME))))
+      my_large_malloc(&pagecache->mem_size,
+                      MYF(MY_WME) | my_large_pages_flag())))
     {
       /*
         Allocate memory for blocks, hash_links and hash entries;