Madvise Transparent Huge Pages for large allocations (#59858)

To get things going with #56521
I've made a minimal implementation that mirrors one from numpy
(https://github.com/numpy/numpy/blob/7c0e2e4224c6feb04a2ac4aa851f49a2c2f6189f/numpy/_core/src/multiarray/alloc.c#L113).

What this does: changes `jl_gc_managed_malloc(size_t sz)` to check
whether requested allocation is big enough to benefit from huge pages.
And if so, we ensure the allocation is page aligned and then appropriate
`madvise` is called on the memory pointer.

For a simple "fill memory" test I see around 2x timing improvement.
```julia
function f(N)
    mem =  Memory{Int}(undef, N)
    mem .= 0
    mem[end]
end

f(1)
@time f(1_000_000)
```
```
0.001464 seconds (2 allocations: 7.633 MiB) # this branch
0.003431 seconds (2 allocations: 7.633 MiB) # master
```

I would appreciate help with this PR as I have no experience writing C
code and little knowledge of julia internals. In particular I think it
would make sense to have a startup option controlling minimal eligible
allocation size which should default to system's hugepage size - for
this initial implementation the same constant as in numpy is hardcoded.

---------

Co-authored-by: Artem Solod <>

Author: artemsolod

src/gc-common.h

@@ -7,6 +7,9 @@
 #include "julia_internal.h"
 #ifndef _OS_WINDOWS_
 #include <sys/mman.h>
+#ifndef MADV_HUGEPAGE
+#define MADV_HUGEPAGE 14 //  Compatibility for kernels < 2.6.38 (as in numpy implementation)
+#endif
 #if defined(_OS_DARWIN_) && !defined(MAP_ANONYMOUS)
 #define MAP_ANONYMOUS MAP_ANON
 #endif
@@ -135,6 +138,7 @@ STATIC_INLINE void jl_free_aligned(void *p) JL_NOTSAFEPOINT
 #endif
 #define malloc_cache_align(sz) jl_malloc_aligned(sz, JL_CACHE_BYTE_ALIGNMENT)
 #define realloc_cache_align(p, sz, oldsz) jl_realloc_aligned(p, sz, oldsz, JL_CACHE_BYTE_ALIGNMENT)
+#define malloc_page_align(sz) jl_malloc_aligned(sz, jl_getpagesize())
 
 // =========================================================================== //
 // Pointer tagging

src/gc-stock.c

@@ -3854,7 +3854,22 @@ JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz)
 #ifdef _OS_WINDOWS_
     DWORD last_error = GetLastError();
 #endif
-    void *b = malloc_cache_align(allocsz);
+    #ifdef MADV_HUGEPAGE
+        void *b = NULL;
+        if (allocsz >= 1u<<18u) {
+            b = malloc_page_align(allocsz);
+            if (jl_hugepage_size > 0) {
+                size_t leftover = jl_hugepage_size - (allocsz % jl_hugepage_size);
+                if ((leftover <= allocsz / 4) || (leftover == jl_hugepage_size))  // limit fragmentation
+                    madvise(b, allocsz, MADV_HUGEPAGE);
+            }
+        }
+        else {
+            b = malloc_cache_align(allocsz);
+        }
+    #else
+        void *b = malloc_cache_align(allocsz);
+    #endif
     if (b == NULL)
         jl_throw(jl_memory_exception);
 

src/init.c

@@ -48,6 +48,7 @@ extern void jl_fin_stackwalk(void);
 jl_array_t *jl_module_init_order;
 
 JL_DLLEXPORT size_t jl_page_size;
+JL_DLLEXPORT size_t jl_hugepage_size;
 
 void jl_init_stack_limits(int ismaster, void **stack_lo, void **stack_hi)
 {
@@ -715,6 +716,7 @@ JL_DLLEXPORT void jl_init_(jl_image_buf_t sysimage)
     libsupport_init();
     jl_safepoint_init();
     jl_page_size = jl_getpagesize();
+    jl_hugepage_size = (size_t)jl_gethugepagesize();
     htable_new(&jl_current_modules, 0);
     init_global_mutexes();
     jl_precompile_toplevel_module = NULL;

src/jl_exported_funcs.inc

@@ -182,6 +182,7 @@
     XX(jl_gensym) \
     XX(jl_getaffinity) \
     XX(jl_getallocationgranularity) \
+    XX(jl_gethugepagesize) \
     XX(jl_getnameinfo) \
     XX(jl_getpagesize) \
     XX(jl_get_ARCH) \

src/julia.h

@@ -2066,6 +2066,7 @@ JL_DLLEXPORT int jl_cpu_threads(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int jl_effective_threads(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT long jl_getpagesize(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT long jl_getallocationgranularity(void) JL_NOTSAFEPOINT;
+JL_DLLEXPORT long jl_gethugepagesize(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT int jl_is_debugbuild(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT jl_sym_t *jl_get_UNAME(void) JL_NOTSAFEPOINT;
 JL_DLLEXPORT jl_sym_t *jl_get_ARCH(void) JL_NOTSAFEPOINT;

src/julia_internal.h

@@ -440,6 +440,7 @@ extern arraylist_t eytzinger_image_tree;
 extern arraylist_t eytzinger_idxs;
 
 extern JL_DLLEXPORT size_t jl_page_size;
+extern JL_DLLEXPORT size_t jl_hugepage_size;
 extern JL_DLLEXPORT jl_value_t *jl_typeinf_func JL_GLOBALLY_ROOTED;
 extern JL_DLLEXPORT jl_value_t *jl_compile_and_emit_func JL_GLOBALLY_ROOTED;
 extern JL_DLLEXPORT size_t jl_typeinf_world;

src/sys.c

@@ -10,6 +10,7 @@
 #include <errno.h>
 #include <signal.h>
 #include <fcntl.h>
+#include <stdio.h>
 
 #include "julia.h"
 #include "julia_internal.h"
@@ -610,6 +611,41 @@ JL_DLLEXPORT long jl_getallocationgranularity(void) JL_NOTSAFEPOINT
 }
 #endif
 
+JL_DLLEXPORT long jl_gethugepagesize(void) JL_NOTSAFEPOINT
+{
+#if defined(_OS_LINUX_)
+    long detected = 0;
+    FILE *f = fopen("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", "r");
+    if (f) {
+        unsigned long long size = 0;
+        if (fscanf(f, "%llu", &size) == 1 && size > 0) {
+            detected = (long)size;
+        }
+        fclose(f);
+    }
+    if (detected == 0) {
+        f = fopen("/proc/meminfo", "r");
+        if (f) {
+            char line[256];
+            while (fgets(line, sizeof(line), f)) {
+                unsigned long long kb = 0;
+                if (sscanf(line, "Hugepagesize:%llu kB", &kb) == 1 && kb > 0) {
+                    detected = (long)(kb * 1024ULL);
+                    break;
+                }
+            }
+            fclose(f);
+        }
+    }
+    if (detected == 0) {
+        detected = 2 * 1024 * 1024; // 2 MiB fallback
+    }
+    return detected;
+#else
+    return 0;
+#endif
+}
+
 JL_DLLEXPORT long jl_SC_CLK_TCK(void)
 {
 #ifndef _OS_WINDOWS_