head 1.1; branch 1.1.1; access ; symbols jemailoc-5-3-1:1.1.1.1 JASONE:1.1.1; locks ; strict; comment @ * @; 1.1 date 2026.04.19.16.20.55; author christos; state Exp; branches 1.1.1.1; next ; commitid otienUsF45k2uACG; 1.1.1.1 date 2026.04.19.16.20.55; author christos; state Exp; branches ; next ; commitid otienUsF45k2uACG; desc @@ 1.1 log @Initial revision @ text @#include "test/jemalloc_test.h" #include "jemalloc/internal/hpa.h" #include "jemalloc/internal/hpa_utils.h" #include "jemalloc/internal/nstime.h" #define SHARD_IND 111 #define ALLOC_MAX (HUGEPAGE) typedef struct test_data_s test_data_t; struct test_data_s { /* * Must be the first member -- we convert back and forth between the * test_data_t and the hpa_shard_t; */ hpa_shard_t shard; hpa_central_t central; base_t *base; edata_cache_t shard_edata_cache; emap_t emap; }; static hpa_shard_opts_t test_hpa_shard_opts_default = { /* slab_max_alloc */ ALLOC_MAX, /* hugification_threshold */ HUGEPAGE, /* dirty_mult */ FXP_INIT_PERCENT(25), /* deferral_allowed */ false, /* hugify_delay_ms */ 10 * 1000, /* hugify_sync */ false, /* min_purge_interval_ms */ 5 * 1000, /* experimental_max_purge_nhp */ -1, /* purge_threshold */ 1, /* min_purge_delay_ms */ 0, /* hugify_style */ hpa_hugify_style_lazy}; static hpa_shard_t * create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) { bool err; base_t *base = base_new(TSDN_NULL, /* ind */ SHARD_IND, &ehooks_default_extent_hooks, /* metadata_use_hooks */ true); assert_ptr_not_null(base, ""); test_data_t *test_data = malloc(sizeof(test_data_t)); assert_ptr_not_null(test_data, ""); test_data->base = base; err = edata_cache_init(&test_data->shard_edata_cache, base); assert_false(err, ""); err = emap_init(&test_data->emap, test_data->base, /* zeroed */ false); assert_false(err, ""); err = hpa_central_init(&test_data->central, test_data->base, hooks); assert_false(err, ""); sec_opts_t sec_opts; sec_opts.nshards = 0; tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central, &test_data->emap, test_data->base, &test_data->shard_edata_cache, SHARD_IND, opts, &sec_opts); assert_false(err, ""); return (hpa_shard_t *)test_data; } static void destroy_test_data(hpa_shard_t *shard) { test_data_t *test_data = (test_data_t *)shard; base_delete(TSDN_NULL, test_data->base); free(test_data); } static uintptr_t defer_bump_ptr = HUGEPAGE * 123; static void * defer_test_map(size_t size) { void *result = (void *)defer_bump_ptr; defer_bump_ptr += size; return result; } static void defer_test_unmap(void *ptr, size_t size) { (void)ptr; (void)size; } static size_t ndefer_purge_calls = 0; static void defer_test_purge(void *ptr, size_t size) { (void)ptr; (void)size; ++ndefer_purge_calls; } static size_t ndefer_vec_purge_calls = 0; static bool defer_vectorized_purge(void *vec, size_t vlen, size_t nbytes) { (void)vec; (void)nbytes; ++ndefer_vec_purge_calls; return false; } static size_t ndefer_hugify_calls = 0; static bool defer_test_hugify(void *ptr, size_t size, bool sync) { ++ndefer_hugify_calls; return false; } static size_t ndefer_dehugify_calls = 0; static void defer_test_dehugify(void *ptr, size_t size) { ++ndefer_dehugify_calls; } static nstime_t defer_curtime; static void defer_test_curtime(nstime_t *r_time, bool first_reading) { *r_time = defer_curtime; } static uint64_t defer_test_ms_since(nstime_t *past_time) { return (nstime_ns(&defer_curtime) - nstime_ns(past_time)) / 1000 / 1000; } TEST_BEGIN(test_vectorized_purge) { test_skip_if(!hpa_supported() || opt_process_madvise_max_batch == 0 || HUGEPAGE_PAGES <= 4); assert(opt_process_madvise_max_batch == 64); hpa_hooks_t hooks; hooks.map = &defer_test_map; hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; hpa_shard_opts_t opts = test_hpa_shard_opts_default; opts.deferral_allowed = true; opts.min_purge_interval_ms = 0; ndefer_vec_purge_calls = 0; ndefer_purge_calls = 0; hpa_shard_t *shard = create_test_data(&hooks, &opts); bool deferred_work_generated = false; nstime_init(&defer_curtime, 0); tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); enum { NALLOCS = 8 * HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Deallocate almost 3 hugepages out of 8, and to force batching * leave the 2nd and 4th PAGE in the first 3 hugepages. */ for (int i = 0; i < 3 * (int)HUGEPAGE_PAGES; i++) { int j = i % HUGEPAGE_PAGES; if (j != 1 && j != 3) { pai_dalloc(tsdn, &shard->pai, edatas[i], &deferred_work_generated); } } hpa_shard_do_deferred_work(tsdn, shard); /* * We purge from 2 huge pages, each one 3 dirty continous segments. * For opt_process_madvise_max_batch = 64, that is all just one call */ expect_zu_eq(1, ndefer_vec_purge_calls, "Expect single purge"); ndefer_vec_purge_calls = 0; destroy_test_data(shard); } TEST_END TEST_BEGIN(test_purge_more_than_one_batch_pages) { test_skip_if(!hpa_supported() || (opt_process_madvise_max_batch < HPA_PURGE_BATCH_MAX) || HUGEPAGE_PAGES <= 4); hpa_hooks_t hooks; hooks.map = &defer_test_map; hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; hpa_shard_opts_t opts = test_hpa_shard_opts_default; opts.deferral_allowed = true; opts.min_purge_interval_ms = 0; opts.dirty_mult = FXP_INIT_PERCENT(1); ndefer_vec_purge_calls = 0; ndefer_purge_calls = 0; ndefer_hugify_calls = 0; ndefer_dehugify_calls = 0; hpa_shard_t *shard = create_test_data(&hooks, &opts); bool deferred_work_generated = false; nstime_init(&defer_curtime, 0); tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); enum { NALLOCS = HPA_PURGE_BATCH_MAX * 3 * HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } for (int i = 0; i < HPA_PURGE_BATCH_MAX * 2 * (int)HUGEPAGE_PAGES; i++) { pai_dalloc( tsdn, &shard->pai, edatas[i], &deferred_work_generated); } hpa_shard_do_deferred_work(tsdn, shard); /* * Strict minimum purge interval is not set, we should purge as long as * we have dirty pages. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* We have page batch size = HPA_PURGE_BATCH_MAX. We have * HPA_PURGE_BATCH_MAX active pages, 2 * HPA_PURGE_BATCH_MAX dirty. * To achieve the balance of 1% max dirty we need to purge more than one * batch. */ size_t nexpected = 2; expect_zu_eq(nexpected, ndefer_vec_purge_calls, "Expect purge"); expect_zu_eq(0, ndefer_purge_calls, "Expect no non-vec purge"); ndefer_vec_purge_calls = 0; destroy_test_data(shard); } TEST_END int main(void) { return test_no_reentrancy( test_vectorized_purge, test_purge_more_than_one_batch_pages); } @ 1.1.1.1 log @Import jemalloc-5.3.1 (previous was 5.3.0) This release includes over 390 commits spanning bug fixes, new features, performance optimizations, and portability improvements. Multiple percent of system-level metric improvements were measured in tested production workloads. The release has gone through large-scale production testing at Meta. New features: Support pvalloc. (@@Lapenkov: 5b1f2cc) Add double free detection for the debug build. (@@izaitsevfb: 36366f3, @@guangli-dai: 42daa1a, @@divanorama: 1897f18) Add compile-time option --enable-pageid to enable memory mapping annotation. (@@devnexen: 4fc5c4f) Add runtime option prof_bt_max to control the max stack depth for profiling. (@@guangli-dai: a0734fd) Add compile-time option --enable-force-getenv to use getenv instead of secure_getenv. (@@interwq: 481bbfc) Add compile-time option --disable-dss to disable the usage of sbrk(2). (@@Svetlitski: ea5b7be) Add runtime option tcache_ncached_max to control the number of items in each size bin in the thread cache. (@@guangli-dai: 8a22d10) Add runtime option calloc_madvise_threshold to determine if kernel or memset is used to zero the allocations for calloc. (@@nullptr0-0: 5081c16) Add compile-time option --disable-user-config to disable reading the runtime configurations from /etc/malloc.conf or environment variable MALLOC_CONF. (@@roblabla: c17bf8b) Add runtime option disable_large_size_classes to guard the new usable size calculation, which minimizes the memory overhead for large allocations, i.e., >= 4 * PAGE. (@@guangli-dai: c067a55, 8347f10) Enable process_madvise usage, add runtime option process_madvise_max_batch to control the max # of regions in each madvise batch. (@@interwq: 22440a0, @@spredolac: 4246475) Add mallctl interfaces: opt.prof_bt_max (@@guangli-dai: a0734fd) arena..name to set and get arena names. (@@guangli-dai: ba19d2c) thread.tcache.max to set and get the tcache_max of the current thread. (@@guangli-dai: a442d9b) thread.tcache.ncached_max.write and thread.tcache.ncached_max.read_sizeclass to set and get the ncached_max setup of the current thread. (@@guangli-dai: 630f7de, 6b197fd) arenas.hugepage to return the hugepage size used, also exported to malloc stats. (@@ilvokhin: 90c627e) approximate_stats.active to return an estimate of the current active bytes, which should not be compared with other stats retrieved. (@@guangli-dai: 0988583) Bug fixes: Prevent potential deadlocks in decaying during reentrancy. (@@interwq: 434a68e) Fix segfault in extent coalescing. (@@Svetlitski: 12311fe) Add null pointer detections in mallctl calls. (@@Svetlitski: dc0a184, 0288126) Make mallctl arenas.lookup triable without crashing on invalid pointers. (@@auxten: 019cccc, 5bac384) Demote sampled allocations for proper deallocations during arena_reset. (@@Svetlitski: 62648c8) Fix jemalloc's read(2) and write(2). (@@Svetlitski: d2c9ed3, @@lexprfuncall: 9fdc116) Fix the pkg-config metadata file. (@@BtbN: ed7e6fe, ce8ce99) Fix the autogen.sh so that it accepts quoted extra options. (@@honggyukim: f6fe6ab) Fix rallocx() to set errno to ENOMEM upon OOMing. (@@arter97: 38056fe, @@interwq: 83b0757) Avoid stack overflow for internal variable array usage. (@@nullptr0-0: 47c9bcd, 48f66cf, @@xinydev: 9169e92) Fix background thread initialization race. (@@puzpuzpuz: 4d0ffa0) Guard os_page_id against a NULL address. (@@lexprfuncall: 79cc7dc) Handle tcache init failures gracefully. (@@lexprfuncall: a056c20) Fix missing release of acquired neighbor edata in extent_try_coalesce_impl. (@@spredolac: 675ab07) Fix memory leak of old curr_reg on san_bump_grow_locked failure. (@@spredolac: 5904a42) Fix large alloc nrequests under-counting on cache misses. (@@spredolac: 3cc56d3) Portability improvements: Fix the build in C99. (@@abaelhe: 56ddbea) Add pthread_setaffinity_np detection for non Linux/BSD platforms. (@@devnexen: 4c95c95) Make VARIABLE_ARRAY compatible with compilers not supporting VLA, i.e., Visual Studio C compiler in C11 or C17 modes. (@@madscientist: be65438) Fix the build on Linux using musl library. (@@marv: aba1645, 45249cf) Reduce the memory overhead in small allocation sampling for systems with larger page sizes, e.g., ARM. (@@Svetlitski: 5a858c6) Add C23's free_sized and free_aligned_sized. (@@Svetlitski: cdb2c0e) Enable heap profiling on MacOS. (@@nullptr0-0: 4b555c1) Fix incorrect printing on 32bit. (@@sundb: 630434b) Make JEMALLOC_CXX_THROW compatible with C++ versions newer than C++17. (@@r-barnes, @@guangli-dai: 21bcc0a) Fix mmap tag conflicts on MacOS. (@@kdrag0n: c893fcd) Fix monotonic timer assumption for win32. (@@burtonli: 8dc97b1) Fix VM over-reservation on systems with larger pages, e.g., aarch64. (@@interwq: cd05b19) Remove unreachable() macro conditionally to prevent definition conflicts for C23+. (@@appujee: d8486b2, 4b88bdd) Fix dlsym failure observed on FreeBSD. (@@rhelmot: 86bbaba) Change the default page size to 64KB on aarch64 Linux. (@@lexprfuncall: 9442300) Update config.guess and config.sub to the latest version. (@@lexprfuncall: c51949e) Determine the page size on Android from NDK header files. (@@lexprfuncall: c51abba) Improve the portability of grep patterns in configure.ac. (@@lexprfuncall: 365747b) Add compile-time option --with-cxx-stdlib to specify the C++ standard library. (@@yuxuanchen1997: a10ef3e) Optimizations and refactors: Enable tcache for deallocation-only threads. (@@interwq: 143e9c4) Inline to accelerate operator delete. (@@guangli-dai: e8f9f13) Optimize pairing heap's performance. (@@deadalnix: 5266152, be6da4f, 543e2d6, 10d7131, 92aa52c, @@Svetlitski: 36ca0c1) Inline the storage for thread name in the profiling data. (@@interwq: ce0b7ab, e62aa47) Optimize a hot function edata_cmp_summary_comp to accelerate it. (@@Svetlitski: 6841110, @@guangli-dai: 0181aaa) Allocate thread cache using the base allocator, which enables thread cache to use thp when metadata_thp is turned on. (@@interwq: 72cfdce) Allow oversize arena not to purge immediately when background threads are enabled, although the default decay time is 0 to be back compatible. (@@interwq: d131331) Optimize thread-local storage implementation on Windows. (@@mcfi: 9e123a8, 3a0d9cd) Optimize fast path to allow static size class computation. (@@interwq: 323ed2e) Redesign tcache GC to regulate the frequency and make it locality-aware. The new design is default on, guarded by option experimental_tcache_gc. (@@nullptr0-0: 0c88be9, e2c9f3a, 14d5dc1, @@deadalnix: 5afff2e) Reduce the arena switching overhead by avoiding forced purging when background thread is enabled. (@@interwq: a3910b9) Improve the reuse efficiency by limiting the maximum coalesced size for large extents. (@@jiebinn: 3c14707) Refactor thread events to allow registration of users' thread events and remove prof_threshold as the built-in event. (@@spredolac: e6864c6, 015b017, 34ace91) Documentation: Update Windows building instructions. (@@Lapenkov: 3713932) Add vcpkg installation instructions. (@@LilyWangLL: c0c9783) Update profiling internals with an example. (@@jordalgo: b04e766) @ text @@