From 0ca6be039fe09e4963e166848cb29e20f2dcf699 Mon Sep 17 00:00:00 2001 From: Laurens Kuiper Date: Thu, 8 Aug 2024 11:02:12 +0200 Subject: [PATCH] update jemalloc --- extension/jemalloc/jemalloc/CMakeLists.txt | 1 + extension/jemalloc/jemalloc/README.md | 4 +- .../include/jemalloc/internal/arena_externs.h | 1 - .../jemalloc/internal/arena_inlines_b.h | 155 +++++++++++++++++- .../include/jemalloc/internal/arena_structs.h | 2 +- .../include/jemalloc/internal/batcher.h | 46 ++++++ .../jemalloc/include/jemalloc/internal/bin.h | 74 ++++++++- .../include/jemalloc/internal/bin_info.h | 11 ++ .../include/jemalloc/internal/bin_stats.h | 5 + .../jemalloc/include/jemalloc/internal/emap.h | 4 +- .../include/jemalloc/internal/hpa_opts.h | 12 +- .../internal/jemalloc_internal_defs.h | 6 +- .../internal/jemalloc_internal_macros.h | 12 ++ .../jemalloc/internal/public_namespace.h | 4 +- .../jemalloc/internal/public_unnamespace.h | 4 +- .../include/jemalloc/internal/witness.h | 3 +- .../jemalloc/include/jemalloc/jemalloc.h | 30 ++-- .../jemalloc/include/jemalloc/jemalloc_defs.h | 4 +- .../include/jemalloc/jemalloc_macros.h | 10 +- .../include/jemalloc/jemalloc_mangle.h | 8 +- .../include/jemalloc/jemalloc_mangle_jet.h | 8 +- .../include/jemalloc/jemalloc_rename.h | 4 +- extension/jemalloc/jemalloc/src/arena.c | 98 ++++++----- extension/jemalloc/jemalloc/src/batcher.c | 96 +++++++++++ extension/jemalloc/jemalloc/src/bin.c | 48 +++++- extension/jemalloc/jemalloc/src/bin_info.c | 24 +++ extension/jemalloc/jemalloc/src/emap.c | 2 +- extension/jemalloc/jemalloc/src/hpa.c | 23 ++- extension/jemalloc/jemalloc/src/jemalloc.c | 28 +++- extension/jemalloc/jemalloc/src/malloc_io.c | 82 +-------- extension/jemalloc/jemalloc/src/pages.c | 2 +- extension/jemalloc/jemalloc_config.py | 1 + extension/jemalloc/jemalloc_extension.cpp | 2 +- 33 files changed, 640 insertions(+), 174 deletions(-) create mode 100644 extension/jemalloc/jemalloc/include/jemalloc/internal/batcher.h create mode 100644 extension/jemalloc/jemalloc/src/batcher.c diff --git a/extension/jemalloc/jemalloc/CMakeLists.txt b/extension/jemalloc/jemalloc/CMakeLists.txt index 129899baded..fda94d989b9 100644 --- a/extension/jemalloc/jemalloc/CMakeLists.txt +++ b/extension/jemalloc/jemalloc/CMakeLists.txt @@ -5,6 +5,7 @@ set(JEMALLOC_C_FILES src/arena.c src/background_thread.c src/base.c + src/batcher.c src/bin.c src/bin_info.c src/bitmap.c diff --git a/extension/jemalloc/jemalloc/README.md b/extension/jemalloc/jemalloc/README.md index 090afffa92b..9e971d5fb56 100644 --- a/extension/jemalloc/jemalloc/README.md +++ b/extension/jemalloc/jemalloc/README.md @@ -65,7 +65,7 @@ Add this to `jemalloc.h`: We also supply our own config string in `jemalloc.c`. Define this just after the `#include`s. ```c++ -#define JE_MALLOC_CONF_BUFFER_SIZE 200; +#define JE_MALLOC_CONF_BUFFER_SIZE 200 char JE_MALLOC_CONF_BUFFER[JE_MALLOC_CONF_BUFFER_SIZE]; ``` This is what `jemalloc_constructor` in `jemalloc.c` should look like: @@ -79,7 +79,7 @@ jemalloc_constructor(void) { bgt_count = 1; } // decay is in ms - unsigned long long decay = DUCKDB_DECAY_DELAY * 1000; + unsigned long long decay = DUCKDB_JEMALLOC_DECAY * 1000; #ifdef DEBUG snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "junk:true,oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, cpu_count, bgt_count); #else diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_externs.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_externs.h index f91bd888f97..3d0329fce33 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_externs.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_externs.h @@ -103,7 +103,6 @@ void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config); bool arena_init_huge(arena_t *a0); -bool arena_is_huge(unsigned arena_ind); arena_t *arena_choose_huge(tsd_t *tsd); bin_t *arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned *binshard); diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_inlines_b.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_inlines_b.h index a891b35cd40..ea246cc5bdd 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_inlines_b.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_inlines_b.h @@ -563,10 +563,11 @@ arena_dalloc_bin_locked_begin(arena_dalloc_bin_locked_info_t *info, * stats updates, which happen during finish (this lets running counts get left * in a register). */ -JEMALLOC_ALWAYS_INLINE bool +JEMALLOC_ALWAYS_INLINE void arena_dalloc_bin_locked_step(tsdn_t *tsdn, arena_t *arena, bin_t *bin, arena_dalloc_bin_locked_info_t *info, szind_t binind, edata_t *slab, - void *ptr) { + void *ptr, edata_t **dalloc_slabs, unsigned ndalloc_slabs, + unsigned *dalloc_slabs_count, edata_list_active_t *dalloc_slabs_extra) { const bin_info_t *bin_info = &bin_infos[binind]; size_t regind = arena_slab_regind(info, binind, slab, ptr); slab_data_t *slab_data = edata_slab_data_get(slab); @@ -586,12 +587,17 @@ arena_dalloc_bin_locked_step(tsdn_t *tsdn, arena_t *arena, bin_t *bin, if (nfree == bin_info->nregs) { arena_dalloc_bin_locked_handle_newly_empty(tsdn, arena, slab, bin); - return true; + + if (*dalloc_slabs_count < ndalloc_slabs) { + dalloc_slabs[*dalloc_slabs_count] = slab; + (*dalloc_slabs_count)++; + } else { + edata_list_active_append(dalloc_slabs_extra, slab); + } } else if (nfree == 1 && slab != bin->slabcur) { arena_dalloc_bin_locked_handle_newly_nonempty(tsdn, arena, slab, bin); } - return false; } JEMALLOC_ALWAYS_INLINE void @@ -604,10 +610,149 @@ arena_dalloc_bin_locked_finish(tsdn_t *tsdn, arena_t *arena, bin_t *bin, } } +JEMALLOC_ALWAYS_INLINE void +arena_bin_flush_batch_impl(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + arena_dalloc_bin_locked_info_t *dalloc_bin_info, unsigned binind, + edata_t **dalloc_slabs, unsigned ndalloc_slabs, unsigned *dalloc_count, + edata_list_active_t *dalloc_slabs_extra) { + assert(binind < bin_info_nbatched_sizes); + bin_with_batch_t *batched_bin = (bin_with_batch_t *)bin; + size_t nelems_to_pop = batcher_pop_begin(tsdn, + &batched_bin->remote_frees); + + bin_batching_test_mid_pop(nelems_to_pop); + if (nelems_to_pop == BATCHER_NO_IDX) { + malloc_mutex_assert_not_owner(tsdn, + &batched_bin->remote_frees.mtx); + return; + } else { + malloc_mutex_assert_owner(tsdn, + &batched_bin->remote_frees.mtx); + } + + size_t npushes = batcher_pop_get_pushes(tsdn, + &batched_bin->remote_frees); + bin_remote_free_data_t remote_free_data[BIN_REMOTE_FREE_ELEMS_MAX]; + for (size_t i = 0; i < nelems_to_pop; i++) { + remote_free_data[i] = batched_bin->remote_free_data[i]; + } + batcher_pop_end(tsdn, &batched_bin->remote_frees); + + for (size_t i = 0; i < nelems_to_pop; i++) { + arena_dalloc_bin_locked_step(tsdn, arena, bin, dalloc_bin_info, + binind, remote_free_data[i].slab, remote_free_data[i].ptr, + dalloc_slabs, ndalloc_slabs, dalloc_count, + dalloc_slabs_extra); + } + + bin->stats.batch_pops++; + bin->stats.batch_pushes += npushes; + bin->stats.batch_pushed_elems += nelems_to_pop; +} + +typedef struct arena_bin_flush_batch_state_s arena_bin_flush_batch_state_t; +struct arena_bin_flush_batch_state_s { + arena_dalloc_bin_locked_info_t info; + + /* + * Bin batching is subtle in that there are unusual edge cases in which + * it can trigger the deallocation of more slabs than there were items + * flushed (say, if every original deallocation triggered a slab + * deallocation, and so did every batched one). So we keep a small + * backup array for any "extra" slabs, as well as a a list to allow a + * dynamic number of ones exceeding that array. + */ + edata_t *dalloc_slabs[8]; + unsigned dalloc_slab_count; + edata_list_active_t dalloc_slabs_extra; +}; + +JEMALLOC_ALWAYS_INLINE unsigned +arena_bin_batch_get_ndalloc_slabs(unsigned preallocated_slabs) { + if (preallocated_slabs > bin_batching_test_ndalloc_slabs_max) { + return bin_batching_test_ndalloc_slabs_max; + } + return preallocated_slabs; +} + +JEMALLOC_ALWAYS_INLINE void +arena_bin_flush_batch_after_lock(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + unsigned binind, arena_bin_flush_batch_state_t *state) { + if (binind >= bin_info_nbatched_sizes) { + return; + } + + arena_dalloc_bin_locked_begin(&state->info, binind); + state->dalloc_slab_count = 0; + edata_list_active_init(&state->dalloc_slabs_extra); + + unsigned preallocated_slabs = (unsigned)(sizeof(state->dalloc_slabs) + / sizeof(state->dalloc_slabs[0])); + unsigned ndalloc_slabs = arena_bin_batch_get_ndalloc_slabs( + preallocated_slabs); + + arena_bin_flush_batch_impl(tsdn, arena, bin, &state->info, binind, + state->dalloc_slabs, ndalloc_slabs, + &state->dalloc_slab_count, &state->dalloc_slabs_extra); +} + +JEMALLOC_ALWAYS_INLINE void +arena_bin_flush_batch_before_unlock(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + unsigned binind, arena_bin_flush_batch_state_t *state) { + if (binind >= bin_info_nbatched_sizes) { + return; + } + + arena_dalloc_bin_locked_finish(tsdn, arena, bin, &state->info); +} + +static inline bool +arena_bin_has_batch(szind_t binind) { + return binind < bin_info_nbatched_sizes; +} + +JEMALLOC_ALWAYS_INLINE void +arena_bin_flush_batch_after_unlock(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + unsigned binind, arena_bin_flush_batch_state_t *state) { + if (!arena_bin_has_batch(binind)) { + return; + } + /* + * The initialization of dalloc_slabs_extra is guarded by an + * arena_bin_has_batch check higher up the stack. But the clang + * analyzer forgets this down the stack, triggering a spurious error + * reported here. + */ + JEMALLOC_CLANG_ANALYZER_SUPPRESS { + bin_batching_test_after_unlock(state->dalloc_slab_count, + edata_list_active_empty(&state->dalloc_slabs_extra)); + } + for (unsigned i = 0; i < state->dalloc_slab_count; i++) { + edata_t *slab = state->dalloc_slabs[i]; + arena_slab_dalloc(tsdn, arena_get_from_edata(slab), slab); + } + while (!edata_list_active_empty(&state->dalloc_slabs_extra)) { + edata_t *slab = edata_list_active_first( + &state->dalloc_slabs_extra); + edata_list_active_remove(&state->dalloc_slabs_extra, slab); + arena_slab_dalloc(tsdn, arena_get_from_edata(slab), slab); + } +} + static inline bin_t * arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) { bin_t *shard0 = (bin_t *)((byte_t *)arena + arena_bin_offsets[binind]); - return shard0 + binshard; + bin_t *ret; + if (arena_bin_has_batch(binind)) { + ret = (bin_t *)((bin_with_batch_t *)shard0 + binshard); + } else { + ret = shard0 + binshard; + } + assert(binind >= SC_NBINS - 1 + || (uintptr_t)ret < (uintptr_t)arena + + arena_bin_offsets[binind + 1]); + + return ret; } #endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */ diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_structs.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_structs.h index 803ed25c81c..56e12f958fe 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_structs.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/arena_structs.h @@ -104,7 +104,7 @@ struct arena_s { JEMALLOC_WARN_ON_USAGE("Do not use this field directly. " "Use `arena_get_bin` instead.") JEMALLOC_ALIGNED(CACHELINE) - bin_t all_bins[0]; + bin_with_batch_t all_bins[0]; }; #endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_H */ diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/batcher.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/batcher.h new file mode 100644 index 00000000000..40c8b35f7ce --- /dev/null +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/batcher.h @@ -0,0 +1,46 @@ +#ifndef JEMALLOC_INTERNAL_BATCHER_H +#define JEMALLOC_INTERNAL_BATCHER_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/mutex.h" + +#define BATCHER_NO_IDX ((size_t)-1) + +typedef struct batcher_s batcher_t; +struct batcher_s { + /* + * Optimize for locality -- nelems_max and nelems are always touched + * togehter, along with the front of the mutex. The end of the mutex is + * only touched if there's contention. + */ + atomic_zu_t nelems; + size_t nelems_max; + size_t npushes; + malloc_mutex_t mtx; +}; + +void batcher_init(batcher_t *batcher, size_t nelems_max); + +/* + * Returns an index (into some user-owned array) to use for pushing, or + * BATCHER_NO_IDX if no index is free. If the former, the caller must call + * batcher_push_end once done. + */ +size_t batcher_push_begin(tsdn_t *tsdn, batcher_t *batcher, + size_t elems_to_push); +void batcher_push_end(tsdn_t *tsdn, batcher_t *batcher); + +/* + * Returns the number of items to pop, or BATCHER_NO_IDX if there are none. + * If the former, must be followed by a call to batcher_pop_end. + */ +size_t batcher_pop_begin(tsdn_t *tsdn, batcher_t *batcher); +size_t batcher_pop_get_pushes(tsdn_t *tsdn, batcher_t *batcher); +void batcher_pop_end(tsdn_t *tsdn, batcher_t *batcher); + +void batcher_prefork(tsdn_t *tsdn, batcher_t *batcher); +void batcher_postfork_parent(tsdn_t *tsdn, batcher_t *batcher); +void batcher_postfork_child(tsdn_t *tsdn, batcher_t *batcher); + +#endif /* JEMALLOC_INTERNAL_BATCHER_H */ diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/bin.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/bin.h index ed27c18f1ee..c49afea6db2 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/bin.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/bin.h @@ -2,12 +2,60 @@ #define JEMALLOC_INTERNAL_BIN_H #include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/batcher.h" #include "jemalloc/internal/bin_stats.h" #include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/edata.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/sc.h" +#define BIN_REMOTE_FREE_ELEMS_MAX 16 + +#ifdef JEMALLOC_JET +extern void (*bin_batching_test_after_push_hook)(size_t idx); +extern void (*bin_batching_test_mid_pop_hook)(size_t elems_to_pop); +extern void (*bin_batching_test_after_unlock_hook)(unsigned slab_dalloc_count, + bool list_empty); +#endif + +#ifdef JEMALLOC_JET +extern unsigned bin_batching_test_ndalloc_slabs_max; +#else +static const unsigned bin_batching_test_ndalloc_slabs_max = (unsigned)-1; +#endif + +JEMALLOC_ALWAYS_INLINE void +bin_batching_test_after_push(size_t idx) { + (void)idx; +#ifdef JEMALLOC_JET + if (bin_batching_test_after_push_hook != NULL) { + bin_batching_test_after_push_hook(idx); + } +#endif +} + +JEMALLOC_ALWAYS_INLINE void +bin_batching_test_mid_pop(size_t elems_to_pop) { + (void)elems_to_pop; +#ifdef JEMALLOC_JET + if (bin_batching_test_mid_pop_hook != NULL) { + bin_batching_test_mid_pop_hook(elems_to_pop); + } +#endif +} + +JEMALLOC_ALWAYS_INLINE void +bin_batching_test_after_unlock(unsigned slab_dalloc_count, bool list_empty) { + (void)slab_dalloc_count; + (void)list_empty; +#ifdef JEMALLOC_JET + if (bin_batching_test_after_unlock_hook != NULL) { + bin_batching_test_after_unlock_hook(slab_dalloc_count, + list_empty); + } +#endif +} + /* * A bin contains a set of extents that are currently being used for slab * allocations. @@ -42,6 +90,19 @@ struct bin_s { edata_list_active_t slabs_full; }; +typedef struct bin_remote_free_data_s bin_remote_free_data_t; +struct bin_remote_free_data_s { + void *ptr; + edata_t *slab; +}; + +typedef struct bin_with_batch_s bin_with_batch_t; +struct bin_with_batch_s { + bin_t bin; + batcher_t remote_frees; + bin_remote_free_data_t remote_free_data[BIN_REMOTE_FREE_ELEMS_MAX]; +}; + /* A set of sharded bins of the same size class. */ typedef struct bins_s bins_t; struct bins_s { @@ -54,12 +115,12 @@ bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size, size_t end_size, size_t nshards); /* Initializes a bin to empty. Returns true on error. */ -bool bin_init(bin_t *bin); +bool bin_init(bin_t *bin, unsigned binind); /* Forking. */ -void bin_prefork(tsdn_t *tsdn, bin_t *bin); -void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin); -void bin_postfork_child(tsdn_t *tsdn, bin_t *bin); +void bin_prefork(tsdn_t *tsdn, bin_t *bin, bool has_batch); +void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin, bool has_batch); +void bin_postfork_child(tsdn_t *tsdn, bin_t *bin, bool has_batch); /* Stats. */ static inline void @@ -77,6 +138,11 @@ bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) { stats->reslabs += bin->stats.reslabs; stats->curslabs += bin->stats.curslabs; stats->nonfull_slabs += bin->stats.nonfull_slabs; + + stats->batch_failed_pushes += bin->stats.batch_failed_pushes; + stats->batch_pushes += bin->stats.batch_pushes; + stats->batch_pushed_elems += bin->stats.batch_pushed_elems; + malloc_mutex_unlock(tsdn, &bin->lock); } diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/bin_info.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/bin_info.h index b6175550e1c..88d58c91650 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/bin_info.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/bin_info.h @@ -44,6 +44,17 @@ struct bin_info_s { bitmap_info_t bitmap_info; }; +/* The maximum size a size class can be and still get batching behavior. */ +extern size_t opt_bin_info_max_batched_size; +/* The number of batches per batched size class. */ +extern size_t opt_bin_info_remote_free_max_batch; +// The max number of pending elems (across all batches) +extern size_t opt_bin_info_remote_free_max; + +extern szind_t bin_info_nbatched_sizes; +extern unsigned bin_info_nbatched_bins; +extern unsigned bin_info_nunbatched_bins; + extern bin_info_t bin_infos[SC_NBINS]; void bin_info_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]); diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/bin_stats.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/bin_stats.h index f95b9e9c445..334c166de20 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/bin_stats.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/bin_stats.h @@ -48,6 +48,11 @@ struct bin_stats_s { /* Current size of nonfull slabs heap in this bin. */ size_t nonfull_slabs; + + uint64_t batch_pops; + uint64_t batch_failed_pushes; + uint64_t batch_pushes; + uint64_t batch_pushed_elems; }; typedef struct bin_stats_data_s bin_stats_data_t; diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/emap.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/emap.h index 08262f1f1e7..7ac0ae95a8d 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/emap.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/emap.h @@ -186,13 +186,13 @@ emap_edata_is_acquired(tsdn_t *tsdn, emap_t *emap, edata_t *edata) { */ EMAP_DECLARE_RTREE_CTX; rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &emap->rtree, - rtree_ctx, (uintptr_t)edata_base_get(edata), /* dependent */ true, + rtree_ctx, (uintptr_t)edata_base_get(edata), /* dependent */ false, /* init_missing */ false); if (elm == NULL) { return true; } rtree_contents_t contents = rtree_leaf_elm_read(tsdn, &emap->rtree, elm, - /* dependent */ true); + /* dependent */ false); if (contents.edata == NULL || contents.metadata.state == extent_state_active || edata_state_in_transition(contents.metadata.state)) { diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/hpa_opts.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/hpa_opts.h index 6e58c86b056..93add641d79 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/hpa_opts.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/hpa_opts.h @@ -49,6 +49,14 @@ struct hpa_shard_opts_s { * Minimum amount of time between purges. */ uint64_t min_purge_interval_ms; + + /* + * Strictly respect minimum amout of time between purges. + * + * This is an option to provide backward compatibility for staged rollout of + * purging logic fix. + */ + bool strict_min_purge_interval; }; #define HPA_SHARD_OPTS_DEFAULT { \ @@ -69,7 +77,9 @@ struct hpa_shard_opts_s { /* hugify_delay_ms */ \ 10 * 1000, \ /* min_purge_interval_ms */ \ - 5 * 1000 \ + 5 * 1000, \ + /* strict_min_purge_interval */ \ + false \ } #endif /* JEMALLOC_INTERNAL_HPA_OPTS_H */ diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h index 33342ffb37d..f583cd6be63 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h @@ -309,7 +309,7 @@ */ #ifdef _MSC_VER [[noreturn]] __forceinline void msvc_unreachable() { - __assume(false); + __assume(false); } #define JEMALLOC_INTERNAL_UNREACHABLE msvc_unreachable #else @@ -564,7 +564,9 @@ #endif /* If defined, use volatile asm during benchmarks. */ -// #define JEMALLOC_HAVE_ASM_VOLATILE +#ifdef __APPLE__ +#define JEMALLOC_HAVE_ASM_VOLATILE +#endif /* * If defined, support the use of rdtscp to get the time stamp counter diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h index 40df5febcdf..407e868aea6 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h @@ -116,6 +116,18 @@ # define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS #endif +#ifdef __clang_analyzer__ +# define JEMALLOC_CLANG_ANALYZER +#endif + +#ifdef JEMALLOC_CLANG_ANALYZER +# define JEMALLOC_CLANG_ANALYZER_SUPPRESS __attribute__((suppress)) +# define JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(v) = v +#else +# define JEMALLOC_CLANG_ANALYZER_SUPPRESS +# define JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(v) +#endif + #define JEMALLOC_SUPPRESS_WARN_ON_USAGE(...) \ JEMALLOC_DIAGNOSTIC_PUSH \ JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED \ diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/public_namespace.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/public_namespace.h index 3a9419837ff..64cedadfb66 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/public_namespace.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/public_namespace.h @@ -2,6 +2,8 @@ #define je_calloc JEMALLOC_N(calloc) #define je_dallocx JEMALLOC_N(dallocx) #define je_free JEMALLOC_N(free) +#define je_free_sized JEMALLOC_N(free_sized) +#define je_free_aligned_sized JEMALLOC_N(free_aligned_sized) #define je_mallctl JEMALLOC_N(mallctl) #define je_mallctlbymib JEMALLOC_N(mallctlbymib) #define je_mallctlnametomib JEMALLOC_N(mallctlnametomib) @@ -12,7 +14,7 @@ #define je_malloc_stats_print JEMALLOC_N(malloc_stats_print) #define je_malloc_usable_size JEMALLOC_N(malloc_usable_size) #define je_mallocx JEMALLOC_N(mallocx) -#define je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 JEMALLOC_N(smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9) +#define je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 JEMALLOC_N(smallocx_a25b9b8ba91881964be3083db349991bbbbf1661) #define je_nallocx JEMALLOC_N(nallocx) #define je_posix_memalign JEMALLOC_N(posix_memalign) #define je_rallocx JEMALLOC_N(rallocx) diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/public_unnamespace.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/public_unnamespace.h index bbd77d46e80..7709fd7d2df 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/public_unnamespace.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/public_unnamespace.h @@ -2,6 +2,8 @@ #undef je_calloc #undef je_dallocx #undef je_free +#undef je_free_sized +#undef je_free_aligned_sized #undef je_mallctl #undef je_mallctlbymib #undef je_mallctlnametomib @@ -12,7 +14,7 @@ #undef je_malloc_stats_print #undef je_malloc_usable_size #undef je_mallocx -#undef je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 +#undef je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 #undef je_nallocx #undef je_posix_memalign #undef je_rallocx diff --git a/extension/jemalloc/jemalloc/include/jemalloc/internal/witness.h b/extension/jemalloc/jemalloc/include/jemalloc/internal/witness.h index 937ca2d5803..afee1246af5 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/internal/witness.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/internal/witness.h @@ -64,9 +64,10 @@ enum witness_rank_e { WITNESS_RANK_BASE, WITNESS_RANK_ARENA_LARGE, WITNESS_RANK_HOOK, + WITNESS_RANK_BIN, WITNESS_RANK_LEAF=0x1000, - WITNESS_RANK_BIN = WITNESS_RANK_LEAF, + WITNESS_RANK_BATCHER=WITNESS_RANK_LEAF, WITNESS_RANK_ARENA_STATS = WITNESS_RANK_LEAF, WITNESS_RANK_COUNTER_ACCUM = WITNESS_RANK_LEAF, WITNESS_RANK_DSS = WITNESS_RANK_LEAF, diff --git a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc.h b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc.h index b728d5df949..b92f42ab48f 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc.h @@ -6,7 +6,7 @@ extern "C" { #endif // DuckDB uses a 10s decay -#define DUCKDB_DECAY_DELAY 10 +#define DUCKDB_JEMALLOC_DECAY 10 /* Defined if __attribute__((...)) syntax is supported. */ #define JEMALLOC_HAVE_ATTR @@ -30,14 +30,14 @@ extern "C" { #define JEMALLOC_HAVE_ATTR_COLD /* Defined if deprecated attribute is supported. */ -/* #undef JEMALLOC_HAVE_ATTR_DEPRECATED */ +#define JEMALLOC_HAVE_ATTR_DEPRECATED /* * Define overrides for non-standard allocator-related functions if they are * present on the system. */ /* #undef JEMALLOC_OVERRIDE_MEMALIGN */ -/* #undef JEMALLOC_OVERRIDE_VALLOC */ +#define JEMALLOC_OVERRIDE_VALLOC /* #undef JEMALLOC_OVERRIDE_PVALLOC */ /* @@ -64,7 +64,7 @@ extern "C" { # endif #endif -/* sizeof(void *) == 2^LG_SIZEOF_PTR. */ + /* sizeof(void *) == 2^LG_SIZEOF_PTR. */ #include #ifdef _MSC_VER # define LG_SIZEOF_PTR LG_SIZEOF_PTR_WIN @@ -84,6 +84,8 @@ extern "C" { # define je_calloc duckdb_je_calloc # define je_dallocx duckdb_je_dallocx # define je_free duckdb_je_free +# define je_free_sized duckdb_je_free_sized +# define je_free_aligned_sized duckdb_je_free_aligned_sized # define je_mallctl duckdb_je_mallctl # define je_mallctlbymib duckdb_je_mallctlbymib # define je_mallctlnametomib duckdb_je_mallctlnametomib @@ -94,7 +96,7 @@ extern "C" { # define je_malloc_stats_print duckdb_je_malloc_stats_print # define je_malloc_usable_size duckdb_je_malloc_usable_size # define je_mallocx duckdb_je_mallocx -# define je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 duckdb_je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 +# define je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 duckdb_je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 # define je_nallocx duckdb_je_nallocx # define je_posix_memalign duckdb_je_posix_memalign # define je_rallocx duckdb_je_rallocx @@ -112,13 +114,13 @@ extern "C" { #include #include -#define JEMALLOC_VERSION "5.3.0-172-gfa451de17fff73cc03c31ec8cd817d62927d1ff9" +#define JEMALLOC_VERSION "5.3.0-196-ga25b9b8ba91881964be3083db349991bbbbf1661" #define JEMALLOC_VERSION_MAJOR 5 #define JEMALLOC_VERSION_MINOR 3 #define JEMALLOC_VERSION_BUGFIX 0 -#define JEMALLOC_VERSION_NREV 172 -#define JEMALLOC_VERSION_GID "fa451de17fff73cc03c31ec8cd817d62927d1ff9" -#define JEMALLOC_VERSION_GID_IDENT fa451de17fff73cc03c31ec8cd817d62927d1ff9 +#define JEMALLOC_VERSION_NREV 196 +#define JEMALLOC_VERSION_GID "a25b9b8ba91881964be3083db349991bbbbf1661" +#define JEMALLOC_VERSION_GID_IDENT a25b9b8ba91881964be3083db349991bbbbf1661 #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -160,7 +162,7 @@ extern "C" { #define MALLCTL_ARENAS_DESTROYED 4097 #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) -# define JEMALLOC_CXX_THROW throw() +# define JEMALLOC_CXX_THROW noexcept (true) #else # define JEMALLOC_CXX_THROW #endif @@ -438,6 +440,8 @@ struct extent_hooks_s { # define calloc je_calloc # define dallocx je_dallocx # define free je_free +# define free_sized je_free_sized +# define free_aligned_sized je_free_aligned_sized # define mallctl je_mallctl # define mallctlbymib je_mallctlbymib # define mallctlnametomib je_mallctlnametomib @@ -448,7 +452,7 @@ struct extent_hooks_s { # define malloc_stats_print je_malloc_stats_print # define malloc_usable_size je_malloc_usable_size # define mallocx je_mallocx -# define smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 +# define smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 # define nallocx je_nallocx # define posix_memalign je_posix_memalign # define rallocx je_rallocx @@ -472,6 +476,8 @@ struct extent_hooks_s { # undef je_calloc # undef je_dallocx # undef je_free +# undef je_free_sized +# undef je_free_aligned_sized # undef je_mallctl # undef je_mallctlbymib # undef je_mallctlnametomib @@ -482,7 +488,7 @@ struct extent_hooks_s { # undef je_malloc_stats_print # undef je_malloc_usable_size # undef je_mallocx -# undef je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 +# undef je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 # undef je_nallocx # undef je_posix_memalign # undef je_rallocx diff --git a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_defs.h b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_defs.h index 1fb7f314514..5b9ee3641c3 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_defs.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_defs.h @@ -21,7 +21,7 @@ #define JEMALLOC_HAVE_ATTR_COLD /* Defined if deprecated attribute is supported. */ -/* #undef JEMALLOC_HAVE_ATTR_DEPRECATED */ +#define JEMALLOC_HAVE_ATTR_DEPRECATED /* * Define overrides for non-standard allocator-related functions if they are @@ -56,7 +56,7 @@ #endif /* sizeof(void *) == 2^LG_SIZEOF_PTR. */ -#include "limits.h" +#include #ifdef _MSC_VER # define LG_SIZEOF_PTR LG_SIZEOF_PTR_WIN #elif INTPTR_MAX == INT64_MAX diff --git a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_macros.h b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_macros.h index bfafee98303..44ab5a3fa04 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_macros.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_macros.h @@ -4,13 +4,13 @@ #include #include -#define JEMALLOC_VERSION "5.3.0-172-gfa451de17fff73cc03c31ec8cd817d62927d1ff9" +#define JEMALLOC_VERSION "5.3.0-196-ga25b9b8ba91881964be3083db349991bbbbf1661" #define JEMALLOC_VERSION_MAJOR 5 #define JEMALLOC_VERSION_MINOR 3 #define JEMALLOC_VERSION_BUGFIX 0 -#define JEMALLOC_VERSION_NREV 172 -#define JEMALLOC_VERSION_GID "fa451de17fff73cc03c31ec8cd817d62927d1ff9" -#define JEMALLOC_VERSION_GID_IDENT fa451de17fff73cc03c31ec8cd817d62927d1ff9 +#define JEMALLOC_VERSION_NREV 196 +#define JEMALLOC_VERSION_GID "a25b9b8ba91881964be3083db349991bbbbf1661" +#define JEMALLOC_VERSION_GID_IDENT a25b9b8ba91881964be3083db349991bbbbf1661 #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -52,7 +52,7 @@ #define MALLCTL_ARENAS_DESTROYED 4097 #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) -# define JEMALLOC_CXX_THROW throw() +# define JEMALLOC_CXX_THROW noexcept (true) #else # define JEMALLOC_CXX_THROW #endif diff --git a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_mangle.h b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_mangle.h index 66cbdd56fa3..cb4379aab1e 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_mangle.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_mangle.h @@ -13,6 +13,8 @@ # define calloc je_calloc # define dallocx je_dallocx # define free je_free +# define free_sized je_free_sized +# define free_aligned_sized je_free_aligned_sized # define mallctl je_mallctl # define mallctlbymib je_mallctlbymib # define mallctlnametomib je_mallctlnametomib @@ -23,7 +25,7 @@ # define malloc_stats_print je_malloc_stats_print # define malloc_usable_size je_malloc_usable_size # define mallocx je_mallocx -# define smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 +# define smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 # define nallocx je_nallocx # define posix_memalign je_posix_memalign # define rallocx je_rallocx @@ -47,6 +49,8 @@ # undef je_calloc # undef je_dallocx # undef je_free +# undef je_free_sized +# undef je_free_aligned_sized # undef je_mallctl # undef je_mallctlbymib # undef je_mallctlnametomib @@ -57,7 +61,7 @@ # undef je_malloc_stats_print # undef je_malloc_usable_size # undef je_mallocx -# undef je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 +# undef je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 # undef je_nallocx # undef je_posix_memalign # undef je_rallocx diff --git a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_mangle_jet.h b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_mangle_jet.h index ca1695c2922..02c97ee1a3e 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_mangle_jet.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_mangle_jet.h @@ -13,6 +13,8 @@ # define calloc jet_calloc # define dallocx jet_dallocx # define free jet_free +# define free_sized jet_free_sized +# define free_aligned_sized jet_free_aligned_sized # define mallctl jet_mallctl # define mallctlbymib jet_mallctlbymib # define mallctlnametomib jet_mallctlnametomib @@ -23,7 +25,7 @@ # define malloc_stats_print jet_malloc_stats_print # define malloc_usable_size jet_malloc_usable_size # define mallocx jet_mallocx -# define smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 jet_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 +# define smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 jet_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 # define nallocx jet_nallocx # define posix_memalign jet_posix_memalign # define rallocx jet_rallocx @@ -47,6 +49,8 @@ # undef jet_calloc # undef jet_dallocx # undef jet_free +# undef jet_free_sized +# undef jet_free_aligned_sized # undef jet_mallctl # undef jet_mallctlbymib # undef jet_mallctlnametomib @@ -57,7 +61,7 @@ # undef jet_malloc_stats_print # undef jet_malloc_usable_size # undef jet_mallocx -# undef jet_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 +# undef jet_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 # undef jet_nallocx # undef jet_posix_memalign # undef jet_rallocx diff --git a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_rename.h b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_rename.h index d7a1ae21feb..ac0d2dffa22 100644 --- a/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_rename.h +++ b/extension/jemalloc/jemalloc/include/jemalloc/jemalloc_rename.h @@ -8,6 +8,8 @@ # define je_calloc duckdb_je_calloc # define je_dallocx duckdb_je_dallocx # define je_free duckdb_je_free +# define je_free_sized duckdb_je_free_sized +# define je_free_aligned_sized duckdb_je_free_aligned_sized # define je_mallctl duckdb_je_mallctl # define je_mallctlbymib duckdb_je_mallctlbymib # define je_mallctlnametomib duckdb_je_mallctlnametomib @@ -18,7 +20,7 @@ # define je_malloc_stats_print duckdb_je_malloc_stats_print # define je_malloc_usable_size duckdb_je_malloc_usable_size # define je_mallocx duckdb_je_mallocx -# define je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 duckdb_je_smallocx_fa451de17fff73cc03c31ec8cd817d62927d1ff9 +# define je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 duckdb_je_smallocx_a25b9b8ba91881964be3083db349991bbbbf1661 # define je_nallocx duckdb_je_nallocx # define je_posix_memalign duckdb_je_posix_memalign # define je_rallocx duckdb_je_rallocx diff --git a/extension/jemalloc/jemalloc/src/arena.c b/extension/jemalloc/jemalloc/src/arena.c index 8c87d67f6cf..21010279d58 100644 --- a/extension/jemalloc/jemalloc/src/arena.c +++ b/extension/jemalloc/jemalloc/src/arena.c @@ -45,7 +45,6 @@ size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; uint32_t arena_bin_offsets[SC_NBINS]; -static unsigned nbins_total; static unsigned huge_arena_ind; @@ -662,10 +661,17 @@ arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, edata_t *slab) { } static void -arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) { +arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin, unsigned binind) { edata_t *slab; malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + + if (arena_bin_has_batch(binind)) { + bin_with_batch_t *batched_bin = (bin_with_batch_t *)bin; + batcher_init(&batched_bin->remote_frees, + BIN_REMOTE_FREE_ELEMS_MAX); + } + if (bin->slabcur != NULL) { slab = bin->slabcur; bin->slabcur = NULL; @@ -816,7 +822,8 @@ arena_reset(tsd_t *tsd, arena_t *arena) { /* Bins. */ for (unsigned i = 0; i < SC_NBINS; i++) { for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { - arena_bin_reset(tsd, arena, arena_get_bin(arena, i, j)); + arena_bin_reset(tsd, arena, arena_get_bin(arena, i, j), + i); } } pa_shard_reset(tsd_tsdn(tsd), &arena->pa_shard); @@ -1081,8 +1088,18 @@ arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, unsigned binshard; bin_t *bin = arena_bin_choose(tsdn, arena, binind, &binshard); + /* + * This has some fields that are conditionally initialized down batch + * flush pathways. This can trigger static analysis warnings deeper + * down in the static. The accesses are guarded by the same checks as + * the initialization, but the analysis isn't able to track that across + * multiple stack frames. + */ + arena_bin_flush_batch_state_t batch_flush_state + JEMALLOC_CLANG_ANALYZER_SILENCE_INIT({0}); label_refill: malloc_mutex_lock(tsdn, &bin->lock); + arena_bin_flush_batch_after_lock(tsdn, arena, bin, binind, &batch_flush_state); while (filled < nfill) { /* Try batch-fill from slabcur first. */ @@ -1137,7 +1154,11 @@ arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, cache_bin->tstats.nrequests = 0; } + arena_bin_flush_batch_before_unlock(tsdn, arena, bin, binind, + &batch_flush_state); malloc_mutex_unlock(tsdn, &bin->lock); + arena_bin_flush_batch_after_unlock(tsdn, arena, bin, binind, + &batch_flush_state); if (alloc_and_retry) { assert(fresh_slab == NULL); @@ -1428,12 +1449,16 @@ arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, edata_t *edata, void *ptr) { malloc_mutex_lock(tsdn, &bin->lock); arena_dalloc_bin_locked_info_t info; arena_dalloc_bin_locked_begin(&info, binind); - bool ret = arena_dalloc_bin_locked_step(tsdn, arena, bin, - &info, binind, edata, ptr); + edata_t *dalloc_slabs[1]; + unsigned dalloc_slabs_count = 0; + arena_dalloc_bin_locked_step(tsdn, arena, bin, &info, binind, edata, + ptr, dalloc_slabs, /* ndalloc_slabs */ 1, &dalloc_slabs_count, + /* dalloc_slabs_extra */ NULL); arena_dalloc_bin_locked_finish(tsdn, arena, bin, &info); malloc_mutex_unlock(tsdn, &bin->lock); - if (ret) { + if (dalloc_slabs_count != 0) { + assert(dalloc_slabs[0] == edata); arena_slab_dalloc(tsdn, arena, edata); } } @@ -1672,7 +1697,6 @@ arena_t * arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) { arena_t *arena; base_t *base; - unsigned i; if (ind == 0) { base = b0get(); @@ -1685,15 +1709,12 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) { } size_t arena_size = ALIGNMENT_CEILING(sizeof(arena_t), CACHELINE) + - sizeof(bin_t) * nbins_total; + sizeof(bin_with_batch_t) * bin_info_nbatched_bins + + sizeof(bin_t) * bin_info_nunbatched_bins; arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE); if (arena == NULL) { goto label_error; } - JEMALLOC_SUPPRESS_WARN_ON_USAGE( - assert((uintptr_t)&arena->all_bins[nbins_total -1] + sizeof(bin_t) <= - (uintptr_t)arena + arena_size); - ) atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED); atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED); @@ -1733,12 +1754,13 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) { /* Initialize bins. */ atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE); - for (i = 0; i < nbins_total; i++) { - JEMALLOC_SUPPRESS_WARN_ON_USAGE( - bool err = bin_init(&arena->all_bins[i]); - ) - if (err) { - goto label_error; + for (unsigned i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_t *bin = arena_get_bin(arena, i, j); + bool err = bin_init(bin, i); + if (err) { + goto label_error; + } } } @@ -1867,14 +1889,6 @@ arena_init_huge(arena_t *a0) { return huge_enabled; } -bool -arena_is_huge(unsigned arena_ind) { - if (huge_arena_ind == 0) { - return false; - } - return (arena_ind == huge_arena_ind); -} - bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) { arena_dirty_decay_ms_default_set(opt_dirty_decay_ms); @@ -1890,8 +1904,9 @@ arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) { ) for (szind_t i = 0; i < SC_NBINS; i++) { arena_bin_offsets[i] = cur_offset; - nbins_total += bin_infos[i].n_shards; - cur_offset += (uint32_t)(bin_infos[i].n_shards * sizeof(bin_t)); + uint32_t bin_sz = (i < bin_info_nbatched_sizes + ? sizeof(bin_with_batch_t) : sizeof(bin_t)); + cur_offset += (uint32_t)bin_infos[i].n_shards * bin_sz; } return pa_central_init(&arena_pa_central_global, base, hpa, &hpa_hooks_default); @@ -1941,19 +1956,21 @@ arena_prefork7(tsdn_t *tsdn, arena_t *arena) { void arena_prefork8(tsdn_t *tsdn, arena_t *arena) { - for (unsigned i = 0; i < nbins_total; i++) { - JEMALLOC_SUPPRESS_WARN_ON_USAGE( - bin_prefork(tsdn, &arena->all_bins[i]); - ) + for (szind_t i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_t *bin = arena_get_bin(arena, i, j); + bin_prefork(tsdn, bin, arena_bin_has_batch(i)); + } } } void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { - for (unsigned i = 0; i < nbins_total; i++) { - JEMALLOC_SUPPRESS_WARN_ON_USAGE( - bin_postfork_parent(tsdn, &arena->all_bins[i]); - ) + for (szind_t i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_t *bin = arena_get_bin(arena, i, j); + bin_postfork_parent(tsdn, bin, arena_bin_has_batch(i)); + } } malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); @@ -1990,10 +2007,11 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { } } - for (unsigned i = 0; i < nbins_total; i++) { - JEMALLOC_SUPPRESS_WARN_ON_USAGE( - bin_postfork_child(tsdn, &arena->all_bins[i]); - ) + for (szind_t i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_t *bin = arena_get_bin(arena, i, j); + bin_postfork_child(tsdn, bin, arena_bin_has_batch(i)); + } } malloc_mutex_postfork_child(tsdn, &arena->large_mtx); diff --git a/extension/jemalloc/jemalloc/src/batcher.c b/extension/jemalloc/jemalloc/src/batcher.c new file mode 100644 index 00000000000..2570b3a91b5 --- /dev/null +++ b/extension/jemalloc/jemalloc/src/batcher.c @@ -0,0 +1,96 @@ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/batcher.h" + +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/atomic.h" + +void +batcher_init(batcher_t *batcher, size_t nelems_max) { + atomic_store_zu(&batcher->nelems, 0, ATOMIC_RELAXED); + batcher->nelems_max = nelems_max; + batcher->npushes = 0; + malloc_mutex_init(&batcher->mtx, "batcher", WITNESS_RANK_BATCHER, + malloc_mutex_rank_exclusive); +} + +/* + * Returns an index (into some user-owned array) to use for pushing, or + * BATCHER_NO_IDX if no index is free. + */ +size_t batcher_push_begin(tsdn_t *tsdn, batcher_t *batcher, + size_t elems_to_push) { + assert(elems_to_push > 0); + size_t nelems_guess = atomic_load_zu(&batcher->nelems, ATOMIC_RELAXED); + if (nelems_guess + elems_to_push > batcher->nelems_max) { + return BATCHER_NO_IDX; + } + malloc_mutex_lock(tsdn, &batcher->mtx); + size_t nelems = atomic_load_zu(&batcher->nelems, ATOMIC_RELAXED); + if (nelems + elems_to_push > batcher->nelems_max) { + malloc_mutex_unlock(tsdn, &batcher->mtx); + return BATCHER_NO_IDX; + } + assert(elems_to_push <= batcher->nelems_max - nelems); + /* + * We update nelems at push time (instead of during pop) so that other + * racing accesses of the batcher can fail fast instead of trying to + * acquire a mutex only to discover that there's no space for them. + */ + atomic_store_zu(&batcher->nelems, nelems + elems_to_push, ATOMIC_RELAXED); + batcher->npushes++; + return nelems; +} + +size_t +batcher_pop_get_pushes(tsdn_t *tsdn, batcher_t *batcher) { + malloc_mutex_assert_owner(tsdn, &batcher->mtx); + size_t npushes = batcher->npushes; + batcher->npushes = 0; + return npushes; +} + +void +batcher_push_end(tsdn_t *tsdn, batcher_t *batcher) { + malloc_mutex_assert_owner(tsdn, &batcher->mtx); + assert(atomic_load_zu(&batcher->nelems, ATOMIC_RELAXED) > 0); + malloc_mutex_unlock(tsdn, &batcher->mtx); +} + +size_t +batcher_pop_begin(tsdn_t *tsdn, batcher_t *batcher) { + size_t nelems_guess = atomic_load_zu(&batcher->nelems, ATOMIC_RELAXED); + assert(nelems_guess <= batcher->nelems_max); + if (nelems_guess == 0) { + return BATCHER_NO_IDX; + } + malloc_mutex_lock(tsdn, &batcher->mtx); + size_t nelems = atomic_load_zu(&batcher->nelems, ATOMIC_RELAXED); + assert(nelems <= batcher->nelems_max); + if (nelems == 0) { + malloc_mutex_unlock(tsdn, &batcher->mtx); + return BATCHER_NO_IDX; + } + atomic_store_zu(&batcher->nelems, 0, ATOMIC_RELAXED); + return nelems; +} + +void batcher_pop_end(tsdn_t *tsdn, batcher_t *batcher) { + assert(atomic_load_zu(&batcher->nelems, ATOMIC_RELAXED) == 0); + malloc_mutex_unlock(tsdn, &batcher->mtx); +} + +void +batcher_prefork(tsdn_t *tsdn, batcher_t *batcher) { + malloc_mutex_prefork(tsdn, &batcher->mtx); +} + +void +batcher_postfork_parent(tsdn_t *tsdn, batcher_t *batcher) { + malloc_mutex_postfork_parent(tsdn, &batcher->mtx); +} + +void +batcher_postfork_child(tsdn_t *tsdn, batcher_t *batcher) { + malloc_mutex_postfork_child(tsdn, &batcher->mtx); +} diff --git a/extension/jemalloc/jemalloc/src/bin.c b/extension/jemalloc/jemalloc/src/bin.c index fa20458705a..267aa0f37db 100644 --- a/extension/jemalloc/jemalloc/src/bin.c +++ b/extension/jemalloc/jemalloc/src/bin.c @@ -6,6 +6,14 @@ #include "jemalloc/internal/sc.h" #include "jemalloc/internal/witness.h" +#ifdef JEMALLOC_JET +unsigned bin_batching_test_ndalloc_slabs_max = (unsigned)-1; +void (*bin_batching_test_after_push_hook)(size_t push_idx); +void (*bin_batching_test_mid_pop_hook)(size_t nelems_to_pop); +void (*bin_batching_test_after_unlock_hook)(unsigned slab_dalloc_count, + bool list_empty); +#endif + bool bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size, size_t end_size, size_t nshards) { @@ -39,7 +47,7 @@ bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) { } bool -bin_init(bin_t *bin) { +bin_init(bin_t *bin, unsigned binind) { if (malloc_mutex_init(&bin->lock, "bin", WITNESS_RANK_BIN, malloc_mutex_rank_exclusive)) { return true; @@ -50,20 +58,52 @@ bin_init(bin_t *bin) { if (config_stats) { memset(&bin->stats, 0, sizeof(bin_stats_t)); } + if (arena_bin_has_batch(binind)) { + bin_with_batch_t *batched_bin = (bin_with_batch_t *)bin; + batcher_init(&batched_bin->remote_frees, + opt_bin_info_remote_free_max); + } return false; } void -bin_prefork(tsdn_t *tsdn, bin_t *bin) { +bin_prefork(tsdn_t *tsdn, bin_t *bin, bool has_batch) { malloc_mutex_prefork(tsdn, &bin->lock); + if (has_batch) { + /* + * The batch mutex has lower rank than the bin mutex (as it must + * -- it's acquired later). But during forking, we go + * bin-at-a-time, so that we acquire mutex on bin 0, then on + * the bin 0 batcher, then on bin 1. This is a safe ordering + * (it's ordered by the index of arenas and bins within those + * arenas), but will trigger witness errors that would + * otherwise force another level of arena forking that breaks + * bin encapsulation (because the witness API doesn't "know" + * about arena or bin ordering -- it just sees that the batcher + * has a lower rank than the bin). So instead we exclude the + * batcher mutex from witness checking during fork (which is + * the only time we touch multiple bins at once) by passing + * TSDN_NULL. + */ + bin_with_batch_t *batched = (bin_with_batch_t *)bin; + batcher_prefork(TSDN_NULL, &batched->remote_frees); + } } void -bin_postfork_parent(tsdn_t *tsdn, bin_t *bin) { +bin_postfork_parent(tsdn_t *tsdn, bin_t *bin, bool has_batch) { malloc_mutex_postfork_parent(tsdn, &bin->lock); + if (has_batch) { + bin_with_batch_t *batched = (bin_with_batch_t *)bin; + batcher_postfork_parent(TSDN_NULL, &batched->remote_frees); + } } void -bin_postfork_child(tsdn_t *tsdn, bin_t *bin) { +bin_postfork_child(tsdn_t *tsdn, bin_t *bin, bool has_batch) { malloc_mutex_postfork_child(tsdn, &bin->lock); + if (has_batch) { + bin_with_batch_t *batched = (bin_with_batch_t *)bin; + batcher_postfork_child(TSDN_NULL, &batched->remote_frees); + } } diff --git a/extension/jemalloc/jemalloc/src/bin_info.c b/extension/jemalloc/jemalloc/src/bin_info.c index 8629ef8817d..f8a64ae3185 100644 --- a/extension/jemalloc/jemalloc/src/bin_info.c +++ b/extension/jemalloc/jemalloc/src/bin_info.c @@ -3,8 +3,26 @@ #include "jemalloc/internal/bin_info.h" +/* + * We leave bin-batching disabled by default, with other settings chosen mostly + * empirically; across the test programs I looked at they provided the most bang + * for the buck. With other default settings, these choices for bin batching + * result in them consuming far less memory (even in the worst case) than the + * tcaches themselves, the arena, etc. + * Note that we always try to pop all bins on every arena cache bin lock + * operation, so the typical memory waste is far less than this (and only on + * hot bins, which tend to be large anyways). + */ +size_t opt_bin_info_max_batched_size = 0; /* 192 is a good default. */ +size_t opt_bin_info_remote_free_max_batch = 4; +size_t opt_bin_info_remote_free_max = BIN_REMOTE_FREE_ELEMS_MAX; + bin_info_t bin_infos[SC_NBINS]; +szind_t bin_info_nbatched_sizes; +unsigned bin_info_nbatched_bins; +unsigned bin_info_nunbatched_bins; + static void bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], bin_info_t infos[SC_NBINS]) { @@ -20,6 +38,12 @@ bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER( bin_info->nregs); bin_info->bitmap_info = bitmap_info; + if (bin_info->reg_size <= opt_bin_info_max_batched_size) { + bin_info_nbatched_sizes++; + bin_info_nbatched_bins += bin_info->n_shards; + } else { + bin_info_nunbatched_bins += bin_info->n_shards; + } } } diff --git a/extension/jemalloc/jemalloc/src/emap.c b/extension/jemalloc/jemalloc/src/emap.c index 9cc95a724a9..f7d5c25a5d1 100644 --- a/extension/jemalloc/jemalloc/src/emap.c +++ b/extension/jemalloc/jemalloc/src/emap.c @@ -74,7 +74,7 @@ emap_try_acquire_edata_neighbor_impl(tsdn_t *tsdn, emap_t *emap, edata_t *edata, } rtree_contents_t neighbor_contents = rtree_leaf_elm_read(tsdn, - &emap->rtree, elm, /* dependent */ true); + &emap->rtree, elm, /* dependent */ false); if (!extent_can_acquire_neighbor(edata, neighbor_contents, pai, expected_state, forward, expanding)) { return NULL; diff --git a/extension/jemalloc/jemalloc/src/hpa.c b/extension/jemalloc/jemalloc/src/hpa.c index 99d1f03341b..49d6b037cdb 100644 --- a/extension/jemalloc/jemalloc/src/hpa.c +++ b/extension/jemalloc/jemalloc/src/hpa.c @@ -378,6 +378,18 @@ static bool hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) { malloc_mutex_assert_owner(tsdn, &shard->mtx); + /* + * Make sure we respect purge interval setting and don't purge + * too frequently. + */ + if (shard->opts.strict_min_purge_interval) { + uint64_t since_last_purge_ms = shard->central->hooks.ms_since( + &shard->last_purge); + if (since_last_purge_ms < shard->opts.min_purge_interval_ms) { + return false; + } + } + hpdata_t *to_purge = psset_pick_purge(&shard->psset); if (to_purge == NULL) { return false; @@ -537,9 +549,16 @@ hpa_shard_maybe_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard, purged = false; while (hpa_should_purge(tsdn, shard) && nops < max_ops) { purged = hpa_try_purge(tsdn, shard); - if (purged) { - nops++; + if (!purged) { + /* + * It is fine if we couldn't purge as sometimes + * we try to purge just to unblock + * hugification, but there is maybe no dirty + * pages at all at the moment. + */ + break; } + nops++; } hugified = hpa_try_hugify(tsdn, shard); if (hugified) { diff --git a/extension/jemalloc/jemalloc/src/jemalloc.c b/extension/jemalloc/jemalloc/src/jemalloc.c index 5f2e5b35b74..c69fe964adc 100644 --- a/extension/jemalloc/jemalloc/src/jemalloc.c +++ b/extension/jemalloc/jemalloc/src/jemalloc.c @@ -433,11 +433,8 @@ arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) { if (ind == 0) { return; } - /* - * Avoid creating a new background thread just for the huge arena, which - * purges eagerly by default. - */ - if (have_background_thread && !arena_is_huge(ind)) { + + if (have_background_thread) { if (background_thread_create(tsdn_tsd(tsdn), ind)) { malloc_printf(": error in background thread " "creation for arena %u. Abort.\n", ind); @@ -1331,6 +1328,21 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], } while (vlen_left > 0); CONF_CONTINUE; } + CONF_HANDLE_SIZE_T(opt_bin_info_max_batched_size, + "max_batched_size", 0, SIZE_T_MAX, + CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, + /* clip */ true) + CONF_HANDLE_SIZE_T(opt_bin_info_remote_free_max_batch, + "remote_free_max_batch", 0, + BIN_REMOTE_FREE_ELEMS_MAX, + CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, + /* clip */ true) + CONF_HANDLE_SIZE_T(opt_bin_info_remote_free_max, + "remote_free_max", 0, + BIN_REMOTE_FREE_ELEMS_MAX, + CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, + /* clip */ true) + if (CONF_MATCH("tcache_ncached_max")) { bool err = tcache_bin_info_default_init( v, vlen); @@ -1545,6 +1557,10 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], "hpa_min_purge_interval_ms", 0, 0, CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false); + CONF_HANDLE_BOOL( + opt_hpa_opts.strict_min_purge_interval, + "hpa_strict_min_purge_interval"); + if (CONF_MATCH("hpa_dirty_mult")) { if (CONF_MATCH_VALUE("-1")) { opt_hpa_opts.dirty_mult = (fxp_t)-1; @@ -4259,7 +4275,7 @@ jemalloc_constructor(void) { bgt_count = 1; } // decay is in ms - unsigned long long decay = DUCKDB_DECAY_DELAY * 1000; + unsigned long long decay = DUCKDB_JEMALLOC_DECAY * 1000; #ifdef DEBUG snprintf(JE_MALLOC_CONF_BUFFER, JE_MALLOC_CONF_BUFFER_SIZE, "junk:true,oversize_threshold:268435456,dirty_decay_ms:%llu,muzzy_decay_ms:%llu,narenas:%llu,max_background_threads:%llu", decay, decay, cpu_count, bgt_count); #else diff --git a/extension/jemalloc/jemalloc/src/malloc_io.c b/extension/jemalloc/jemalloc/src/malloc_io.c index 47d9fdb3126..192d82081cc 100644 --- a/extension/jemalloc/jemalloc/src/malloc_io.c +++ b/extension/jemalloc/jemalloc/src/malloc_io.c @@ -86,69 +86,6 @@ malloc_write(const char *s) { } } -// taken from https://ae1020.github.io/fixing-strerror_r-posix-debacle/ -int strerror_fixed(int err, char *buf, size_t buflen) { - assert(buflen != 0); - - buf[0] = (char)255; // never valid in UTF-8 sequences - int old_errno = errno; - intptr_t r = (intptr_t)strerror_r(err, buf, buflen); - int new_errno = errno; - - if (r == -1 || new_errno != old_errno) { - // - // errno was changed, so probably the return value is just -1 or - // something else that doesn't provide info. - // - malloc_snprintf(buf, buflen, "errno %d in strerror_r call", new_errno); - } - else if (r == 0) { - // - // The GNU version always succeds and should never return 0 (NULL). - // - // "The XSI-compliant strerror_r() function returns 0 on success. - // On error, a (positive) error number is returned (since glibc - // 2.13), or -1 is returned and errno is set to indicate the error - // (glibc versions before 2.13)." - // - // Documentation isn't clear on whether the buffer is terminated if - // the message is too long, or ERANGE always returned. Terminate. - // - buf[buflen - 1] = '\0'; - } - else if (r == EINVAL) { // documented result from XSI strerror_r - malloc_snprintf(buf, buflen, "bad errno %d for strerror_r()", err); - } - else if (r == ERANGE) { // documented result from XSI strerror_r - malloc_snprintf(buf, buflen, "bad buflen for errno %d", err); - } - else if (r == (intptr_t)buf) { - // - // The GNU version gives us our error back as a pointer if it - // filled the buffer successfully. Sanity check that. - // - if (buf[0] == (char)255) { - assert(false); - strncpy(buf, "strerror_r didn't update buffer", buflen); - } - } - else if (r < 256) { // extremely unlikely to be string buffer pointer - assert(false); - strncpy(buf, "Unknown XSI strerror_r error result code", buflen); - } - else { - // The GNU version never fails, but may return an immutable string - // instead of filling the buffer. Unknown errors get an - // "unknown error" message. The result is always null terminated. - // - // (This is the risky part, if `r` is not a valid pointer but some - // weird large int return result from XSI strerror_r.) - // - strncpy(buf, (const char*)r, buflen); - } - return 0; -} - /* * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so * provide a wrapper. @@ -159,19 +96,16 @@ buferror(int err, char *buf, size_t buflen) { FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, (LPSTR)buf, (DWORD)buflen, NULL); return 0; +#elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE) + char *b = strerror_r(err, buf, buflen); + if (b != buf) { + strncpy(buf, b, buflen); + buf[buflen-1] = '\0'; + } + return 0; #else - return strerror_fixed(err, buf, buflen); + return strerror_r(err, buf, buflen); #endif -// #elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE) -// char *b = strerror_r(err, buf, buflen); -// if (b != buf) { -// strncpy(buf, b, buflen); -// buf[buflen-1] = '\0'; -// } -// return 0; -// #else -// return strerror_r(err, buf, buflen); -// #endif } uintmax_t diff --git a/extension/jemalloc/jemalloc/src/pages.c b/extension/jemalloc/jemalloc/src/pages.c index 8cf2fd9f876..981aae9bbba 100644 --- a/extension/jemalloc/jemalloc/src/pages.c +++ b/extension/jemalloc/jemalloc/src/pages.c @@ -17,7 +17,7 @@ #include /* ilog2 */ #endif #ifdef JEMALLOC_HAVE_VM_MAKE_TAG -#define PAGES_FD_TAG VM_MAKE_TAG(101U) +#define PAGES_FD_TAG VM_MAKE_TAG(254U) #else #define PAGES_FD_TAG -1 #endif diff --git a/extension/jemalloc/jemalloc_config.py b/extension/jemalloc/jemalloc_config.py index 6dedff00ecc..494f4f56926 100644 --- a/extension/jemalloc/jemalloc_config.py +++ b/extension/jemalloc/jemalloc_config.py @@ -14,6 +14,7 @@ 'extension/jemalloc/jemalloc/src/arena.c', 'extension/jemalloc/jemalloc/src/background_thread.c', 'extension/jemalloc/jemalloc/src/base.c', + 'extension/jemalloc/jemalloc/src/batcher.c', 'extension/jemalloc/jemalloc/src/bin.c', 'extension/jemalloc/jemalloc/src/bin_info.c', 'extension/jemalloc/jemalloc/src/bitmap.c', diff --git a/extension/jemalloc/jemalloc_extension.cpp b/extension/jemalloc/jemalloc_extension.cpp index a51d7c371bb..10a49cf99a1 100644 --- a/extension/jemalloc/jemalloc_extension.cpp +++ b/extension/jemalloc/jemalloc_extension.cpp @@ -58,7 +58,7 @@ static inline string PurgeArenaString(idx_t arena_idx) { } int64_t JemallocExtension::DecayDelay() { - return DUCKDB_DECAY_DELAY; + return DUCKDB_JEMALLOC_DECAY; } void JemallocExtension::ThreadFlush(idx_t threshold) {