diff --git a/include/os/freebsd/spl/sys/kmem.h b/include/os/freebsd/spl/sys/kmem.h index c633799318d5..ae786f0e20ca 100644 --- a/include/os/freebsd/spl/sys/kmem.h +++ b/include/os/freebsd/spl/sys/kmem.h @@ -49,6 +49,7 @@ MALLOC_DECLARE(M_SOLARIS); #define KM_NOSLEEP M_NOWAIT #define KM_NORMALPRI 0 #define KMC_NODEBUG UMA_ZONE_NODUMP +#define KMC_RECLAIMABLE 0x0 typedef struct vmem vmem_t; diff --git a/include/os/linux/spl/sys/kmem_cache.h b/include/os/linux/spl/sys/kmem_cache.h index b159bb52d111..1240f28d6482 100644 --- a/include/os/linux/spl/sys/kmem_cache.h +++ b/include/os/linux/spl/sys/kmem_cache.h @@ -45,6 +45,7 @@ typedef enum kmc_bit { KMC_BIT_TOTAL = 18, /* Proc handler helper bit */ KMC_BIT_ALLOC = 19, /* Proc handler helper bit */ KMC_BIT_MAX = 20, /* Proc handler helper bit */ + KMC_BIT_RECLAIMABLE = 21, /* Can be freed by shrinker */ } kmc_bit_t; /* kmem move callback return values */ @@ -66,9 +67,7 @@ typedef enum kmem_cbrc { #define KMC_TOTAL (1 << KMC_BIT_TOTAL) #define KMC_ALLOC (1 << KMC_BIT_ALLOC) #define KMC_MAX (1 << KMC_BIT_MAX) - -#define KMC_REAP_CHUNK INT_MAX -#define KMC_DEFAULT_SEEKS 1 +#define KMC_RECLAIMABLE (1 << KMC_BIT_RECLAIMABLE) extern struct list_head spl_kmem_cache_list; extern struct rw_semaphore spl_kmem_cache_sem; diff --git a/include/sys/zfs_context.h b/include/sys/zfs_context.h index 750ca612b962..0b35b73e4df2 100644 --- a/include/sys/zfs_context.h +++ b/include/sys/zfs_context.h @@ -414,6 +414,7 @@ void procfs_list_add(procfs_list_t *procfs_list, void *p); #define KM_NORMALPRI 0 /* not needed with UMEM_DEFAULT */ #define KMC_NODEBUG UMC_NODEBUG #define KMC_KVMEM 0x0 +#define KMC_RECLAIMABLE 0x0 #define kmem_alloc(_s, _f) umem_alloc(_s, _f) #define kmem_zalloc(_s, _f) umem_zalloc(_s, _f) #define kmem_free(_b, _s) umem_free(_b, _s) diff --git a/module/os/freebsd/zfs/abd_os.c b/module/os/freebsd/zfs/abd_os.c index 3b812271f98b..fb5c46ecf7c2 100644 --- a/module/os/freebsd/zfs/abd_os.c +++ b/module/os/freebsd/zfs/abd_os.c @@ -300,7 +300,7 @@ void abd_init(void) { abd_chunk_cache = kmem_cache_create("abd_chunk", PAGE_SIZE, 0, - NULL, NULL, NULL, NULL, 0, KMC_NODEBUG); + NULL, NULL, NULL, NULL, 0, KMC_NODEBUG | KMC_RECLAIMABLE); wmsum_init(&abd_sums.abdstat_struct_size, 0); wmsum_init(&abd_sums.abdstat_scatter_cnt, 0); diff --git a/module/os/freebsd/zfs/zfs_znode.c b/module/os/freebsd/zfs/zfs_znode.c index 0eea2a849416..afbea9798c33 100644 --- a/module/os/freebsd/zfs/zfs_znode.c +++ b/module/os/freebsd/zfs/zfs_znode.c @@ -236,7 +236,7 @@ zfs_znode_init(void) ASSERT3P(znode_cache, ==, NULL); znode_cache = kmem_cache_create("zfs_znode_cache", sizeof (znode_t), 0, zfs_znode_cache_constructor, - zfs_znode_cache_destructor, NULL, NULL, NULL, 0); + zfs_znode_cache_destructor, NULL, NULL, NULL, KMC_RECLAIMABLE); } static znode_t * diff --git a/module/os/linux/spl/spl-kmem-cache.c b/module/os/linux/spl/spl-kmem-cache.c index 42821ad60256..e323d8c59ed9 100644 --- a/module/os/linux/spl/spl-kmem-cache.c +++ b/module/os/linux/spl/spl-kmem-cache.c @@ -152,6 +152,8 @@ kv_alloc(spl_kmem_cache_t *skc, int size, int flags) gfp_t lflags = kmem_flags_convert(flags); void *ptr; + if (skc->skc_flags & KMC_RECLAIMABLE) + lflags |= __GFP_RECLAIMABLE; ptr = spl_vmalloc(size, lflags | __GFP_HIGHMEM); /* Resulting allocated memory will be page aligned */ @@ -432,6 +434,8 @@ spl_emergency_alloc(spl_kmem_cache_t *skc, int flags, void **obj) if (!empty) return (-EEXIST); + if (skc->skc_flags & KMC_RECLAIMABLE) + lflags |= __GFP_RECLAIMABLE; ske = kmalloc(sizeof (*ske), lflags); if (ske == NULL) return (-ENOMEM); @@ -671,6 +675,7 @@ spl_magazine_destroy(spl_kmem_cache_t *skc) * KMC_KVMEM Force kvmem backed SPL cache * KMC_SLAB Force Linux slab backed cache * KMC_NODEBUG Disable debugging (unsupported) + * KMC_RECLAIMABLE Memory can be freed under pressure */ spl_kmem_cache_t * spl_kmem_cache_create(const char *name, size_t size, size_t align, @@ -788,6 +793,9 @@ spl_kmem_cache_create(const char *name, size_t size, size_t align, if (size > spl_kmem_cache_slab_limit) goto out; + if (skc->skc_flags & KMC_RECLAIMABLE) + slabflags |= SLAB_RECLAIM_ACCOUNT; + #if defined(SLAB_USERCOPY) /* * Required for PAX-enabled kernels if the slab is to be diff --git a/module/os/linux/spl/spl-zlib.c b/module/os/linux/spl/spl-zlib.c index 8c6282ee5d16..a7b6c14ee150 100644 --- a/module/os/linux/spl/spl-zlib.c +++ b/module/os/linux/spl/spl-zlib.c @@ -202,7 +202,7 @@ spl_zlib_init(void) zlib_workspace_cache = kmem_cache_create( "spl_zlib_workspace_cache", size, 0, NULL, NULL, NULL, NULL, NULL, - KMC_KVMEM); + KMC_KVMEM | KMC_RECLAIMABLE); if (!zlib_workspace_cache) return (-ENOMEM); diff --git a/module/os/linux/zfs/abd_os.c b/module/os/linux/zfs/abd_os.c index d3255dcbc0f7..79c89e7ab216 100644 --- a/module/os/linux/zfs/abd_os.c +++ b/module/os/linux/zfs/abd_os.c @@ -281,7 +281,7 @@ abd_alloc_chunks(abd_t *abd, size_t size) struct sg_table table; struct scatterlist *sg; struct page *page, *tmp_page = NULL; - gfp_t gfp = __GFP_NOWARN | GFP_NOIO; + gfp_t gfp = __GFP_RECLAIMABLE | __GFP_NOWARN | GFP_NOIO; gfp_t gfp_comp = (gfp | __GFP_NORETRY | __GFP_COMP) & ~__GFP_RECLAIM; unsigned int max_order = MIN(zfs_abd_scatter_max_order, ABD_MAX_ORDER - 1); @@ -403,7 +403,7 @@ abd_alloc_chunks(abd_t *abd, size_t size) struct scatterlist *sg = NULL; struct sg_table table; struct page *page; - gfp_t gfp = __GFP_NOWARN | GFP_NOIO; + gfp_t gfp = __GFP_RECLAIMABLE | __GFP_NOWARN | GFP_NOIO; int nr_pages = abd_chunkcnt_for_bytes(size); int i = 0; @@ -762,7 +762,7 @@ abd_init(void) int i; abd_cache = kmem_cache_create("abd_t", sizeof (abd_t), - 0, NULL, NULL, NULL, NULL, NULL, 0); + 0, NULL, NULL, NULL, NULL, NULL, KMC_RECLAIMABLE); wmsum_init(&abd_sums.abdstat_struct_size, 0); wmsum_init(&abd_sums.abdstat_linear_cnt, 0); diff --git a/module/os/linux/zfs/zfs_znode.c b/module/os/linux/zfs/zfs_znode.c index b99df188c64b..265153e011e7 100644 --- a/module/os/linux/zfs/zfs_znode.c +++ b/module/os/linux/zfs/zfs_znode.c @@ -194,7 +194,8 @@ zfs_znode_init(void) ASSERT(znode_cache == NULL); znode_cache = kmem_cache_create("zfs_znode_cache", sizeof (znode_t), 0, zfs_znode_cache_constructor, - zfs_znode_cache_destructor, NULL, NULL, NULL, KMC_SLAB); + zfs_znode_cache_destructor, NULL, NULL, NULL, + KMC_SLAB | KMC_RECLAIMABLE); ASSERT(znode_hold_cache == NULL); znode_hold_cache = kmem_cache_create("zfs_znode_hold_cache", diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 8c403f2a4b86..b53cee4b5831 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -1258,7 +1258,7 @@ buf_init(void) } hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE, - 0, hdr_full_cons, hdr_full_dest, NULL, NULL, NULL, 0); + 0, hdr_full_cons, hdr_full_dest, NULL, NULL, NULL, KMC_RECLAIMABLE); hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only", HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, NULL, NULL, NULL, 0); diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c index 21632f5714cb..a40e918fc999 100644 --- a/module/zfs/dnode.c +++ b/module/zfs/dnode.c @@ -298,7 +298,7 @@ dnode_init(void) { ASSERT(dnode_cache == NULL); dnode_cache = kmem_cache_create("dnode_t", sizeof (dnode_t), - 0, dnode_cons, dnode_dest, NULL, NULL, NULL, 0); + 0, dnode_cons, dnode_dest, NULL, NULL, NULL, KMC_RECLAIMABLE); kmem_cache_set_move(dnode_cache, dnode_move); wmsum_init(&dnode_sums.dnode_hold_dbuf_hold, 0); diff --git a/module/zfs/lz4_zfs.c b/module/zfs/lz4_zfs.c index e28215cf3501..a3b9e7070373 100644 --- a/module/zfs/lz4_zfs.c +++ b/module/zfs/lz4_zfs.c @@ -867,7 +867,8 @@ void lz4_init(void) { lz4_cache = kmem_cache_create("lz4_cache", - sizeof (struct refTables), 0, NULL, NULL, NULL, NULL, NULL, 0); + sizeof (struct refTables), 0, NULL, NULL, NULL, NULL, NULL, + KMC_RECLAIMABLE); } void diff --git a/module/zfs/sa.c b/module/zfs/sa.c index bb4cf246ea07..3f0ba6c4cf98 100644 --- a/module/zfs/sa.c +++ b/module/zfs/sa.c @@ -235,7 +235,7 @@ sa_cache_init(void) { sa_cache = kmem_cache_create("sa_cache", sizeof (sa_handle_t), 0, sa_cache_constructor, - sa_cache_destructor, NULL, NULL, NULL, 0); + sa_cache_destructor, NULL, NULL, NULL, KMC_RECLAIMABLE); } void diff --git a/module/zfs/zio.c b/module/zfs/zio.c index bf4aaf715f19..7d90aea3b765 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -194,6 +194,10 @@ zio_init(void) cflags = (zio_exclude_metadata || size > zio_buf_debug_limit) ? KMC_NODEBUG : 0; data_cflags = KMC_NODEBUG; + if (abd_size_alloc_linear(size)) { + cflags |= KMC_RECLAIMABLE; + data_cflags |= KMC_RECLAIMABLE; + } if (cflags == data_cflags) { /* * Resulting kmem caches would be identical.