Bound explicit user prefetch to a fraction of the ARC

posix_fadvise(POSIX_FADV_WILLNEED) prefetch goes through dmu_prefetch(),
which issued every block of the request with no cap on outstanding I/O.
A large hint could pin memory without bound and OOM the system (#15776);
only the speculative prefetcher was capped.

Add dmu_prefetch_user() for explicit user requests and route the Linux
and FreeBSD fadvise WILLNEED paths through it.  It bounds the
outstanding prefetch to a budget from arc_prefetch_max_bytes() -- a
quarter of the adaptive ARC target when warm, and of the midpoint toward
arc_c_max while the cache is still cold, so a hint right after boot is
not starved.  dmu_prefetch() and the internal callers (bpobj, receive,
zvol) are left unthrottled.

Each request takes up to half the remaining budget, so concurrent hints
share it rather than the first taking everything, and that budget also
drives the block-range split -- a tight budget prefetches all of the
indirects and only some data, which suits a following random read.  The
whole issued range is accounted once against the budget and released per
block as the reads complete; the outstanding total is exported through
the dmu_prefetch kstat.

Add functional/fadvise/fadvise_willneed_limit.  It pins a small ARC,
issues a POSIX_FADV_WILLNEED over a file several times the resulting
budget on a slowed vdev, and asserts the outstanding explicit-prefetch
bytes (the dmu_prefetch kstat) never exceed arc_c_max / 4 and drain back
to zero.  Regression coverage for the #15776 runaway.

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: Alexander Motin <[email protected]>
Signed-off-by: Michael Heller <[email protected]>
Closes #15776
Closes #18798

include/sys/arc.h

@@ -337,6 +337,7 @@ boolean_t arc_async_flush_guid_inuse(uint64_t load_guid);
 uint64_t arc_all_memory(void);
 uint64_t arc_default_max(uint64_t min, uint64_t allmem);
 uint64_t arc_target_bytes(void);
+uint64_t arc_boot_target_bytes(void);
 void arc_set_limits(uint64_t);
 void arc_init(void);
 void arc_fini(void);

include/sys/dmu.h

@@ -958,6 +958,8 @@ extern uint_t zfs_max_recordsize;
  */
 void dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
 	uint64_t len, enum zio_priority pri);
+void dmu_prefetch_user(objset_t *os, uint64_t object, int64_t level,
+    uint64_t offset, uint64_t len, enum zio_priority pri);
 void dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
 	uint64_t len, enum zio_priority pri);
 void dmu_prefetch_dnode(objset_t *os, uint64_t object, enum zio_priority pri);

module/os/freebsd/zfs/zfs_vnops_os.c

@@ -6755,7 +6755,7 @@ zfs_freebsd_advise(struct vop_advise_args *ap)
 		 * is a larger sequential access pattern, perhaps dmu_zfetch
 		 * will detect it.
 		 */
-		dmu_prefetch(os, zp->z_id, 0, start, len,
+		dmu_prefetch_user(os, zp->z_id, 0, start, len,
 		    ZIO_PRIORITY_ASYNC_READ);
 		break;
 	case POSIX_FADV_DONTNEED:

module/os/linux/zfs/zpl_file.c

@@ -820,7 +820,7 @@ zpl_fadvise(struct file *filp, loff_t offset, loff_t len, int advice)
 
 	if (advice == POSIX_FADV_WILLNEED) {
 		loff_t rlen = len ? len : i_size_read(ip) - offset;
-		dmu_prefetch(os, zp->z_id, 0, offset, rlen,
+		dmu_prefetch_user(os, zp->z_id, 0, offset, rlen,
 		    ZIO_PRIORITY_ASYNC_READ);
 		if (!zn_has_cached_data(zp, offset, offset + rlen - 1)) {
 			zfs_exit(zfsvfs, FTAG);

module/zfs/arc.c

@@ -8019,6 +8019,19 @@ arc_target_bytes(void)
 	return (arc_c);
 }
 
+/*
+ * Byte budget for a single explicit (user) prefetch request, e.g.
+ * POSIX_FADV_WILLNEED.  Follows the adaptive ARC target (arc_c) once the cache
+ * is warm, but while cold -- when arc_c still sits near arc_c_min -- uses the
+ * midpoint toward arc_c_max so a hint issued right after boot is not starved.
+ * The caller applies the fraction that may be outstanding at once.
+ */
+uint64_t
+arc_boot_target_bytes(void)
+{
+	return (arc_warm ? arc_c : (arc_c + arc_c_max) / 2);
+}
+
 void
 arc_set_limits(uint64_t allmem)
 {

module/zfs/dmu.c

@@ -47,6 +47,7 @@
 #include <sys/dsl_synctask.h>
 #include <sys/dsl_prop.h>
 #include <sys/dmu_zfetch.h>
+#include <sys/kstat.h>
 #include <sys/zfs_ioctl.h>
 #include <sys/zap.h>
 #include <sys/zio_checksum.h>
@@ -698,6 +699,38 @@ dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, const void *tag)
  * in cache, they will be asynchronously read in.  Dnode read by dnode_hold()
  * is currently synchronous.
  */
+/*
+ * Outstanding bytes of explicit (user-requested) prefetch in flight, bounded
+ * by dmu_prefetch_user() so a large POSIX_FADV_WILLNEED hint cannot pin memory
+ * without limit and OOM the system (#15776).  Internal prefetch callers are
+ * not throttled.  Exported through the "dmustats" kstat.
+ */
+static uint64_t dmu_prefetch_bytes_active;
+
+static struct {
+	kstat_named_t prefetch_bytes_active;
+} dmu_stats = {
+	{ "prefetch_bytes_active",	KSTAT_DATA_UINT64 },
+};
+
+static kstat_t *dmu_ksp;
+
+static int
+dmu_kstats_update(kstat_t *ksp, int rw)
+{
+	(void) ksp;
+	if (rw == KSTAT_WRITE)
+		return (EACCES);
+	dmu_stats.prefetch_bytes_active.value.ui64 =
+	    atomic_load_64(&dmu_prefetch_bytes_active);
+	return (0);
+}
+
+static void dmu_prefetch_user_done(void *arg, uint64_t level, uint64_t blkid,
+    boolean_t issued);
+static void dmu_prefetch_by_dnode_impl(dnode_t *dn, int64_t level,
+    uint64_t offset, uint64_t len, zio_priority_t pri, uint64_t maxbytes);
+
 void
 dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
     uint64_t len, zio_priority_t pri)
@@ -717,9 +750,48 @@ dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
 	dnode_rele(dn, FTAG);
 }
 
+/*
+ * Like dmu_prefetch(), but for explicit user requests (e.g.
+ * POSIX_FADV_WILLNEED) that may span an arbitrarily large range.  Bound the
+ * outstanding prefetch so a large hint cannot pin memory without limit and OOM
+ * the system (#15776).  The budget is a quarter of arc_boot_target_bytes():
+ * the adaptive ARC target (arc_c) once the cache is warm, so it
+ * tightens under memory pressure; while the cache is still cold it uses the
+ * midpoint toward arc_c_max instead, so a hint issued right after boot -- when
+ * arc_c has not grown yet -- is not starved.
+ */
 void
-dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
+dmu_prefetch_user(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
     uint64_t len, zio_priority_t pri)
+{
+	dnode_t *dn;
+
+	if (dmu_prefetch_max == 0 || len == 0) {
+		dmu_prefetch_dnode(os, object, pri);
+		return;
+	}
+
+	if (dnode_hold(os, object, FTAG, &dn) != 0)
+		return;
+
+	uint64_t maxbytes = arc_boot_target_bytes() / 4;
+	dmu_prefetch_by_dnode_impl(dn, level, offset, len, pri, maxbytes);
+
+	dnode_rele(dn, FTAG);
+}
+
+static void
+dmu_prefetch_user_done(void *arg, uint64_t level, uint64_t blkid,
+    boolean_t issued)
+{
+	(void) level, (void) blkid, (void) issued;
+	atomic_add_64(&dmu_prefetch_bytes_active,
+	    -(int64_t)(uintptr_t)arg);
+}
+
+static void
+dmu_prefetch_by_dnode_impl(dnode_t *dn, int64_t level, uint64_t offset,
+    uint64_t len, zio_priority_t pri, uint64_t maxbytes)
 {
 	int64_t level2 = level;
 	uint64_t start, end, start2, end2;
@@ -729,6 +801,24 @@ dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
 	 * level, and following blocks [start2, end2) at higher level2.
 	 */
 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
+
+	/*
+	 * When bounding explicit prefetch (maxbytes != 0), take up to half the
+	 * remaining budget for this request, so concurrent hints each get a
+	 * slice instead of the first taking everything.  The resulting cap
+	 * drives the block-range split below in place of dmu_prefetch_max, so a
+	 * tight budget pushes more of the range up to the cheap indirect level
+	 * -- prefetching all the indirects and only some data, which is what a
+	 * following random-access pattern wants.  maxbytes == 0 is the
+	 * unthrottled internal path and keeps the plain dmu_prefetch_max split.
+	 */
+	uint64_t cap = dmu_prefetch_max;
+	if (maxbytes != 0) {
+		uint64_t active = atomic_load_64(&dmu_prefetch_bytes_active);
+		uint64_t headroom = maxbytes > active ? maxbytes - active : 0;
+		cap = MIN(dmu_prefetch_max, headroom >> 1);
+	}
+
 	if (dn->dn_datablkshift != 0) {
 
 		/*
@@ -751,14 +841,14 @@ dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
 		end2 = dbuf_whichblock(dn, level, offset + len - 1) + 1;
 		uint8_t ibs = dn->dn_indblkshift;
 		uint8_t bs = (level == 0) ? dn->dn_datablkshift : ibs;
-		uint_t limit = P2ROUNDUP(dmu_prefetch_max, 1 << bs) >> bs;
+		uint_t limit = P2ROUNDUP(cap, 1 << bs) >> bs;
 		start2 = end = MIN(end2, start + limit);
 
 		/*
 		 * Find level2 where [start2, end2) fits into dmu_prefetch_max.
 		 */
 		uint8_t ibps = ibs - SPA_BLKPTRSHIFT;
-		limit = P2ROUNDUP(dmu_prefetch_max, 1 << ibs) >> ibs;
+		limit = P2ROUNDUP(cap, 1 << ibs) >> ibs;
 		if (limit == 0)
 			end2 = start2;
 		do {
@@ -772,13 +862,45 @@ dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
 		end = start + (level == 0 && offset < dn->dn_datablksz);
 	}
 
-	for (uint64_t i = start; i < end; i++)
-		dbuf_prefetch(dn, level, i, pri, 0);
-	for (uint64_t i = start2; i < end2; i++)
-		dbuf_prefetch(dn, level2, i, pri, 0);
+	/*
+	 * Byte size of the blocks issued by each loop: data blocks (or
+	 * indirects, if a higher level was requested) in the first, indirect
+	 * blocks in the second.  Account the whole issued range against the
+	 * budget in one shot up front -- one atomic add, not one per block --
+	 * and let each block's completion callback release its own size, so the
+	 * counter drains back as the reads complete.
+	 */
+	uint64_t blksz1 = (level == 0) ? dn->dn_datablksz :
+	    (1ULL << dn->dn_indblkshift);
+	uint64_t blksz2 = 1ULL << dn->dn_indblkshift;
+	dbuf_prefetch_fn cb = NULL;
+	if (maxbytes != 0) {
+		uint64_t issued = (end - start) * blksz1 +
+		    (end2 - start2) * blksz2;
+		if (issued != 0) {
+			atomic_add_64(&dmu_prefetch_bytes_active, issued);
+			cb = dmu_prefetch_user_done;
+		}
+	}
+
+	for (uint64_t i = start; i < end; i++) {
+		(void) dbuf_prefetch_impl(dn, level, i, pri, 0, cb,
+		    (void *)(uintptr_t)blksz1);
+	}
+	for (uint64_t i = start2; i < end2; i++) {
+		(void) dbuf_prefetch_impl(dn, level2, i, pri, 0, cb,
+		    (void *)(uintptr_t)blksz2);
+	}
 	rw_exit(&dn->dn_struct_rwlock);
 }
 
+void
+dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset,
+    uint64_t len, zio_priority_t pri)
+{
+	dmu_prefetch_by_dnode_impl(dn, level, offset, len, pri, 0);
+}
+
 /*
  * Prime a prefetch for sequential accesses from offset for at least len bytes.
  */
@@ -3002,6 +3124,16 @@ dmu_init(void)
 	l2arc_init();
 	arc_init();
 	dbuf_init();
+
+	dmu_ksp = kstat_create("zfs", 0, "dmustats", "misc",
+	    KSTAT_TYPE_NAMED,
+	    sizeof (dmu_stats) / sizeof (kstat_named_t),
+	    KSTAT_FLAG_VIRTUAL);
+	if (dmu_ksp != NULL) {
+		dmu_ksp->ks_data = &dmu_stats;
+		dmu_ksp->ks_update = dmu_kstats_update;
+		kstat_install(dmu_ksp);
+	}
 }
 
 void
@@ -3016,6 +3148,13 @@ dmu_fini(void)
 	dmu_objset_fini();
 	sa_cache_fini();
 	zfs_dbgmsg_fini();
+
+	if (dmu_ksp != NULL) {
+		kstat_delete(dmu_ksp);
+		dmu_ksp = NULL;
+	}
+	ASSERT0(atomic_load_64(&dmu_prefetch_bytes_active));
+
 	abd_fini();
 }
 

tests/runfiles/common.run

@@ -748,7 +748,7 @@ tests = ['exec_001_pos', 'exec_002_neg']
 tags = ['functional', 'exec']
 
 [tests/functional/fadvise]
-tests = ['fadvise_dontneed', 'fadvise_willneed']
+tests = ['fadvise_dontneed', 'fadvise_willneed', 'fadvise_willneed_limit']
 tags = ['functional', 'fadvise']
 
 [tests/functional/failmode]

tests/zfs-tests/tests/Makefile.am

@@ -1645,6 +1645,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/fadvise/cleanup.ksh \
 	functional/fadvise/fadvise_dontneed.ksh \
 	functional/fadvise/fadvise_willneed.ksh \
+	functional/fadvise/fadvise_willneed_limit.ksh \
 	functional/fadvise/setup.ksh \
 	functional/failmode/cleanup.ksh \
 	functional/failmode/failmode_dmu_tx_wait.ksh \

tests/zfs-tests/tests/functional/fadvise/fadvise_willneed_limit.ksh

@@ -0,0 +1,119 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or https://opensource.org/licenses/CDDL-1.0.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2026 by Michael Heller. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/include/math.shlib
+
+#
+# DESCRIPTION:
+# A POSIX_FADV_WILLNEED hint over a file much larger than the prefetch budget
+# must not pin more than the budget's worth of outstanding prefetch.  This is
+# the #15776 runaway: explicit user prefetch had no in-flight cap and could
+# grow zio/ABD memory without bound until the system OOMed.  The cap is
+# MAX(arc_c / 4, arc_c_max >> 4); with a small pinned ARC that is arc_c / 4.
+#
+# STRATEGY:
+# 1. Pin the ARC small (arc_min < arc_max) so arc_c_max is small and known; the
+#    budget can never exceed arc_c_max/4 (arc_c <= arc_c_max).
+# 2. Create a file several times larger than that budget.
+# 3. Inject a slow vdev so issued prefetch stays outstanding long enough to
+#    observe, and export/import so the hint actually issues reads.
+# 4. Storm POSIX_FADV_WILLNEED over the file while sampling the outstanding
+#    explicit-prefetch bytes (the dmustats kstat).
+# 5. Assert the peak is > 0 (the path was exercised) and within the budget.
+# 6. Assert it drains back to 0 (no accounting leak).
+#
+
+verify_runnable "global"
+
+FILE=$TESTDIR/$TESTFILE0
+DISK=${DISKS%% *}
+# arc_min must be strictly below arc_max (and arc_max >= MIN_ARC_MAX == 64M),
+# otherwise the arc_max tuning is ignored.
+ARC_MIN_BYTES=$((64 * 1024 * 1024))
+ARC_MAX_BYTES=$((256 * 1024 * 1024))
+
+function cleanup
+{
+	zinject -c all > /dev/null 2>&1
+	restore_tunable ARC_MAX
+	restore_tunable ARC_MIN
+	[[ -e $FILE ]] && log_must rm -f $FILE
+}
+
+log_assert "Explicit WILLNEED prefetch is bounded to a fraction of the ARC (#15776)"
+log_onexit cleanup
+
+save_tunable ARC_MAX
+save_tunable ARC_MIN
+log_must set_tunable64 ARC_MIN $ARC_MIN_BYTES
+log_must set_tunable64 ARC_MAX $ARC_MAX_BYTES
+
+# The kernel budget is MAX(arc_c / 4, arc_c_max >> 4).  Since arc_c <= arc_c_max
+# at all times, the budget can never exceed arc_c_max / 4 -- a stable ceiling we
+# can assert against without racing arc_c growth during the storm.
+typeset -i arc_cmax=$(kstat arcstats.c_max)
+typeset -i budget=$((arc_cmax / 4))
+log_note "arc_c_max=$arc_cmax -> prefetch budget ceiling=$budget bytes"
+
+# A file several times the budget, so a single hint far exceeds it.
+typeset -i fsize=$((budget * 4))
+log_must file_write -o create -f $FILE -b 1048576 -c $((fsize / 1048576))
+sync_pool $TESTPOOL
+
+# Cold the cache so WILLNEED issues reads, then slow the vdev so the issued
+# prefetch stays outstanding while we sample it.
+log_must zpool export $TESTPOOL
+log_must zpool import -d /dev $TESTPOOL
+log_must zinject -d $DISK -D 20:1 $TESTPOOL
+
+# Storm WILLNEED and track the peak outstanding explicit-prefetch bytes.
+typeset -i peak=0 cur
+typeset -i deadline=$((SECONDS + 5))
+while (( SECONDS < deadline )); do
+	file_fadvise -f $FILE -a POSIX_FADV_WILLNEED
+	cur=$(kstat dmustats.prefetch_bytes_active)
+	(( cur > peak )) && peak=$cur
+done
+log_note "peak outstanding explicit prefetch = $peak bytes (budget $budget)"
+
+# The path must have been exercised, and must have stayed within the budget
+# (plus a small margin for the one-block-per-caller reserve overshoot).
+log_must test $peak -gt 0
+typeset -i ceil=$((budget + budget / 10 + 1048576))
+log_must test $peak -le $ceil
+
+# Clear the fault and confirm the accounting drains back to zero (no leak).
+log_must zinject -c all
+typeset -i i=0
+while (( $(kstat dmustats.prefetch_bytes_active) != 0 && i < 60 )); do
+	sleep 1
+	(( i += 1 ))
+done
+log_must test $(kstat dmustats.prefetch_bytes_active) -eq 0
+
+log_pass "WILLNEED prefetch stayed within the ARC budget and drained to zero"