Predict and throttle buffers dirtied by the sync context

Block cloning may cause lots of dirty buffers in the sync context
for BRT/DDT updates, while in open context dirtying only indirect
blocks, and so remaining almost invisible for write throttling.
With active cloning I was able to make dirty reach 16GB and more
while having the limit of only 4GB.

In case of cloned/deduped blocks delete the problem is not so bad,
thanks to zfs_max_async_dedup_frees limit.  But on smaller systems
or with higher tunable setting it might still be a problem.

To handle the cloning make dmu_tx_count_clone() predict amount
of the dirty buffers that might be produced by the sync context
for this transaction, summarise those for each transaction group,
and account them in write throttling as real dirty buffers.
When sync context as part of DDT/BRT writing start dirtying MOS
buffers, subtract that dirty from the predicted reserve.  This way
completions of the DDT/BRT write ZIOs will gradually open the
throttle for the next open TXG.  Any remaining balance caused by
mispredictions is cleared at the end of each TXG.

To handle the deletes, increase the same reserve value when issuing
free ZIOs for deduped or cloned blocks, so that new writes/deletes
would be throttled.  Same time, throttle the async deletes once the
amount of dirty data for the TXG (real + predicted) gets too high.

Tests with this patch show that the amount of real dirty data stays
within the limit for both clones and deletes for both cloning and
dedup even when BRT/DDT reach substantial sizes.

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Alexander Motin <[email protected]>
Closes #18859

include/sys/brt.h

@@ -45,6 +45,7 @@ extern uint64_t brt_get_saved(spa_t *spa);
 extern uint64_t brt_get_ratio(spa_t *spa);
 
 extern boolean_t brt_maybe_exists(spa_t *spa, const blkptr_t *bp);
+extern uint64_t brt_sync_dirty_est(spa_t *spa);
 extern void brt_init(void);
 extern void brt_fini(void);
 

include/sys/ddt.h

@@ -403,6 +403,7 @@ extern uint64_t ddt_get_dedup_used(spa_t *spa);
 extern uint64_t ddt_get_dedup_saved(spa_t *spa);
 extern uint64_t ddt_get_pool_dedup_ratio(spa_t *spa);
 extern int ddt_get_pool_dedup_cached(spa_t *spa, uint64_t *psize);
+extern uint64_t ddt_sync_dirty_est(spa_t *spa);
 
 extern ddt_t *ddt_select(spa_t *spa, const blkptr_t *bp);
 extern void ddt_enter(ddt_t *ddt);

include/sys/ddt_impl.h

@@ -179,6 +179,8 @@ typedef struct {
 
 extern const ddt_ops_t ddt_zap_ops;
 
+extern unsigned int ddt_zap_default_bs;
+
 /* Dedup log API */
 extern void ddt_log_begin(ddt_t *ddt, size_t nentries, dmu_tx_t *tx,
     ddt_log_update_t *dlu);

include/sys/dmu_tx.h

@@ -60,6 +60,10 @@ struct dmu_tx {
 	uint64_t tx_lastsnap_txg;
 	uint64_t tx_lasttried_txg;
 	txg_handle_t tx_txgh;
+
+	/* sync context dirty space reservation for DDT/BRT updates */
+	uint64_t tx_sync_reserve;
+
 	void *tx_tempreserve_cookie;
 	struct dmu_tx_hold *tx_needassign_txh;
 

include/sys/dsl_pool.h

@@ -118,6 +118,8 @@ typedef struct dsl_pool {
 	kcondvar_t dp_spaceavail_cv;
 	uint64_t dp_dirty_pertxg[TXG_SIZE];
 	uint64_t dp_dirty_total;
+	uint64_t dp_sync_reserve_pertxg[TXG_SIZE];
+	uint64_t dp_sync_reserve_total;
 	uint64_t dp_long_free_dirty_pertxg[TXG_SIZE];
 
 	aggsum_t dp_wrlog_pertxg[TXG_SIZE];
@@ -170,7 +172,10 @@ uint64_t dsl_pool_deferred_space(dsl_pool_t *dp);
 void dsl_pool_wrlog_count(dsl_pool_t *dp, int64_t size, uint64_t txg);
 boolean_t dsl_pool_need_wrlog_delay(dsl_pool_t *dp);
 void dsl_pool_dirty_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx);
+void dsl_pool_dirty_mos_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx);
 void dsl_pool_undirty_space(dsl_pool_t *dp, int64_t space, uint64_t txg);
+void dsl_pool_sync_reserve(dsl_pool_t *dp, uint64_t space, dmu_tx_t *tx);
+void dsl_pool_sync_unreserve(dsl_pool_t *dp, uint64_t space, uint64_t txg);
 void dsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp);
 void dsl_free_sync(zio_t *pio, dsl_pool_t *dp, uint64_t txg,
     const blkptr_t *bpp);

module/zfs/brt.c

@@ -930,6 +930,19 @@ brt_maybe_exists(spa_t *spa, const blkptr_t *bp)
 	return (brt_vdev_lookup(spa, brtvd, off));
 }
 
+/*
+ * Estimate the worst-case amount of MOS data the sync thread may dirty
+ * to add, update or remove one BRT entry: one ZAP leaf block.  Unlike
+ * the DDT, BRT ZAPs do not use prehashed keys, so even consecutive
+ * offsets scatter across leaves and rarely combine.
+ */
+uint64_t
+brt_sync_dirty_est(spa_t *spa)
+{
+	(void) spa;
+	return (1ULL << brt_zap_default_bs);
+}
+
 uint64_t
 brt_get_dspace(spa_t *spa)
 {

module/zfs/ddt.c

@@ -1474,6 +1474,20 @@ ddt_key_compare(const void *x1, const void *x2)
 	return (0);
 }
 
+/*
+ * Estimate the worst-case amount of MOS data the sync thread may dirty
+ * to add, update or remove one DDT entry: one ZAP leaf block.  This is
+ * an underestimation for entries changing class (two leaves in two
+ * different ZAPs plus indirects), but sequential log flush usually
+ * combines many entries per leaf, erring the other way.
+ */
+uint64_t
+ddt_sync_dirty_est(spa_t *spa)
+{
+	(void) spa;
+	return (1ULL << ddt_zap_default_bs);
+}
+
 /* Create the containing dir for this DDT and bump the feature count */
 static void
 ddt_create_dir(ddt_t *ddt, dmu_tx_t *tx)

module/zfs/ddt_zap.c

@@ -36,7 +36,7 @@
 #include <sys/dnode.h>
 #include <sys/zio_compress.h>
 
-static unsigned int ddt_zap_default_bs = 15;
+unsigned int ddt_zap_default_bs = 15;
 static unsigned int ddt_zap_default_ibs = 15;
 
 #define	DDT_ZAP_COMPRESS_BYTEORDER_MASK	0x80

module/zfs/dmu_objset.c

@@ -3041,9 +3041,10 @@ dmu_objset_willuse_space(objset_t *os, int64_t space, dmu_tx_t *tx)
 
 	if (ds != NULL) {
 		dsl_dir_willuse_space(ds->ds_dir, aspace, tx);
+		dsl_pool_dirty_space(dmu_tx_pool(tx), space, tx);
+	} else {
+		dsl_pool_dirty_mos_space(dmu_tx_pool(tx), space, tx);
 	}
-
-	dsl_pool_dirty_space(dmu_tx_pool(tx), space, tx);
 }
 
 /*

module/zfs/dmu_tx.c

@@ -36,6 +36,7 @@
 #include <sys/dsl_pool.h>
 #include <sys/zap_impl.h>
 #include <sys/spa.h>
+#include <sys/brt.h>
 #include <sys/brt_impl.h>
 #include <sys/sa.h>
 #include <sys/sa_impl.h>
@@ -561,6 +562,12 @@ dmu_tx_count_clone(dmu_tx_hold_t *txh, uint64_t off, uint64_t len,
 	ASSERT(blksz != 0);
 	ASSERT0(off % blksz);
 
+	/*
+	 * The last block of the range is cloned in full even if the range
+	 * covers it only partially, in case it is the last block of the file.
+	 */
+	len = P2ROUNDUP(len, (uint64_t)blksz);
+
 	(void) zfs_refcount_add_many(&txh->txh_memory_tohold,
 	    len / blksz * sizeof (brt_entry_t), FTAG);
 
@@ -577,12 +584,21 @@ dmu_tx_count_clone(dmu_tx_hold_t *txh, uint64_t off, uint64_t len,
 		err = dmu_tx_check_ioerr(zio, dn, 1, i);
 		if (err != 0) {
 			tx->tx_err = err;
-			break;
+			(void) zio_wait(zio);
+			return;
 		}
 	}
 	err = zio_wait(zio);
-	if (err != 0)
+	if (err != 0) {
 		tx->tx_err = err;
+		return;
+	}
+
+	/*
+	 * Each cloned block may dirty one BRT ZAP leaf in sync context.
+	 */
+	tx->tx_sync_reserve += len / blksz *
+	    brt_sync_dirty_est(tx->tx_pool->dp_spa);
 }
 
 void
@@ -964,15 +980,14 @@ dmu_tx_delay(dmu_tx_t *tx, uint64_t dirty)
 	/* Calculate minimum transaction time for the dirty data amount. */
 	delay_min_bytes =
 	    zfs_dirty_data_max * zfs_delay_min_dirty_percent / 100;
-	if (dirty > delay_min_bytes) {
+	if (dirty >= zfs_dirty_data_max) {
 		/*
-		 * The caller has already waited until we are under the max.
-		 * We make them pass us the amount of dirty data so we don't
-		 * have to handle the case of it being >= the max, which
-		 * could cause a divide-by-zero if it's == the max.
+		 * The caller has already waited until the dirty data is
+		 * under the max, but the sync context reservations added
+		 * on top of it may push the sum beyond it.
 		 */
-		ASSERT3U(dirty, <, zfs_dirty_data_max);
-
+		tx_time = zfs_delay_max_ns;
+	} else if (dirty > delay_min_bytes) {
 		tx_time = zfs_delay_scale * (dirty - delay_min_bytes) /
 		    (zfs_dirty_data_max - dirty);
 	}
@@ -1145,6 +1160,8 @@ dmu_tx_try_assign(dmu_tx_t *tx)
 			return (err);
 	}
 
+	dsl_pool_sync_reserve(tx->tx_pool, tx->tx_sync_reserve, tx);
+
 	DMU_TX_STAT_BUMP(dmu_tx_assigned);
 
 	return (0);
@@ -1357,7 +1374,7 @@ dmu_tx_wait(dmu_tx_t *tx)
 			DMU_TX_STAT_BUMP(dmu_tx_dirty_over_max);
 		while (dp->dp_dirty_total >= zfs_dirty_data_max)
 			cv_wait(&dp->dp_spaceavail_cv, &dp->dp_lock);
-		dirty = dp->dp_dirty_total;
+		dirty = dp->dp_dirty_total + dp->dp_sync_reserve_total;
 		mutex_exit(&dp->dp_lock);
 
 		dmu_tx_delay(tx, dirty);

module/zfs/dsl_pool.c

@@ -883,6 +883,9 @@ dsl_pool_sync_done(dsl_pool_t *dp, uint64_t txg)
 		dmu_buf_rele(ds->ds_dbuf, zilog);
 	}
 
+	/* Release whatever is left of this txg's sync dirty reservations. */
+	dsl_pool_sync_unreserve(dp, UINT64_MAX, txg);
+
 	dsl_pool_wrlog_clear(dp, txg);
 
 	ASSERT(!dmu_objset_is_dirty(dp->dp_meta_objset, txg));
@@ -980,7 +983,8 @@ dsl_pool_need_dirty_delay(dsl_pool_t *dp)
 	 * 32-bit due to memory constraints.  Pool-wide locks in hot path may
 	 * be too expensive, while we do not need a precise result here.
 	 */
-	return (dp->dp_dirty_total > delay_min_bytes);
+	return (dp->dp_dirty_total + dp->dp_sync_reserve_total >
+	    delay_min_bytes);
 }
 
 static boolean_t
@@ -988,7 +992,8 @@ dsl_pool_need_dirty_sync(dsl_pool_t *dp, uint64_t txg)
 {
 	uint64_t dirty_min_bytes =
 	    zfs_dirty_data_max * zfs_dirty_data_sync_percent / 100;
-	uint64_t dirty = dp->dp_dirty_pertxg[txg & TXG_MASK];
+	uint64_t dirty = dp->dp_dirty_pertxg[txg & TXG_MASK] +
+	    dp->dp_sync_reserve_pertxg[txg & TXG_MASK];
 
 	return (dirty > dirty_min_bytes);
 }
@@ -1009,6 +1014,41 @@ dsl_pool_dirty_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx)
 	}
 }
 
+/*
+ * Account for dirtied MOS data.  If dirtied in syncing context, in
+ * addition to the regular dirty space accounting it consumes the sync
+ * reservations made for the expected sync overhead (DDT/BRT ZAP
+ * updates, etc), so that the same data are not accounted against the
+ * write throttle twice.
+ */
+void
+dsl_pool_dirty_mos_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx)
+{
+	/*
+	 * The MOS may also be dirtied by the pool creation or open
+	 * contexts (e.g. pool history).  Those have no sync reservations
+	 * to consume and are accounted as regular dirty data.
+	 */
+	if (tx->tx_txg != spa_syncing_txg(dp->dp_spa)) {
+		dsl_pool_dirty_space(dp, space, tx);
+		return;
+	}
+
+	if (space <= 0)
+		return;
+
+	uint64_t txgoff = tx->tx_txg & TXG_MASK;
+	mutex_enter(&dp->dp_lock);
+	uint64_t resv = MIN((uint64_t)space,
+	    dp->dp_sync_reserve_pertxg[txgoff]);
+	dp->dp_sync_reserve_pertxg[txgoff] -= resv;
+	ASSERT3U(dp->dp_sync_reserve_total, >=, resv);
+	dp->dp_sync_reserve_total -= resv;
+	dp->dp_dirty_pertxg[txgoff] += space;
+	dsl_pool_dirty_delta(dp, space);
+	mutex_exit(&dp->dp_lock);
+}
+
 void
 dsl_pool_undirty_space(dsl_pool_t *dp, int64_t space, uint64_t txg)
 {
@@ -1028,6 +1068,48 @@ dsl_pool_undirty_space(dsl_pool_t *dp, int64_t space, uint64_t txg)
 	mutex_exit(&dp->dp_lock);
 }
 
+/*
+ * Reserve dirty space for the MOS updates (DDT/BRT ZAPs, etc) expected
+ * to be produced later by the sync thread on behalf of operations either
+ * assigned to this txg in open context or, in case of async destroys,
+ * performed by the sync thread itself earlier in this txg's sync.  While
+ * active, the reservation creates the same write throttle pressure as
+ * regular dirty data.  It is drained as the sync thread actually dirties
+ * MOS buffers, and any remainder is released when the txg sync completes.
+ */
+void
+dsl_pool_sync_reserve(dsl_pool_t *dp, uint64_t space, dmu_tx_t *tx)
+{
+	if (space == 0)
+		return;
+
+	mutex_enter(&dp->dp_lock);
+	dp->dp_sync_reserve_pertxg[tx->tx_txg & TXG_MASK] += space;
+	dp->dp_sync_reserve_total += space;
+	boolean_t needsync = !dmu_tx_is_syncing(tx) &&
+	    dsl_pool_need_dirty_sync(dp, tx->tx_txg);
+	mutex_exit(&dp->dp_lock);
+
+	if (needsync)
+		txg_kick(dp, tx->tx_txg);
+}
+
+void
+dsl_pool_sync_unreserve(dsl_pool_t *dp, uint64_t space, uint64_t txg)
+{
+	ASSERT3U(txg, ==, spa_syncing_txg(dp->dp_spa));
+
+	if (space == 0)
+		return;
+
+	mutex_enter(&dp->dp_lock);
+	space = MIN(space, dp->dp_sync_reserve_pertxg[txg & TXG_MASK]);
+	dp->dp_sync_reserve_pertxg[txg & TXG_MASK] -= space;
+	ASSERT3U(dp->dp_sync_reserve_total, >=, space);
+	dp->dp_sync_reserve_total -= space;
+	mutex_exit(&dp->dp_lock);
+}
+
 static int
 upgrade_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg)
 {

module/zfs/dsl_scan.c

@@ -3716,6 +3716,20 @@ dsl_scan_async_block_should_pause(dsl_scan_t *scn)
 		return (B_TRUE);
 	}
 
+	/*
+	 * Async frees of deduplicated or cloned blocks dirty DDT/BRT
+	 * ZAPs in this txg's sync context, which is not limited by the
+	 * write throttle.  Pause if this txg has already accumulated too
+	 * much dirty data, including the reservations for DDT/BRT updates
+	 * that have not been applied yet at this point of the sync.
+	 */
+	dsl_pool_t *dp = scn->scn_dp;
+	uint64_t txg = spa_syncing_txg(dp->dp_spa) & TXG_MASK;
+	if (dp->dp_dirty_pertxg[txg] + dp->dp_sync_reserve_pertxg[txg] >
+	    zfs_dirty_data_max / 2) {
+		return (B_TRUE);
+	}
+
 	elapsed_nanosecs = getlrtime() - scn->scn_sync_start_time;
 	return (elapsed_nanosecs / (NANOSEC / 2) > zfs_txg_timeout ||
 	    (NSEC2MSEC(elapsed_nanosecs) > scn->scn_async_block_min_time_ms &&
@@ -3746,6 +3760,20 @@ dsl_scan_free_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx)
 		 * async I/O (dedup, clone or gang block).
 		 */
 		scn->scn_async_frees_this_txg++;
+
+		/*
+		 * Reserve dirty space for the DDT/BRT ZAP updates this
+		 * free will produce later in this txg's sync, providing
+		 * feedback for the pause check above.
+		 */
+		spa_t *spa = scn->scn_dp->dp_spa;
+		uint64_t space = 0;
+		if (BP_GET_DEDUP(bp))
+			space = ddt_sync_dirty_est(spa);
+		else if (brt_maybe_exists(spa, bp))
+			space = brt_sync_dirty_est(spa);
+		dsl_pool_sync_reserve(scn->scn_dp, space, tx);
+
 		zio_nowait(zio);
 
 		/*