ZIO: Batch vdev children completions

A vdev child ZIO's life after return from the block layer is three cheap
pipeline stages, yet each child costs its own taskq dispatch and context
switch to get there, only to decrement the parent's child count and die.
Even with increased number of taskqueues this can create a huge lock
contention and scheduler overheads, especially on large systems.

To avoid that extra cost collect the leaf children of a RAIDZ, dRAID or
mirror parents as they come back, and only once the last one is in,
process all of their completions, followed by the parent's one, all on
a single thread.

This optimization is mutually exclusive with I/O scheduler, since
delayed completions there may lead to a deadlock, but for that case I
have somewhat alike optimization idea later.

With this change my tests of 32KB block writes to 3x 5-wide NVMe RAIDZ1
on 64-core system show throughput increase from 13 GiB/s to 17 GiB/s,
while the taskqueue lock contention is completely gone from profiles.

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Alexander Motin <[email protected]>
Closes #18921

include/sys/vdev.h

@@ -176,6 +176,7 @@ extern void vdev_queue_init(vdev_t *vd);
 extern void vdev_queue_fini(vdev_t *vd);
 extern zio_t *vdev_queue_io(zio_t *zio);
 extern void vdev_queue_io_done(zio_t *zio);
+extern boolean_t vdev_should_queue_io(zio_t *zio);
 extern void vdev_queue_change_io_priority(zio_t *zio, zio_priority_t priority);
 
 extern uint32_t vdev_queue_length(vdev_t *vd);

include/sys/zio.h

@@ -473,6 +473,16 @@ enum zio_qstate {
 	ZIO_QS_ACTIVE,
 };
 
+/*
+ * A set of sibling child ZIOs whose completions are processed together on one
+ * thread once all of them have returned from the block layer.  See the comment
+ * above zio_batch_enabled.
+ */
+typedef struct zio_batch {
+	zio_t		*zb_arrived;	/* lock-free LIFO of arrived members */
+	uint64_t	zb_holds;	/* members not yet arrived + creator */
+} zio_batch_t;
+
 struct zio {
 	/* Core information about this I/O */
 	zbookmark_phys_t	io_bookmark;
@@ -559,6 +569,11 @@ struct zio {
 
 	/* Taskq dispatching state */
 	taskq_ent_t	io_tqent;
+
+	/* Completion batching state */
+	zio_batch_t	*io_batch;	/* batch this zio is a member of */
+	zio_batch_t	*io_child_batch; /* batch its vdev children join */
+	zio_t		*io_batch_next;	/* link on zb_arrived */
 };
 
 enum blk_verify_flag {
@@ -640,6 +655,10 @@ extern void zio_delay_init(zio_t *zio);
 extern void zio_delay_interrupt(zio_t *zio);
 extern void zio_deadman(zio_t *zio, const char *tag);
 
+extern void zio_batch_create(zio_t *pio);
+extern void zio_batch_rele(zio_t *pio);
+extern void zio_batch_leave(zio_t *zio);
+
 extern zio_t *zio_walk_parents(zio_t *cio, zio_link_t **);
 extern zio_t *zio_walk_children(zio_t *pio, zio_link_t **);
 extern zio_t *zio_unique_parent(zio_t *cio);

man/man4/zfs.4

@@ -2749,6 +2749,12 @@ using LZ4 and zstd-1 passes is enabled.
 Minimal uncompressed size (inclusive) of a record before the early abort
 heuristic will be attempted.
 .
+.It Sy zio_batch_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
+Collect the vdev children of a RAIDZ, dRAID or mirror I/O operation as they
+complete, and process all of their completions on a single thread once the
+last of them is in, rather than one thread each.
+Applies only to children that bypass the vdev I/O queue.
+.
 .It Sy zio_deadman_log_all Ns = Ns Sy 0 Ns | Ns 1 Pq int
 If non-zero, the zio deadman will produce debugging messages
 .Pq see Sy zfs_dbgmsg_enable

module/zfs/vdev_draid.c

@@ -2256,6 +2256,7 @@ vdev_draid_io_start(zio_t *zio)
 	raidz_map_t *rm = vdev_draid_map_alloc(zio);
 	zio->io_vsd = rm;
 	zio->io_vsd_ops = &vdev_raidz_vsd_ops;
+	zio_batch_create(zio);
 
 	if (zio->io_type == ZIO_TYPE_WRITE) {
 		for (int i = 0; i < rm->rm_nrows; i++) {
@@ -2269,6 +2270,7 @@ vdev_draid_io_start(zio_t *zio)
 		}
 	}
 
+	zio_batch_rele(zio);
 	zio_execute(zio);
 }
 

module/zfs/vdev_mirror.c

@@ -619,6 +619,10 @@ vdev_mirror_io_start(zio_t *zio)
 			 * them in vdev_mirror_io_done() otherwise.
 			 */
 			boolean_t first = B_TRUE;
+
+			if (mm->mm_children > 1)
+				zio_batch_create(zio);
+
 			for (c = 0; c < mm->mm_children; c++) {
 				mc = &mm->mm_child[c];
 
@@ -640,6 +644,8 @@ vdev_mirror_io_start(zio_t *zio)
 				    vdev_mirror_child_done, mc));
 				first = B_FALSE;
 			}
+
+			zio_batch_rele(zio);
 			zio_execute(zio);
 			return;
 		}
@@ -658,6 +664,9 @@ vdev_mirror_io_start(zio_t *zio)
 		children = mm->mm_children;
 	}
 
+	if (children > 1)
+		zio_batch_create(zio);
+
 	while (children--) {
 		mc = &mm->mm_child[c++];
 
@@ -682,6 +691,7 @@ vdev_mirror_io_start(zio_t *zio)
 		    vdev_mirror_child_done, mc));
 	}
 
+	zio_batch_rele(zio);
 	zio_execute(zio);
 }
 

module/zfs/vdev_queue.c

@@ -869,7 +869,7 @@ again:
 	return (zio);
 }
 
-static boolean_t
+boolean_t
 vdev_should_queue_io(zio_t *zio)
 {
 	vdev_t *vd = zio->io_vd;
@@ -950,6 +950,13 @@ vdev_queue_io(zio_t *zio)
 		return (zio);
 	}
 
+	/*
+	 * A zio holding a queue slot can not be part of a completion batch,
+	 * since the slot would then be released only once the whole batch is
+	 * complete.
+	 */
+	zio_batch_leave(zio);
+
 	mutex_enter(&vq->vq_lock);
 	vdev_queue_io_add(vq, zio);
 	nio = vdev_queue_io_to_issue(vq);
@@ -978,9 +985,19 @@ vdev_queue_io_done(zio_t *zio)
 	zio_t *dio, *nio;
 	zio_link_t *zl = NULL;
 
-	hrtime_t now = gethrtime();
-	vq->vq_io_complete_ts = now;
-	vq->vq_io_delta_ts = zio->io_delta = now - zio->io_timestamp;
+	/*
+	 * io_delta is already set if this zio's completion was deferred, so
+	 * that the service time excludes however long it waited.  Batched
+	 * completions are not processed in arrival order, so the resulting
+	 * completion time may predate what vq_io_complete_ts holds.
+	 */
+	if (zio->io_delta == 0)
+		zio->io_delta = gethrtime() - zio->io_timestamp;
+	hrtime_t now = zio->io_timestamp + zio->io_delta;
+	if (now > vq->vq_io_complete_ts) {
+		vq->vq_io_complete_ts = now;
+		vq->vq_io_delta_ts = zio->io_delta;
+	}
 
 	if (zio->io_queue_state == ZIO_QS_NONE)
 		return;

module/zfs/vdev_raidz.c

@@ -2753,6 +2753,8 @@ vdev_raidz_io_start(zio_t *zio)
 
 	zio->io_vsd = rm;
 	zio->io_vsd_ops = &vdev_raidz_vsd_ops;
+	zio_batch_create(zio);
+
 	if (zio->io_type == ZIO_TYPE_WRITE) {
 		for (int i = 0; i < rm->rm_nrows; i++) {
 			vdev_raidz_io_start_write(zio, rm->rm_row[i]);
@@ -2766,6 +2768,7 @@ vdev_raidz_io_start(zio_t *zio)
 		vdev_raidz_io_start_read(zio, rm);
 	}
 
+	zio_batch_rele(zio);
 	zio_execute(zio);
 }
 

module/zfs/zio.c

@@ -164,6 +164,7 @@ static kstat_t *zio_ksp;
 static inline void __zio_execute(zio_t *zio);
 
 static void zio_taskq_dispatch(zio_t *, zio_taskq_type_t, boolean_t);
+static void zio_batch_join(zio_batch_t *, zio_t *);
 
 static int
 zio_kstats_update(kstat_t *ksp, int rw)
@@ -1030,6 +1031,8 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
 void
 zio_destroy(zio_t *zio)
 {
+	ASSERT3P(zio->io_batch, ==, NULL);
+	ASSERT3P(zio->io_child_batch, ==, NULL);
 	metaslab_trace_fini(ZIO_ALLOC_LIST(zio));
 	list_destroy(&zio->io_parent_list);
 	list_destroy(&zio->io_child_list);
@@ -1685,6 +1688,29 @@ zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset,
 	    ZIO_STAGE_VDEV_IO_START >> 1, pipeline);
 	ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);
 
+	/*
+	 * Only children that come back from the block layer gain anything from
+	 * a batch.  Interior ones are dispatched by their own child's
+	 * zio_notify_parent() instead, as are distributed spares, which are
+	 * leaves that issue children of their own.  The scheduler may change
+	 * before a queue slot is actually taken, so vdev_should_queue_io() here
+	 * only keeps the batch away from vdevs that can never use it; the
+	 * binding decision is vdev_queue_io()'s.
+	 */
+	if (pio->io_child_batch != NULL && vd->vdev_ops->vdev_op_leaf &&
+	    vd->vdev_ops != &vdev_draid_spare_ops &&
+	    !vdev_should_queue_io(zio)) {
+		/*
+		 * The batch is dispatched to the taskq chosen for whichever
+		 * member arrives last, so they all have to choose the same one.
+		 * The flags that steer the choice are vdev-inherited, and type
+		 * and priority come from the parent at every call site.
+		 */
+		ASSERT3U(zio->io_type, ==, pio->io_type);
+		ASSERT3U(zio->io_priority, ==, pio->io_priority);
+		zio_batch_join(pio->io_child_batch, zio);
+	}
+
 	return (zio);
 }
 
@@ -2151,7 +2177,8 @@ zio_free_bp_init(zio_t *zio)
  */
 
 static void
-zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline)
+zio_taskq_dispatch_func(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline,
+    task_func_t *func)
 {
 	spa_t *spa = zio->io_spa;
 	zio_type_t t = zio->io_type;
@@ -2183,7 +2210,13 @@ zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline)
 
 	ASSERT3U(q, <, ZIO_TASKQ_TYPES);
 
-	spa_taskq_dispatch(spa, t, q, zio_execute, zio, cutinline);
+	spa_taskq_dispatch(spa, t, q, func, zio, cutinline);
+}
+
+static void
+zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline)
+{
+	zio_taskq_dispatch_func(zio, q, cutinline, zio_execute);
 }
 
 static boolean_t
@@ -2213,9 +2246,170 @@ zio_issue_async(zio_t *zio)
 	return (NULL);
 }
 
+/*
+ * ==========================================================================
+ * Completion batching
+ * ==========================================================================
+ *
+ * A vdev child's entire life after the block layer returns is three pipeline
+ * stages: VDEV_IO_DONE, VDEV_IO_ASSESS and DONE (ZIO_VDEV_CHILD_PIPELINE).
+ * For a parent with many children, such as RAIDZ or a mirror, every child but
+ * the last does nothing there except decrement the parent's child count, yet
+ * each one costs a taskq dispatch and a context switch to get there.
+ *
+ * A batch collects the children of one parent as they return, and once the last
+ * of them is in, runs all of their completions, and then the parent's, on one
+ * thread.  Arrival happens in the block layer completion context, so it is
+ * lock-free: bio_endio() on Linux can run in softirq, where the sleepable
+ * mutex_t is not usable.
+ *
+ * Only children that actually arrive from the block layer join zb_arrived; one
+ * that reaches its completion on a pipeline thread instead just releases its
+ * hold and runs that completion itself, as it would have without any of this.
+ * Building the list on arrival is what allows that, since such a child may run
+ * all the way to zio_destroy() long before the batch does.
+ *
+ * A child that occupies a vdev queue slot must never be a member.  The slot is
+ * released by vdev_queue_io_done(), part of the deferred completion, while a
+ * sibling may still be queued for a slot on another vdev whose slots are in
+ * turn held by the members of other waiting batches -- a cycle that deadlocks.
+ */
+static int zio_batch_enabled = 1;
+
+/*
+ * Open a batch collecting the completions of the vdev children this zio is
+ * about to create, which do little but count down to it.  Every one of those
+ * children must be created before the matching zio_batch_rele().
+ */
+void
+zio_batch_create(zio_t *pio)
+{
+	zio_batch_t *zb;
+
+	ASSERT3P(pio->io_child_batch, ==, NULL);
+
+	if (!zio_batch_enabled)
+		return;
+
+	zb = kmem_alloc(sizeof (*zb), KM_SLEEP);
+	zb->zb_arrived = NULL;
+	zb->zb_holds = 1;		/* creator's hold */
+	pio->io_child_batch = zb;
+}
+
+static void
+zio_batch_run(zio_batch_t *zb)
+{
+	zio_t *zio = zb->zb_arrived, *next;
+
+	kmem_free(zb, sizeof (*zb));
+
+	while (zio != NULL) {
+		next = zio->io_batch_next;
+		/* VDEV_IO_ASSESS may reissue it; it must not rejoin. */
+		zio->io_batch = NULL;
+		zio_execute(zio);
+		zio = next;
+	}
+}
+
+static void
+zio_batch_execute(void *arg)
+{
+	zio_batch_run(((zio_t *)arg)->io_batch);
+}
+
+static void
+zio_batch_join(zio_batch_t *zb, zio_t *zio)
+{
+	ASSERT3P(zio->io_batch, ==, NULL);
+	atomic_inc_64(&zb->zb_holds);
+	zio->io_batch = zb;
+}
+
+/*
+ * Close the batch, once all of its members have been created, dropping the hold
+ * that kept it from running while they were still being created.  Callers do
+ * this before advancing the parent into VDEV_IO_DONE, where it will wait for
+ * them; running the batch first is harmless, since its members only decrement
+ * the parent's child count.  io_child_batch is cleared, so that children
+ * created later, such as the repair writes from vdev_raidz_io_done(), do not
+ * join a batch that is already gone.  May only be called from a context where
+ * running the batch is legal.
+ */
+void
+zio_batch_rele(zio_t *pio)
+{
+	zio_batch_t *zb = pio->io_child_batch;
+
+	if (zb == NULL)
+		return;
+
+	pio->io_child_batch = NULL;
+	if (atomic_dec_64_nv(&zb->zb_holds) == 0)
+		zio_batch_run(zb);
+}
+
+/*
+ * Called in place of a member's taskq dispatch, from the block layer
+ * completion context.  Returns B_TRUE if the zio was absorbed by a batch, in
+ * which case the caller must not touch it again.
+ */
+static boolean_t
+zio_batch_arrive(zio_t *zio)
+{
+	zio_batch_t *zb = zio->io_batch;
+	zio_t *head;
+
+	if (zb == NULL)
+		return (B_FALSE);
+
+	/*
+	 * The completion is deferred, so take the service time here, while it
+	 * still is one: vdev_child_slow_outlier() sits out RAIDZ children based
+	 * on io_delta and io_delay.  A non-zero io_delta also tells the stages
+	 * below when the block layer returned, as io_timestamp + io_delta.
+	 */
+	ASSERT3U(zio->io_timestamp, !=, 0);
+	zio->io_delta = gethrtime() - zio->io_timestamp;
+
+	do {
+		head = zb->zb_arrived;
+		zio->io_batch_next = head;
+	} while (atomic_cas_ptr(&zb->zb_arrived, head, zio) != head);
+
+	if (atomic_dec_64_nv(&zb->zb_holds) == 0) {
+		zio_taskq_dispatch_func(zio, ZIO_TASKQ_INTERRUPT, B_FALSE,
+		    zio_batch_execute);
+	}
+	return (B_TRUE);
+}
+
+/*
+ * Give up a membership, either because the zio is about to take a vdev queue
+ * slot after all, or because it reached its completion on a pipeline thread
+ * rather than from the block layer, and so will run that completion itself.
+ * Clearing io_batch makes this idempotent.  May only be called from a context
+ * where running the batch is legal.
+ */
+void
+zio_batch_leave(zio_t *zio)
+{
+	zio_batch_t *zb = zio->io_batch;
+
+	if (likely(zb == NULL))
+		return;
+
+	zio->io_batch = NULL;
+	if (atomic_dec_64_nv(&zb->zb_holds) == 0)
+		zio_batch_run(zb);
+}
+
 void
 zio_interrupt(void *zio)
 {
+	if (zio_batch_arrive(zio))
+		return;
 	zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE);
 }
 
@@ -4666,6 +4860,7 @@ zio_vdev_io_start(zio_t *zio)
 	uint64_t align;
 	spa_t *spa = zio->io_spa;
 
+	zio->io_delta = 0;
 	zio->io_delay = 0;
 
 	ASSERT0(zio->io_error);
@@ -4859,8 +5054,12 @@ zio_vdev_io_done(zio_t *zio)
 	    zio->io_type == ZIO_TYPE_FLUSH ||
 	    zio->io_type == ZIO_TYPE_TRIM);
 
-	if (zio->io_delay)
-		zio->io_delay = gethrtime() - zio->io_delay;
+	if (zio->io_delay) {
+		/* io_delta is set only if the completion was deferred. */
+		zio->io_delay = (zio->io_delta != 0 ?
+		    zio->io_timestamp + zio->io_delta : gethrtime()) -
+		    zio->io_delay;
+	}
 
 	if (vd != NULL && vd->vdev_ops->vdev_op_leaf &&
 	    vd->vdev_ops != &vdev_draid_spare_ops) {
@@ -4884,6 +5083,12 @@ zio_vdev_io_done(zio_t *zio)
 		}
 	}
 
+	/*
+	 * Left this late so that running the batch, which this may do, is not
+	 * counted as part of this zio's own service time above.
+	 */
+	zio_batch_leave(zio);
+
 	ops->vdev_op_io_done(zio);
 
 	if (unexpected_error && vd->vdev_remove_wanted == B_FALSE)
@@ -4955,6 +5160,9 @@ zio_vdev_io_assess(zio_t *zio)
 		return (NULL);
 	}
 
+	/* A repair write bypass skips VDEV_IO_DONE entirely. */
+	zio_batch_leave(zio);
+
 	if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
 		spa_config_exit(zio->io_spa, SCL_ZIO, zio);
 
@@ -6220,6 +6428,9 @@ ZFS_MODULE_PARAM(zfs_zio, zio_, slow_io_ms, INT, ZMOD_RW,
 ZFS_MODULE_PARAM(zfs_zio, zio_, requeue_io_start_cut_in_line, INT, ZMOD_RW,
 	"Prioritize requeued I/O");
 
+ZFS_MODULE_PARAM(zfs_zio, zio_, batch_enabled, INT, ZMOD_RW,
+	"Batch processing of vdev children I/O completions");
+
 ZFS_MODULE_PARAM(zfs, zfs_, sync_pass_deferred_free,  UINT, ZMOD_RW,
 	"Defer frees starting in this pass");