ZIO: Batch lightweight ZIOs

This further develops concept of ZIO batching from #18921, extending
it from the leaf vdev layer up the stack.  Now it covers the case of
I/O scheduler enabled, not covered before.  I/O aggregation of the
scheduler can complete several ZIOs at once, creating opportunity
for batching.

This change allows ZIO pipeline stages to return more than one ZIO,
chaining them via the field added in previous commit, renaming it
to more generic io_exec_next.  To identify ZIOs to batch, this change
introduces a new ZIO flag ZIO_FLAG_LIGHTWEIGHT, set for members of
a batch.  So once zio_done() or possibly other stage handlers return
a list of ZIOs, zio_execute() has now ability to decide which of them
to execute itself, and which to dispatch to taskqueues as before.

With this change my tests of 32KB block writes to 3x 5-wide NVMe RAIDZ1
on 64-core system with scheduler=on show throughput increase from
15.0GiB/s to 16.1GiB/s, while the taskqueue lock contention is
completely gone from profiles.

Also I've noticed that indirect blocks write ZIOs may get dispatched
to taskqueues by zio_issue_async() only to immeduately get blocked
waiting for children's readiness at zio_write_compress().  Skipping
that dispatch in half reduces sync threads time on random write
workload, though predictably increasing stress on ZIO pipeline by
pumping it much harder. ;)

Reviewed-by: Tony Hutter <[email protected]>
Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Alexander Motin <[email protected]>
Closes #18940

include/sys/zio.h

@@ -235,6 +235,7 @@ typedef uint64_t zio_flag_t;
 #define	ZIO_FLAG_DELEGATED	(1ULL << 32)
 #define	ZIO_FLAG_PREALLOCATED	(1ULL << 33)
 #define	ZIO_FLAG_POSTREAD	(1ULL << 34)
+#define	ZIO_FLAG_LIGHTWEIGHT	(1ULL << 35)
 
 #define	ZIO_ALLOCATOR_NONE	(-1)
 #define	ZIO_HAS_ALLOCATOR(zio)	((zio)->io_allocator != ZIO_ALLOCATOR_NONE)
@@ -573,7 +574,7 @@ struct zio {
 	/* Completion batching state */
 	zio_batch_t	*io_batch;	/* batch this zio is a member of */
 	zio_batch_t	*io_child_batch; /* batch its vdev children join */
-	zio_t		*io_batch_next;	/* link on zb_arrived */
+	zio_t		*io_exec_next;	/* link on a list of zios to execute */
 };
 
 enum blk_verify_flag {
@@ -656,8 +657,8 @@ extern void zio_delay_interrupt(zio_t *zio);
 extern void zio_deadman(zio_t *zio, const char *tag);
 
 extern void zio_batch_create(zio_t *pio);
-extern void zio_batch_rele(zio_t *pio);
-extern void zio_batch_leave(zio_t *zio);
+extern zio_t *zio_batch_rele(zio_t *pio);
+extern zio_t *zio_batch_leave(zio_t *zio);
 
 extern zio_t *zio_walk_parents(zio_t *cio, zio_link_t **);
 extern zio_t *zio_walk_children(zio_t *pio, zio_link_t **);

module/zfs/vdev_draid.c

@@ -2270,8 +2270,7 @@ vdev_draid_io_start(zio_t *zio)
 		}
 	}
 
-	zio_batch_rele(zio);
-	zio_execute(zio);
+	zio_execute(zio_batch_rele(zio));
 }
 
 /*

module/zfs/vdev_mirror.c

@@ -645,8 +645,7 @@ vdev_mirror_io_start(zio_t *zio)
 				first = B_FALSE;
 			}
 
-			zio_batch_rele(zio);
-			zio_execute(zio);
+			zio_execute(zio_batch_rele(zio));
 			return;
 		}
 		/*
@@ -691,8 +690,7 @@ vdev_mirror_io_start(zio_t *zio)
 		    vdev_mirror_child_done, mc));
 	}
 
-	zio_batch_rele(zio);
-	zio_execute(zio);
+	zio_execute(zio_batch_rele(zio));
 }
 
 static int

module/zfs/vdev_queue.c

@@ -955,7 +955,9 @@ vdev_queue_io(zio_t *zio)
 	 * since the slot would then be released only once the whole batch is
 	 * complete.
 	 */
-	zio_batch_leave(zio);
+	zio_t *batch = zio_batch_leave(zio);
+	if (batch != NULL)
+		zio_execute(batch);
 
 	mutex_enter(&vq->vq_lock);
 	vdev_queue_io_add(vq, zio);

module/zfs/vdev_raidz.c

@@ -2768,8 +2768,7 @@ vdev_raidz_io_start(zio_t *zio)
 		vdev_raidz_io_start_read(zio, rm);
 	}
 
-	zio_batch_rele(zio);
-	zio_execute(zio);
+	zio_execute(zio_batch_rele(zio));
 }
 
 /*

module/zfs/zio.c

@@ -832,10 +832,27 @@ zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait)
 	return (waiting);
 }
 
+/*
+ * The zios a pipeline stage hands back to zio_execute() to run once the
+ * current one stops, chained through io_exec_next in the order they were
+ * added.
+ */
+typedef struct zio_next {
+	zio_t		*zn_list;
+	zio_t		**zn_tailp;	/* where the next one is appended */
+} zio_next_t;
+
+static inline void
+zio_next_init(zio_next_t *next)
+{
+	next->zn_list = NULL;
+	next->zn_tailp = &next->zn_list;
+}
+
 __attribute__((always_inline))
 static inline void
 zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait,
-    zio_t **next_to_executep)
+    zio_next_t *nextp)
 {
 	uint64_t *countp = &pio->io_children[zio->io_child_type][wait];
 	int *errorp = &pio->io_child_error[zio->io_child_type];
@@ -878,11 +895,18 @@ zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait,
 		 * overflowing the stack when we have deeply nested
 		 * parent-child relationships, as we do with the "mega zio"
 		 * of writes for spa_sync(), and the chain of ZIL blocks.
+		 *
+		 * More than one parent may become executable at once, and all
+		 * of them go back to the caller.  It is the caller that keeps
+		 * one and dispatches the rest, since only it knows what else
+		 * is already waiting for its thread.
 		 */
-		if (next_to_executep != NULL && *next_to_executep == NULL &&
+		if (nextp != NULL &&
 		    (pio->io_type == zio->io_type ||
 		    (pio->io_type == ZIO_TYPE_NULL && !pio->io_done))) {
-			*next_to_executep = pio;
+			ASSERT3P(pio->io_exec_next, ==, NULL);
+			*nextp->zn_tailp = pio;
+			nextp->zn_tailp = &pio->io_exec_next;
 		} else {
 			zio_taskq_dispatch(pio, type, B_FALSE);
 		}
@@ -1033,6 +1057,7 @@ zio_destroy(zio_t *zio)
 {
 	ASSERT3P(zio->io_batch, ==, NULL);
 	ASSERT3P(zio->io_child_batch, ==, NULL);
+	ASSERT3P(zio->io_exec_next, ==, NULL);
 	metaslab_trace_fini(ZIO_ALLOC_LIST(zio));
 	list_destroy(&zio->io_parent_list);
 	list_destroy(&zio->io_child_list);
@@ -1688,27 +1713,38 @@ zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset,
 	    ZIO_STAGE_VDEV_IO_START >> 1, pipeline);
 	ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);
 
-	/*
-	 * Only children that come back from the block layer gain anything from
-	 * a batch.  Interior ones are dispatched by their own child's
-	 * zio_notify_parent() instead, as are distributed spares, which are
-	 * leaves that issue children of their own.  The scheduler may change
-	 * before a queue slot is actually taken, so vdev_should_queue_io() here
-	 * only keeps the batch away from vdevs that can never use it; the
-	 * binding decision is vdev_queue_io()'s.
-	 */
-	if (pio->io_child_batch != NULL && vd->vdev_ops->vdev_op_leaf &&
-	    vd->vdev_ops != &vdev_draid_spare_ops &&
-	    !vdev_should_queue_io(zio)) {
+	if (pio->io_child_batch != NULL) {
 		/*
-		 * The batch is dispatched to the taskq chosen for whichever
-		 * member arrives last, so they all have to choose the same one.
-		 * The flags that steer the choice are vdev-inherited, and type
-		 * and priority come from the parent at every call site.
+		 * Whatever wakes this child up, all it has left to do are the
+		 * few cheap stages of ZIO_VDEV_CHILD_PIPELINE, so it is better
+		 * run right there than dispatched.
 		 */
-		ASSERT3U(zio->io_type, ==, pio->io_type);
-		ASSERT3U(zio->io_priority, ==, pio->io_priority);
-		zio_batch_join(pio->io_child_batch, zio);
+		zio->io_flags |= ZIO_FLAG_LIGHTWEIGHT;
+
+		/*
+		 * Only children that come back from the block layer gain
+		 * anything from a batch.  Interior ones are dispatched by their
+		 * own child's zio_notify_parent() instead, as are distributed
+		 * spares, which are leaves that issue children of their own.
+		 * The scheduler may change before a queue slot is actually
+		 * taken, so vdev_should_queue_io() here only keeps the batch
+		 * away from vdevs that can never use it; the binding decision
+		 * is vdev_queue_io()'s.
+		 */
+		if (vd->vdev_ops->vdev_op_leaf &&
+		    vd->vdev_ops != &vdev_draid_spare_ops &&
+		    !vdev_should_queue_io(zio)) {
+			/*
+			 * The batch is dispatched to the taskq chosen for
+			 * whichever member arrives last, so they all have to
+			 * choose the same one.  The flags that steer the choice
+			 * are vdev-inherited, and type and priority come from
+			 * the parent at every call site.
+			 */
+			ASSERT3U(zio->io_type, ==, pio->io_type);
+			ASSERT3U(zio->io_priority, ==, pio->io_priority);
+			zio_batch_join(pio->io_child_batch, zio);
+		}
 	}
 
 	return (zio);
@@ -2242,6 +2278,21 @@ static zio_t *
 zio_issue_async(zio_t *zio)
 {
 	ASSERT((zio->io_type != ZIO_TYPE_WRITE) || ZIO_HAS_ALLOCATOR(zio));
+
+	/* Whatever may execute this again, it won't be this thread. */
+	zio->io_pipeline &= ~ZIO_STAGE_ISSUE_ASYNC;
+
+	/*
+	 * A zio whose children are not ready yet, such as an indirect block
+	 * write, has nothing to do in WRITE_COMPRESS but wait for them, so a
+	 * thread dispatched for it would only block.  Do that wait here and let
+	 * whoever wakes it up carry on, since that is not this thread anymore.
+	 */
+	if ((zio->io_pipeline & ZIO_STAGE_WRITE_COMPRESS) &&
+	    zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT |
+	    ZIO_CHILD_GANG_BIT, ZIO_WAIT_READY))
+		return (NULL);
+
 	zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
 	return (NULL);
 }
@@ -2297,26 +2348,36 @@ zio_batch_create(zio_t *pio)
 	pio->io_child_batch = zb;
 }
 
-static void
+/*
+ * Free the batch and return the list of members that arrived on it, for the
+ * caller to execute.  Members arrive by prepending, and are equal peers of one
+ * parent, so their order should not matter; the list is reversed into
+ * completion order only because it is walked here anyway.  Membership is
+ * dropped in that walk, both because VDEV_IO_ASSESS may reissue a member, which
+ * must not rejoin, and so that a member's later zio_batch_leave() does not
+ * touch the batch once it is freed.
+ */
+static zio_t *
 zio_batch_run(zio_batch_t *zb)
 {
-	zio_t *zio = zb->zb_arrived, *next;
+	zio_t *list = NULL, *zio, *next;
+
+	for (zio = zb->zb_arrived; zio != NULL; zio = next) {
+		next = zio->io_exec_next;
+		zio->io_batch = NULL;
+		zio->io_exec_next = list;
+		list = zio;
+	}
 
 	kmem_free(zb, sizeof (*zb));
 
-	while (zio != NULL) {
-		next = zio->io_batch_next;
-		/* VDEV_IO_ASSESS may reissue it; it must not rejoin. */
-		zio->io_batch = NULL;
-		zio_execute(zio);
-		zio = next;
-	}
+	return (list);
 }
 
 static void
 zio_batch_execute(void *arg)
 {
-	zio_batch_run(((zio_t *)arg)->io_batch);
+	zio_execute(zio_batch_run(((zio_t *)arg)->io_batch));
 }
 
 static void
@@ -2331,23 +2392,36 @@ zio_batch_join(zio_batch_t *zb, zio_t *zio)
  * Close the batch, once all of its members have been created, dropping the hold
  * that kept it from running while they were still being created.  Callers do
  * this before advancing the parent into VDEV_IO_DONE, where it will wait for
- * them; running the batch first is harmless, since its members only decrement
- * the parent's child count.  io_child_batch is cleared, so that children
- * created later, such as the repair writes from vdev_raidz_io_done(), do not
- * join a batch that is already gone.  May only be called from a context where
- * running the batch is legal.
+ * them; the members go ahead of it in the list, which is harmless, since all
+ * they do there is decrement its child count.  io_child_batch is cleared, so
+ * that children created later, such as the repair writes from
+ * vdev_raidz_io_done(), do not join a batch that is already gone.  Returns the
+ * parent, preceded by any members that arrived while it was still creating
+ * them, for the caller to execute.
  */
-void
+zio_t *
 zio_batch_rele(zio_t *pio)
 {
 	zio_batch_t *zb = pio->io_child_batch;
+	zio_t *list, *last;
+
+	ASSERT3P(pio->io_exec_next, ==, NULL);
 
 	if (zb == NULL)
-		return;
+		return (pio);
 
 	pio->io_child_batch = NULL;
-	if (atomic_dec_64_nv(&zb->zb_holds) == 0)
-		zio_batch_run(zb);
+	if (atomic_dec_64_nv(&zb->zb_holds) != 0)
+		return (pio);
+
+	if ((list = zio_batch_run(zb)) == NULL)
+		return (pio);
+
+	last = list;
+	while (last->io_exec_next != NULL)
+		last = last->io_exec_next;
+	last->io_exec_next = pio;
+	return (list);
 }
 
 /*
@@ -2375,7 +2449,7 @@ zio_batch_arrive(zio_t *zio)
 
 	do {
 		head = zb->zb_arrived;
-		zio->io_batch_next = head;
+		zio->io_exec_next = head;
 	} while (atomic_cas_ptr(&zb->zb_arrived, head, zio) != head);
 
 	if (atomic_dec_64_nv(&zb->zb_holds) == 0) {
@@ -2389,20 +2463,22 @@ zio_batch_arrive(zio_t *zio)
  * Give up a membership, either because the zio is about to take a vdev queue
  * slot after all, or because it reached its completion on a pipeline thread
  * rather than from the block layer, and so will run that completion itself.
- * Clearing io_batch makes this idempotent.  May only be called from a context
- * where running the batch is legal.
+ * Clearing io_batch makes this idempotent.  Returns the members for the caller
+ * to execute if this was the last hold on the batch, and NULL otherwise.
  */
-void
+zio_t *
 zio_batch_leave(zio_t *zio)
 {
 	zio_batch_t *zb = zio->io_batch;
 
 	if (likely(zb == NULL))
-		return;
+		return (NULL);
 
 	zio->io_batch = NULL;
-	if (atomic_dec_64_nv(&zb->zb_holds) == 0)
-		zio_batch_run(zb);
+	if (atomic_dec_64_nv(&zb->zb_holds) != 0)
+		return (NULL);
+
+	return (zio_batch_run(zb));
 }
 
 void
@@ -2617,68 +2693,130 @@ zio_execute_stack_check(zio_t *zio)
 	return (B_FALSE);
 }
 
+/*
+ * Run one pipeline stage, returning a list of zios to continue with, or NULL
+ * if this thread is done with it.
+ */
+__attribute__((always_inline))
+static inline zio_t *
+zio_execute_stage(zio_t *zio)
+{
+	enum zio_stage pipeline = zio->io_pipeline;
+	enum zio_stage stage = zio->io_stage;
+
+	zio->io_executor = curthread;
+
+	ASSERT(!MUTEX_HELD(&zio->io_lock));
+	ASSERT0P(zio->io_stall);
+	ASSERT(ISP2(stage));
+	ASSERT(pipeline & ~((stage << 1) - 1));
+
+	do {
+		stage <<= 1;
+	} while ((stage & pipeline) == 0);
+
+	ASSERT(stage <= ZIO_STAGE_DONE);
+
+	/*
+	 * If we are in interrupt context and this pipeline stage will grab
+	 * a config lock that is held across I/O, or may wait for an I/O that
+	 * needs an interrupt thread to complete, issue async to avoid deadlock.
+	 *
+	 * For VDEV_IO_START, we cut in line so that the io will be sent to
+	 * disk promptly.
+	 */
+	if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL &&
+	    zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) {
+		boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ?
+		    zio_requeue_io_start_cut_in_line : B_FALSE;
+		zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut);
+		return (NULL);
+	}
+
+	/*
+	 * If the current context doesn't have large enough stacks
+	 * the zio must be issued asynchronously to prevent overflow.
+	 */
+	if (zio_execute_stack_check(zio)) {
+		boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ?
+		    zio_requeue_io_start_cut_in_line : B_FALSE;
+		zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut);
+		return (NULL);
+	}
+
+	zio->io_stage = stage;
+	zio->io_pipeline_trace |= zio->io_stage;
+
+	/*
+	 * The zio pipeline stage returns the next zio to execute (typically
+	 * the same as this one), or NULL if we should stop.  It may also
+	 * chain more zios to it for us to execute later.
+	 */
+	return (zio_pipeline[highbit64(stage) - 1](zio));
+}
+
+/*
+ * Take all but the first of the zios a stage handed back off its head, and
+ * prepend the rest to those already pending.  Dispatch heavyweight ZIOs except
+ * the last, so that they could run in parallel.
+ */
+static inline void
+zio_execute_defer(zio_t *zio, zio_t **pendingp)
+{
+	zio_t *list = NULL, **tailp = &list;
+	zio_t *next;
+
+	for (zio_t *cur = zio->io_exec_next; cur != NULL; cur = next) {
+		next = cur->io_exec_next;
+		cur->io_exec_next = NULL;
+		if ((next != NULL || *pendingp != NULL) &&
+		    !(cur->io_flags & ZIO_FLAG_LIGHTWEIGHT)) {
+			zio_taskq_dispatch(cur,
+			    cur->io_stage < ZIO_STAGE_VDEV_IO_START ?
+			    ZIO_TASKQ_ISSUE : ZIO_TASKQ_INTERRUPT, B_FALSE);
+			continue;
+		}
+		*tailp = cur;
+		tailp = &cur->io_exec_next;
+	}
+
+	*tailp = *pendingp;
+	*pendingp = list;
+	zio->io_exec_next = NULL;
+}
+
 __attribute__((always_inline))
 static inline void
 __zio_execute(zio_t *zio)
 {
-	ASSERT3U(zio->io_queued_timestamp, >, 0);
+	zio_t *pending = zio->io_exec_next;
+	zio->io_exec_next = NULL;
 
-	while (zio->io_stage < ZIO_STAGE_DONE) {
-		enum zio_stage pipeline = zio->io_pipeline;
-		enum zio_stage stage = zio->io_stage;
+	for (;;) {
+		zio_t *last = zio;
+		while ((zio = zio_execute_stage(zio)) != NULL) {
+			if (zio->io_exec_next != NULL)
+				zio_execute_defer(zio, &pending);
 
-		zio->io_executor = curthread;
-
-		ASSERT(!MUTEX_HELD(&zio->io_lock));
-		ASSERT(ISP2(stage));
-		ASSERT0P(zio->io_stall);
-
-		do {
-			stage <<= 1;
-		} while ((stage & pipeline) == 0);
-
-		ASSERT(stage <= ZIO_STAGE_DONE);
-
-		/*
-		 * If we are in interrupt context and this pipeline stage
-		 * will grab a config lock that is held across I/O,
-		 * or may wait for an I/O that needs an interrupt thread
-		 * to complete, issue async to avoid deadlock.
-		 *
-		 * For VDEV_IO_START, we cut in line so that the io will
-		 * be sent to disk promptly.
-		 */
-		if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL &&
-		    zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) {
-			boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ?
-			    zio_requeue_io_start_cut_in_line : B_FALSE;
-			zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut);
-			return;
+			/*
+			 * A heavyweight zio is dispatched if others are already
+			 * waiting for this thread to let them run in parallel.
+			 */
+			if (zio != last && pending != NULL &&
+			    !(zio->io_flags & ZIO_FLAG_LIGHTWEIGHT)) {
+				zio_taskq_dispatch(zio,
+				    zio->io_stage < ZIO_STAGE_VDEV_IO_START ?
+				    ZIO_TASKQ_ISSUE : ZIO_TASKQ_INTERRUPT,
+				    B_FALSE);
+				break;
+			}
+			last = zio;
 		}
 
-		/*
-		 * If the current context doesn't have large enough stacks
-		 * the zio must be issued asynchronously to prevent overflow.
-		 */
-		if (zio_execute_stack_check(zio)) {
-			boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ?
-			    zio_requeue_io_start_cut_in_line : B_FALSE;
-			zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut);
-			return;
-		}
-
-		zio->io_stage = stage;
-		zio->io_pipeline_trace |= zio->io_stage;
-
-		/*
-		 * The zio pipeline stage returns the next zio to execute
-		 * (typically the same as this one), or NULL if we should
-		 * stop.
-		 */
-		zio = zio_pipeline[highbit64(stage) - 1](zio);
-
-		if (zio == NULL)
+		if ((zio = pending) == NULL)
 			return;
+		pending = zio->io_exec_next;
+		zio->io_exec_next = NULL;
 	}
 }
 
@@ -5084,16 +5222,19 @@ zio_vdev_io_done(zio_t *zio)
 	}
 
 	/*
-	 * Left this late so that running the batch, which this may do, is not
-	 * counted as part of this zio's own service time above.
+	 * This zio got here on a pipeline thread rather than from the block
+	 * layer, so it runs its own completion and gives up its membership.
+	 * The batch is chained only below, to keep it clear of whatever
+	 * vdev_op_io_done() may do with this zio.
 	 */
-	zio_batch_leave(zio);
+	zio_t *batch = zio_batch_leave(zio);
 
 	ops->vdev_op_io_done(zio);
 
 	if (unexpected_error && vd->vdev_remove_wanted == B_FALSE)
 		VERIFY0P(vdev_probe(vd, zio));
 
+	zio->io_exec_next = batch;
 	return (zio);
 }
 
@@ -5161,7 +5302,7 @@ zio_vdev_io_assess(zio_t *zio)
 	}
 
 	/* A repair write bypass skips VDEV_IO_DONE entirely. */
-	zio_batch_leave(zio);
+	zio->io_exec_next = zio_batch_leave(zio);
 
 	if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
 		spa_config_exit(zio->io_spa, SCL_ZIO, zio);
@@ -5698,10 +5839,14 @@ zio_ready(zio_t *zio)
 	 * io_parent_list, from 'pio_next' onward, cannot change because
 	 * all parents must wait for us to be done before they can be done.
 	 */
+	zio_next_t next;
+	zio_next_init(&next);
 	for (; pio != NULL; pio = pio_next) {
 		pio_next = zio_walk_parents(zio, &zl);
-		zio_notify_parent(pio, zio, ZIO_WAIT_READY, NULL);
+		zio_notify_parent(pio, zio, ZIO_WAIT_READY, &next);
 	}
+	ASSERT3P(zio->io_exec_next, ==, NULL);
+	zio->io_exec_next = next.zn_list;
 
 	if (zio->io_flags & ZIO_FLAG_NODATA) {
 		if (bp != NULL && BP_IS_GANG(bp)) {
@@ -6066,7 +6211,7 @@ zio_done(zio_t *zio)
 				zio_remove_child(pio, zio, remove_zl);
 				/*
 				 * This is a rare code path, so we don't
-				 * bother with "next_to_execute".
+				 * bother with the "next" list.
 				 */
 				zio_notify_parent(pio, zio, ZIO_WAIT_DONE,
 				    NULL);
@@ -6083,7 +6228,7 @@ zio_done(zio_t *zio)
 			zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE;
 			/*
 			 * This is a rare code path, so we don't bother with
-			 * "next_to_execute".
+			 * the "next" list.
 			 */
 			zio_notify_parent(pio, zio, ZIO_WAIT_DONE, NULL);
 		} else if (zio->io_post & ZIO_POST_SUSPEND) {
@@ -6152,16 +6297,17 @@ zio_done(zio_t *zio)
 	mutex_exit(&zio->io_lock);
 
 	/*
-	 * We are done executing this zio.  We may want to execute a parent
-	 * next.  See the comment in zio_notify_parent().
+	 * We are done executing this zio.  We may want to execute some of its
+	 * parents next.  See the comment in zio_notify_parent().
 	 */
-	zio_t *next_to_execute = NULL;
+	zio_next_t next;
+	zio_next_init(&next);
 	zl = NULL;
 	for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) {
 		zio_link_t *remove_zl = zl;
 		pio_next = zio_walk_parents(zio, &zl);
 		zio_remove_child(pio, zio, remove_zl);
-		zio_notify_parent(pio, zio, ZIO_WAIT_DONE, &next_to_execute);
+		zio_notify_parent(pio, zio, ZIO_WAIT_DONE, &next);
 	}
 
 	if (zio->io_waiter != NULL) {
@@ -6173,7 +6319,7 @@ zio_done(zio_t *zio)
 		zio_destroy(zio);
 	}
 
-	return (next_to_execute);
+	return (next.zn_list);
 }
 
 /*