Write uberblocks only to vdevs used by the txg

Instead of picking random top-level vdevs to write the uberblock to,
prefer the ones actually written during this txg.  This allows idle
vdevs to stay asleep, which is important for pools with spun-down
HDDs.

If fewer than SPA_SYNC_MIN_VDEVS were written, top up from special
and dedup vdevs, which are expected to have no seek penalty.  Pools
without those classes keep the old behavior of topping up from any
vdev, preserving the uberblock redundancy where there is nothing to
gain by reducing it.

While here, pass spa explicitly to vdev_config_sync() and
vdev_uberblock_sync_list() instead of deriving it from svd[0].

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Alexander Motin <[email protected]>
Closes #19003

include/sys/vdev.h

@@ -185,7 +185,8 @@ extern boolean_t vdev_queue_pool_busy(spa_t *spa);
 
 extern void vdev_config_dirty(vdev_t *vd);
 extern void vdev_config_clean(vdev_t *vd);
-extern int vdev_config_sync(vdev_t **svd, int svdcount, uint64_t txg);
+extern int vdev_config_sync(spa_t *spa, vdev_t **svd, int svdcount,
+    uint64_t txg);
 
 extern void vdev_state_dirty(vdev_t *vd);
 extern void vdev_state_clean(vdev_t *vd);
@@ -219,7 +220,8 @@ extern void vdev_label_write(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t
     offset, uint64_t size, zio_done_func_t *done, void *priv, int flags);
 extern int vdev_label_read_bootenv(vdev_t *, nvlist_t *);
 extern int vdev_label_write_bootenv(vdev_t *, nvlist_t *);
-extern int vdev_uberblock_sync_list(vdev_t **, int, struct uberblock *, int);
+extern int vdev_uberblock_sync_list(spa_t *, vdev_t **, int,
+    struct uberblock *, int);
 extern int vdev_uberblock_compare(const struct uberblock *,
     const struct uberblock *);
 extern int vdev_check_boot_reserve(spa_t *, vdev_t *);

module/zfs/spa.c

@@ -6065,7 +6065,8 @@ spa_ld_checkpoint_rewind(spa_t *spa)
 			if (svdcount == SPA_SYNC_MIN_VDEVS)
 				break;
 		}
-		error = vdev_config_sync(svd, svdcount, spa->spa_first_txg);
+		error = vdev_config_sync(spa, svd, svdcount,
+		    spa->spa_first_txg);
 		if (error == 0)
 			spa->spa_last_synced_guid = rvd->vdev_guid;
 		spa_config_exit(spa, SCL_ALL, FTAG);
@@ -11074,6 +11075,59 @@ spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx)
 	} while (dmu_objset_is_dirty(mos, txg));
 }
 
+/*
+ * Select up to SPA_SYNC_MIN_VDEVS top-level vdevs to write the uberblock to.
+ * First take the ones written during this txg, so that the idle ones may stay
+ * asleep.  If there are not enough, top up from special and dedup vdevs, which
+ * are expected to have no seek penalty.  Pools having none of those keep the
+ * old behavior of topping up from any vdev.
+ */
+static int
+spa_select_uberblock_vdevs(spa_t *spa, vdev_t **svd, uint64_t txg)
+{
+	vdev_t *rvd = spa->spa_root_vdev;
+	uint64_t children = rvd->vdev_children;
+	uint64_t c0 = random_in_range(children);
+	boolean_t tiered = spa_has_special(spa) || spa_has_dedup(spa);
+	int svdcount = 0;
+
+	for (int pass = 0; pass < 3; pass++) {
+		if (pass == 2 && svdcount > 0 && tiered)
+			break;
+
+		for (uint64_t c = 0; c < children &&
+		    svdcount < SPA_SYNC_MIN_VDEVS; c++) {
+			vdev_t *vd = rvd->vdev_child[(c0 + c) % children];
+			boolean_t dup = B_FALSE;
+
+			if (vd->vdev_ms_array == 0 || vd->vdev_islog ||
+			    !vdev_is_concrete(vd))
+				continue;
+
+			if (pass == 0 && !txg_list_member(
+			    &spa->spa_vdev_txg_list, vd, TXG_CLEAN(txg)))
+				continue;
+
+			if (pass == 1) {
+				metaslab_class_t *mc = vd->vdev_mg != NULL ?
+				    vd->vdev_mg->mg_class : NULL;
+				if (mc != spa_special_class(spa) &&
+				    mc != spa_dedup_class(spa))
+					continue;
+			}
+
+			for (int i = 0; i < svdcount; i++)
+				dup |= (svd[i] == vd);
+			if (dup)
+				continue;
+
+			svd[svdcount++] = vd;
+		}
+	}
+
+	return (svdcount);
+}
+
 /*
  * Rewrite the vdev configuration (which includes the uberblock) to
  * commit the transaction group.
@@ -11100,30 +11154,12 @@ spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx)
 
 		if (list_is_empty(&spa->spa_config_dirty_list)) {
 			vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL };
-			int svdcount = 0;
-			int children = rvd->vdev_children;
-			int c0 = random_in_range(children);
+			int svdcount = spa_select_uberblock_vdevs(spa, svd,
+			    txg);
 
-			for (int c = 0; c < children; c++) {
-				vdev_t *vd =
-				    rvd->vdev_child[(c0 + c) % children];
-
-				/* Stop when revisiting the first vdev */
-				if (c > 0 && svd[0] == vd)
-					break;
-
-				if (vd->vdev_ms_array == 0 ||
-				    vd->vdev_islog ||
-				    !vdev_is_concrete(vd))
-					continue;
-
-				svd[svdcount++] = vd;
-				if (svdcount == SPA_SYNC_MIN_VDEVS)
-					break;
-			}
-			error = vdev_config_sync(svd, svdcount, txg);
+			error = vdev_config_sync(spa, svd, svdcount, txg);
 		} else {
-			error = vdev_config_sync(rvd->vdev_child,
+			error = vdev_config_sync(spa, rvd->vdev_child,
 			    rvd->vdev_children, txg);
 		}
 

module/zfs/vdev_label.c

@@ -1900,9 +1900,9 @@ vdev_uberblock_sync(zio_t *zio, uint64_t *good_writes,
 
 /* Sync the uberblocks to all vdevs in svd[] */
 int
-vdev_uberblock_sync_list(vdev_t **svd, int svdcount, uberblock_t *ub, int flags)
+vdev_uberblock_sync_list(spa_t *spa, vdev_t **svd, int svdcount,
+    uberblock_t *ub, int flags)
 {
-	spa_t *spa = svd[0]->vdev_spa;
 	zio_t *zio;
 	uint64_t good_writes = 0;
 
@@ -2137,9 +2137,8 @@ vdev_label_sync_list(spa_t *spa, int l, uint64_t txg, int flags)
  * at any time, you can just call it again, and it will resume its work.
  */
 int
-vdev_config_sync(vdev_t **svd, int svdcount, uint64_t txg)
+vdev_config_sync(spa_t *spa, vdev_t **svd, int svdcount, uint64_t txg)
 {
-	spa_t *spa = svd[0]->vdev_spa;
 	uberblock_t *ub = &spa->spa_uberblock;
 	int error = 0;
 	int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
@@ -2230,7 +2229,8 @@ retry:
 	 *	been successfully committed) will be valid with respect
 	 *	to the new uberblocks.
 	 */
-	if ((error = vdev_uberblock_sync_list(svd, svdcount, ub, flags)) != 0) {
+	if ((error = vdev_uberblock_sync_list(spa, svd, svdcount, ub,
+	    flags)) != 0) {
 		if ((flags & ZIO_FLAG_IO_RETRY) != 0) {
 			zfs_dbgmsg("vdev_uberblock_sync_list() returned error "
 			    "%d for pool '%s'", error, spa_name(spa));

module/zfs/vdev_raidz.c

@@ -4742,7 +4742,7 @@ io_error_exit:
 	 */
 	RAIDZ_REFLOW_SET(&spa->spa_ubsync, RRSS_SCRATCH_VALID, logical_size);
 	spa->spa_ubsync.ub_timestamp++;
-	ASSERT0(vdev_uberblock_sync_list(&spa->spa_root_vdev, 1,
+	ASSERT0(vdev_uberblock_sync_list(spa, &spa->spa_root_vdev, 1,
 	    &spa->spa_ubsync, ZIO_FLAG_CONFIG_WRITER));
 	if (spa_multihost(spa))
 		mmp_update_uberblock(spa, &spa->spa_ubsync);
@@ -4804,7 +4804,7 @@ overwrite:
 	RAIDZ_REFLOW_SET(&spa->spa_ubsync, RRSS_SCRATCH_INVALID_SYNCED,
 	    logical_size);
 	spa->spa_ubsync.ub_timestamp++;
-	ASSERT0(vdev_uberblock_sync_list(&spa->spa_root_vdev, 1,
+	ASSERT0(vdev_uberblock_sync_list(spa, &spa->spa_root_vdev, 1,
 	    &spa->spa_ubsync, ZIO_FLAG_CONFIG_WRITER));
 	if (spa_multihost(spa))
 		mmp_update_uberblock(spa, &spa->spa_ubsync);
@@ -4905,7 +4905,7 @@ vdev_raidz_reflow_copy_scratch(spa_t *spa)
 	RAIDZ_REFLOW_SET(&spa->spa_ubsync,
 	    RRSS_SCRATCH_INVALID_SYNCED_ON_IMPORT, logical_size);
 	spa->spa_ubsync.ub_timestamp++;
-	VERIFY0(vdev_uberblock_sync_list(&spa->spa_root_vdev, 1,
+	VERIFY0(vdev_uberblock_sync_list(spa, &spa->spa_root_vdev, 1,
 	    &spa->spa_ubsync, ZIO_FLAG_CONFIG_WRITER));
 	if (spa_multihost(spa))
 		mmp_update_uberblock(spa, &spa->spa_ubsync);