Rate limit Direct I/O verify zevents

Each vdev initializes a vdev_dio_verify_rl rate limiter (governed by
zfs_dio_write_verify_events_per_second), but
zio_dio_chksum_verify_error_report() never consults it, so
dio_verify_rd and dio_verify_wr zevents are posted with no rate
limiting.  A workload that repeatedly trips the Direct I/O verify can
therefore produce an unbounded flood of zevents.

Gate both ereport posts through zfs_ratelimit(&vd->vdev_dio_verify_rl),
as is already done for the other per-vdev ereports (checksum, delay,
deadman).  The vs_dio_verify_errors vdev stat still increments on every
event, so the true count remains observable via zpool status -d.

The dio_write_verify test checks on every iteration that a
dio_verify_wr zevent was posted.  With rate limiting now in effect the
shared limiter window is exhausted after the first iteration, so later
iterations observe zero events and the test fails.  Raise
zfs_dio_write_verify_events_per_second for the duration of that test
(restored in cleanup), adding the matching tunables.cfg alias, so the
events stay observable.

Reviewed-by: Brian Behlendorf <[email protected]>
Signed-off-by: Michael Heller <[email protected]>
Closes #18795

module/zfs/zio.c

@@ -5355,16 +5355,18 @@ zio_dio_chksum_verify_error_report(zio_t *zio)
 		 */
 		zio->io_error = SET_ERROR(EIO);
 		/*
-		 * Report dio_verify_wr ZED event.
+		 * Report dio_verify_wr ZED event, rate limited.
 		 */
-		(void) zfs_ereport_post(FM_EREPORT_ZFS_DIO_VERIFY_WR,
-		    zio->io_spa,  zio->io_vd, &zio->io_bookmark, zio, 0);
+		if (zfs_ratelimit(&zio->io_vd->vdev_dio_verify_rl))
+			(void) zfs_ereport_post(FM_EREPORT_ZFS_DIO_VERIFY_WR,
+			    zio->io_spa, zio->io_vd, &zio->io_bookmark, zio, 0);
 	} else {
 		/*
-		 * Report dio_verify_rd ZED event.
+		 * Report dio_verify_rd ZED event, rate limited.
 		 */
-		(void) zfs_ereport_post(FM_EREPORT_ZFS_DIO_VERIFY_RD,
-		    zio->io_spa, zio->io_vd, &zio->io_bookmark, zio, 0);
+		if (zfs_ratelimit(&zio->io_vd->vdev_dio_verify_rl))
+			(void) zfs_ereport_post(FM_EREPORT_ZFS_DIO_VERIFY_RD,
+			    zio->io_spa, zio->io_vd, &zio->io_bookmark, zio, 0);
 	}
 }
 

tests/zfs-tests/include/tunables.cfg

@@ -39,6 +39,7 @@ DEADMAN_EVENTS_PER_SECOND	deadman_events_per_second	zfs_deadman_events_per_secon
 DEADMAN_FAILMODE		deadman.failmode		zfs_deadman_failmode
 DEADMAN_SYNCTIME_MS		deadman.synctime_ms		zfs_deadman_synctime_ms
 DEADMAN_ZIOTIME_MS		deadman.ziotime_ms		zfs_deadman_ziotime_ms
+DIO_WRITE_VERIFY_EVENTS_PER_SECOND	dio_write_verify_events_per_second	zfs_dio_write_verify_events_per_second
 DISABLE_IVSET_GUID_CHECK	disable_ivset_guid_check	zfs_disable_ivset_guid_check
 DMU_OFFSET_NEXT_SYNC		dmu_offset_next_sync		zfs_dmu_offset_next_sync
 EMBEDDED_SLOG_MIN_MS		embedded_slog_min_ms		zfs_embedded_slog_min_ms

tests/zfs-tests/tests/functional/direct/dio_write_verify.ksh

@@ -60,6 +60,8 @@ function cleanup
 	# Clearing out dio_verify from event logs
 	log_must zpool events -c
 	log_must set_tunable32 VDEV_DIRECT_WR_VERIFY $DIO_WR_VERIFY_TUNABLE
+	log_must set_tunable32 DIO_WRITE_VERIFY_EVENTS_PER_SECOND \
+	    $DIO_WR_EVENTS_TUNABLE
 }
 
 log_assert "Verify checksum verify works for Direct I/O writes."
@@ -75,6 +77,16 @@ NUMBLOCKS=300
 BS=$((128 * 1024)) # 128k
 mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS)
 typeset DIO_WR_VERIFY_TUNABLE=$(get_tunable VDEV_DIRECT_WR_VERIFY)
+typeset DIO_WR_EVENTS_TUNABLE=$(get_tunable DIO_WRITE_VERIFY_EVENTS_PER_SECOND)
+
+#
+# Each iteration below generates Direct I/O write verify failures and checks
+# that dio_verify_wr zevents were posted.  Those zevents are rate limited by
+# zfs_dio_write_verify_events_per_second and the limiter state persists across
+# iterations, so raise the limit while the test runs to avoid dropping the
+# events being counted.  The original value is restored in cleanup.
+#
+log_must set_tunable32 DIO_WRITE_VERIFY_EVENTS_PER_SECOND 1000000
 
 # Get a list of vdevs in our pool
 set -A array $(get_disklist_fullpath $TESTPOOL)