Change checksum & IO delay ratelimit values

Change checksum & IO delay ratelimit thresholds from 5/sec to 20/sec.
This allows zed to actually trigger if a bunch of these events arrive in
a short period of time (zed has a threshold of 10 events in 10 sec).
Previously, if you had, say, 100 checksum errors in 1 sec, it would get
ratelimited to 5/sec which wouldn't trigger zed to fault the drive.

Also, convert the checksum and IO delay thresholds to module params for
easy testing.

Reviewed-by: loli10K <ezomori.nozomu@gmail.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Giuseppe Di Natale <dinatale2@llnl.gov>
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
Closes #7252

diff --git a/include/sys/vdev_impl.h b/include/sys/vdev_impl.h
index 13c495822535999fdf0a9e7d45d33e6581378a59..4f9f1a903b5a21c6db1dd935ec8ff89dec629e22 100644 (file)
--- a/include/sys/vdev_impl.h
+++ b/include/sys/vdev_impl.h
@@ -255,8 +255,6 @@ struct vdev {
         * We rate limit ZIO delay and ZIO checksum events, since they
         * can flood ZED with tons of events when a drive is acting up.
         */
-#define        DELAYS_PER_SECOND 5
-#define        CHECKSUMS_PER_SECOND 5
        zfs_ratelimit_t vdev_delay_rl;
        zfs_ratelimit_t vdev_checksum_rl;
 };

diff --git a/include/sys/zfs_ratelimit.h b/include/sys/zfs_ratelimit.h
index f36e07841f9284f53b06d80c9476fc614432e809..012825fadb223a7bd687c628b49eba0873f6b77b 100644 (file)
--- a/include/sys/zfs_ratelimit.h
+++ b/include/sys/zfs_ratelimit.h
@@ -25,13 +25,19 @@
 typedef struct {
        hrtime_t start;
        unsigned int count;
-       unsigned int burst;             /* Number to allow per interval */
-       unsigned int interval;          /* Interval length in seconds */
+
+       /*
+        * Pointer to number of events per interval.  We do this to
+        * allow the burst to be a (changeable) module parameter.
+        */
+       unsigned int *burst;
+
+       unsigned int interval;  /* Interval length in seconds */
        kmutex_t lock;
 } zfs_ratelimit_t;
 
 int zfs_ratelimit(zfs_ratelimit_t *rl);
-void zfs_ratelimit_init(zfs_ratelimit_t *rl, unsigned int burst,
+void zfs_ratelimit_init(zfs_ratelimit_t *rl, unsigned int *burst,
     unsigned int interval);
 void zfs_ratelimit_fini(zfs_ratelimit_t *rl);
 

diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5
index d4daffde6bc2dc59da1ad3176584fac4a0f8dea3..8d5ac2576141d3cf68829a0c0f9b0d336cebc413 100644 (file)
--- a/man/man5/zfs-module-parameters.5
+++ b/man/man5/zfs-module-parameters.5
@@ -739,6 +739,34 @@ Disable pool import at module load by ignoring the cache file (typically \fB/etc
 Use \fB1\fR for yes (default) and \fB0\fR for no.
 .RE
 
+.sp
+.ne 2
+.na
+\fBzfs_checksums_per_second\fR (int)
+.ad
+.RS 12n
+Rate limit checksum events to this many per second.  Note that this should
+not be set below the zed thresholds (currently 10 checksums over 10 sec)
+or else zed may not trigger any action.
+.sp
+Default value: 20
+.RE
+
+.sp
+.ne 2
+.na
+\fBzfs_commit_timeout_pct\fR (int)
+.ad
+.RS 12n
+This controls the amount of time that a ZIL block (lwb) will remain "open"
+when it isn't "full", and it has a thread waiting for it to be committed to
+stable storage.  The timeout is scaled based on a percentage of the last lwb
+latency to avoid significantly impacting the latency of each individual
+transaction record (itx).
+.sp
+Default value: \fB5\fR%.
+.RE
+
 .sp
 .ne 2
 .na
@@ -866,6 +894,17 @@ Note: \fBzfs_delay_scale\fR * \fBzfs_dirty_data_max\fR must be < 2^64.
 Default value: \fB500,000\fR.
 .RE
 
+.sp
+.ne 2
+.na
+\fBzfs_delays_per_second\fR (int)
+.ad
+.RS 12n
+Rate limit IO delay events to this many per second.
+.sp
+Default value: 20
+.RE
+
 .sp
 .ne 2
 .na

diff --git a/module/zcommon/zfs_comutil.c b/module/zcommon/zfs_comutil.c
index 52cb7e36555906183295358db9770cad17caa734..44cdc852345c50a2e78169330ebc4766ab3a50ef 100644 (file)
--- a/module/zcommon/zfs_comutil.c
+++ b/module/zcommon/zfs_comutil.c
@@ -215,7 +215,7 @@ const char *zfs_history_event_names[ZFS_NUM_LEGACY_HISTORY_EVENTS] = {
  * interval:   Interval time in seconds
  */
 void
-zfs_ratelimit_init(zfs_ratelimit_t *rl, unsigned int burst,
+zfs_ratelimit_init(zfs_ratelimit_t *rl, unsigned int *burst,
     unsigned int interval)
 {
        rl->count = 0;
@@ -270,7 +270,7 @@ zfs_ratelimit(zfs_ratelimit_t *rl)
                rl->start = now;
                rl->count = 0;
        } else {
-               if (rl->count >= rl->burst) {
+               if (rl->count >= *rl->burst) {
                        rc = 0; /* We're ratelimiting */
                }
        }

diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c
index df07d893dba2eee2d93331346862fa47cf8854e3..0786fbb834d753fd6e6bcfbd9588186fb0d0ea19 100644 (file)
--- a/module/zfs/vdev.c
+++ b/module/zfs/vdev.c
@@ -56,6 +56,16 @@
  */
 int metaslabs_per_vdev = 200;
 
+/*
+ * Rate limit delay events to this many IO delays per second.
+ */
+unsigned int zfs_delays_per_second = 20;
+
+/*
+ * Rate limit checksum events after this many checksum errors per second.
+ */
+unsigned int zfs_checksums_per_second = 20;
+
 /*
  * Virtual device management.
  */
@@ -357,8 +367,8 @@ vdev_alloc_common(spa_t *spa, uint_t id, uint64_t guid, vdev_ops_t *ops)
         * and checksum events so that we don't overwhelm ZED with thousands
         * of events when a disk is acting up.
         */
-       zfs_ratelimit_init(&vd->vdev_delay_rl, DELAYS_PER_SECOND, 1);
-       zfs_ratelimit_init(&vd->vdev_checksum_rl, CHECKSUMS_PER_SECOND, 1);
+       zfs_ratelimit_init(&vd->vdev_delay_rl, &zfs_delays_per_second, 1);
+       zfs_ratelimit_init(&vd->vdev_checksum_rl, &zfs_checksums_per_second, 1);
 
        list_link_init(&vd->vdev_config_dirty_node);
        list_link_init(&vd->vdev_state_dirty_node);
@@ -3776,5 +3786,14 @@ module_param(metaslabs_per_vdev, int, 0644);
 MODULE_PARM_DESC(metaslabs_per_vdev,
        "Divide added vdev into approximately (but no more than) this number "
        "of metaslabs");
+
+module_param(zfs_delays_per_second, uint, 0644);
+MODULE_PARM_DESC(zfs_delays_per_second, "Rate limit delay events to this many "
+       "IO delays per second");
+
+module_param(zfs_checksums_per_second, uint, 0644);
+       MODULE_PARM_DESC(zfs_checksums_per_second, "Rate limit checksum events "
+       "to this many checksum errors per second (do not set below zed"
+       "threshold).");
 /* END CSTYLED */
 #endif