Add corruption failure option to zinject(8)

Added a 'corrupt' error option that will flip a bit in the data
after a read operation.  This is useful for generating checksum
errors at the device layer (in a mirror config for example). It
is also used to validate the diagnosis of checksum errors from
the zfs diagnosis engine.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Don Brady <don.brady@intel.com>
Closes #6345

diff --git a/cmd/zinject/zinject.c b/cmd/zinject/zinject.c
index ccd3534d05798c406d153ff2e5cf2a4fbf4e2754..d66a959bc4dc3b0948014a20adf75af54d83ce77 100644 (file)
--- a/cmd/zinject/zinject.c
+++ b/cmd/zinject/zinject.c
@@ -36,12 +36,15 @@
  *
  * Errors can be injected into a particular vdev using the '-d' option.  This
  * option takes a path or vdev GUID to uniquely identify the device within a
- * pool.  There are two types of errors that can be injected, EIO and ENXIO,
- * that can be controlled through the '-e' option.  The default is ENXIO.  For
- * EIO failures, any attempt to read data from the device will return EIO, but
- * subsequent attempt to reopen the device will succeed.  For ENXIO failures,
- * any attempt to read from the device will return EIO, but any attempt to
- * reopen the device will also return ENXIO.
+ * pool.  There are four types of errors that can be injected, IO, ENXIO,
+ * ECHILD, and EILSEQ.  These can be controlled through the '-e' option and the
+ * default is ENXIO.  For EIO failures, any attempt to read data from the device
+ * will return EIO, but a subsequent attempt to reopen the device will succeed.
+ * For ENXIO failures, any attempt to read from the device will return EIO, but
+ * any attempt to reopen the device will also return ENXIO.  The EILSEQ failures
+ * only apply to read operations (-T read) and will flip a bit after the device
+ * has read the original data.
+ *
  * For label faults, the -L option must be specified. This allows faults
  * to be injected into either the nvlist, uberblock, pad1, or pad2 region
  * of all the labels for the specified device.
@@ -231,11 +234,12 @@ usage(void)
            "\t\tspa_vdev_exit() will trigger a panic.\n"
            "\n"
            "\tzinject -d device [-e errno] [-L <nvlist|uber|pad1|pad2>] [-F]\n"
-           "\t    [-T <read|write|free|claim|all>] [-f frequency] pool\n"
+           "\t\t[-T <read|write|free|claim|all>] [-f frequency] pool\n\n"
            "\t\tInject a fault into a particular device or the device's\n"
            "\t\tlabel.  Label injection can either be 'nvlist', 'uber',\n "
            "\t\t'pad1', or 'pad2'.\n"
-           "\t\t'errno' can be 'nxio' (the default), 'io', or 'dtl'.\n"
+           "\t\t'errno' can be 'nxio' (the default), 'io', 'dtl', or\n"
+           "\t\t'corrupt' (bit flip).\n"
            "\t\t'frequency' is a value between 0.0001 and 100.0 that limits\n"
            "\t\tdevice error injection to a percentage of the IOs.\n"
            "\n"
@@ -774,6 +778,8 @@ main(int argc, char **argv)
                                error = ENXIO;
                        } else if (strcasecmp(optarg, "dtl") == 0) {
                                error = ECHILD;
+                       } else if (strcasecmp(optarg, "corrupt") == 0) {
+                               error = EILSEQ;
                        } else {
                                (void) fprintf(stderr, "invalid error type "
                                    "'%s': must be 'io', 'checksum' or "
@@ -981,7 +987,15 @@ main(int argc, char **argv)
 
                if (error == ECKSUM) {
                        (void) fprintf(stderr, "device error type must be "
-                           "'io' or 'nxio'\n");
+                           "'io', 'nxio' or 'corrupt'\n");
+                       libzfs_fini(g_zfs);
+                       return (1);
+               }
+
+               if (error == EILSEQ &&
+                   (record.zi_freq == 0 || io_type != ZIO_TYPE_READ)) {
+                       (void) fprintf(stderr, "device corrupt errors require "
+                           "io type read and a frequency value\n");
                        libzfs_fini(g_zfs);
                        return (1);
                }
@@ -1109,7 +1123,7 @@ main(int argc, char **argv)
                        return (2);
                }
 
-               if (error == ENXIO) {
+               if (error == ENXIO || error == EILSEQ) {
                        (void) fprintf(stderr, "data error type must be "
                            "'checksum' or 'io'\n");
                        libzfs_fini(g_zfs);

diff --git a/include/sys/zio.h b/include/sys/zio.h
index f7baa270bdeda8429909d4d812cb550036eb06c9..30c4ee3ce4c3e46d6ba86457ece6512cdd253a0c 100644 (file)
--- a/include/sys/zio.h
+++ b/include/sys/zio.h
@@ -619,6 +619,8 @@ extern int zio_clear_fault(int id);
 extern void zio_handle_panic_injection(spa_t *spa, char *tag, uint64_t type);
 extern int zio_handle_fault_injection(zio_t *zio, int error);
 extern int zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error);
+extern int zio_handle_device_injections(vdev_t *vd, zio_t *zio, int err1,
+    int err2);
 extern int zio_handle_label_injection(zio_t *zio, int error);
 extern void zio_handle_ignored_writes(zio_t *zio);
 extern hrtime_t zio_handle_io_delay(zio_t *zio);

diff --git a/man/man8/zinject.8 b/man/man8/zinject.8
index 50fecfb64364578eb13be4e117e285e7a8b388f4..6b6c3733ba0048f361cab6b4ab385209e463014c 100644 (file)
--- a/man/man8/zinject.8
+++ b/man/man8/zinject.8
@@ -108,6 +108,7 @@ A vdev specified by path or GUID.
 .BI "\-e" " device_error"
 Specify
 .BR "checksum" " for an ECKSUM error,"
+.BR "corrupt" " to flip a bit in the data after a read,"
 .BR "dtl" " for an ECHILD error,"
 .BR "io" " for an EIO error where reopening the device will succeed, or"
 .BR "nxio" " for an ENXIO error where reopening the device will fail."

diff --git a/module/zfs/zio.c b/module/zfs/zio.c
index 959b9a5a8fe8f992c4ad567d62fb808460d2d831..057a1405f5f56786839f15909d005736a846c876 100644 (file)
--- a/module/zfs/zio.c
+++ b/module/zfs/zio.c
@@ -3472,8 +3472,8 @@ zio_vdev_io_done(zio_t *zio)
                        vdev_cache_write(zio);
 
                if (zio_injection_enabled && zio->io_error == 0)
-                       zio->io_error = zio_handle_device_injection(vd,
-                           zio, EIO);
+                       zio->io_error = zio_handle_device_injections(vd, zio,
+                           EIO, EILSEQ);
 
                if (zio_injection_enabled && zio->io_error == 0)
                        zio->io_error = zio_handle_label_injection(zio, EIO);

diff --git a/module/zfs/zio_inject.c b/module/zfs/zio_inject.c
index 4a4d431e33bcff22d66a28e4ea23c6a919e1ac0b..e1ea825d7bf09cde4b719eed1a29c564e0ac1e5c 100644 (file)
--- a/module/zfs/zio_inject.c
+++ b/module/zfs/zio_inject.c
@@ -271,9 +271,24 @@ zio_handle_label_injection(zio_t *zio, int error)
        return (ret);
 }
 
+/*ARGSUSED*/
+static int
+zio_inject_bitflip_cb(void *data, size_t len, void *private)
+{
+       ASSERTV(zio_t *zio = private);
+       uint8_t *buffer = data;
+       uint_t byte = spa_get_random(len);
 
-int
-zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
+       ASSERT(zio->io_type == ZIO_TYPE_READ);
+
+       /* flip a single random bit in an abd data buffer */
+       buffer[byte] ^= 1 << spa_get_random(8);
+
+       return (1);     /* stop after first flip */
+}
+
+static int
+zio_handle_device_injection_impl(vdev_t *vd, zio_t *zio, int err1, int err2)
 {
        inject_handler_t *handler;
        int ret = 0;
@@ -311,7 +326,8 @@ zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
                            handler->zi_record.zi_iotype != zio->io_type)
                                continue;
 
-                       if (handler->zi_record.zi_error == error) {
+                       if (handler->zi_record.zi_error == err1 ||
+                           handler->zi_record.zi_error == err2) {
                                /*
                                 * limit error injection if requested
                                 */
@@ -322,7 +338,7 @@ zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
                                 * For a failed open, pretend like the device
                                 * has gone away.
                                 */
-                               if (error == ENXIO)
+                               if (err1 == ENXIO)
                                        vd->vdev_stat.vs_aux =
                                            VDEV_AUX_OPEN_FAILED;
 
@@ -335,7 +351,21 @@ zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
                                    zio != NULL)
                                        zio->io_flags |= ZIO_FLAG_IO_RETRY;
 
-                               ret = error;
+                               /*
+                                * EILSEQ means flip a bit after a read
+                                */
+                               if (handler->zi_record.zi_error == EILSEQ) {
+                                       if (zio == NULL)
+                                               break;
+
+                                       /* locate buffer data and flip a bit */
+                                       (void) abd_iterate_func(zio->io_abd, 0,
+                                           zio->io_size, zio_inject_bitflip_cb,
+                                           zio);
+                                       break;
+                               }
+
+                               ret = handler->zi_record.zi_error;
                                break;
                        }
                        if (handler->zi_record.zi_error == ENXIO) {
@@ -350,6 +380,18 @@ zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
        return (ret);
 }
 
+int
+zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
+{
+       return (zio_handle_device_injection_impl(vd, zio, error, INT_MAX));
+}
+
+int
+zio_handle_device_injections(vdev_t *vd, zio_t *zio, int err1, int err2)
+{
+       return (zio_handle_device_injection_impl(vd, zio, err1, err2));
+}
+
 /*
  * Simulate hardware that ignores cache flushes.  For requested number
  * of seconds nix the actual writing to disk.