]> granicus.if.org Git - zfs/blobdiff - cmd/zpool/zpool_main.c
OpenZFS 9166 - zfs storage pool checkpoint
[zfs] / cmd / zpool / zpool_main.c
index 1d31fde469a53b30a2af225d2bfc4af33313355c..5aca0d32594a398db81faeec115d44f6bd55c7b6 100644 (file)
@@ -36,6 +36,7 @@
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <getopt.h>
 #include <libgen.h>
 #include <libintl.h>
 #include <libuutil.h>
@@ -76,6 +77,8 @@ static int zpool_do_add(int, char **);
 static int zpool_do_remove(int, char **);
 static int zpool_do_labelclear(int, char **);
 
+static int zpool_do_checkpoint(int, char **);
+
 static int zpool_do_list(int, char **);
 static int zpool_do_iostat(int, char **);
 static int zpool_do_status(int, char **);
@@ -131,6 +134,7 @@ typedef enum {
        HELP_ATTACH,
        HELP_CLEAR,
        HELP_CREATE,
+       HELP_CHECKPOINT,
        HELP_DESTROY,
        HELP_DETACH,
        HELP_EXPORT,
@@ -254,6 +258,8 @@ static zpool_command_t command_table[] = {
        { NULL },
        { "labelclear", zpool_do_labelclear,    HELP_LABELCLEAR         },
        { NULL },
+       { "checkpoint", zpool_do_checkpoint,    HELP_CHECKPOINT         },
+       { NULL },
        { "list",       zpool_do_list,          HELP_LIST               },
        { "iostat",     zpool_do_iostat,        HELP_IOSTAT             },
        { "status",     zpool_do_status,        HELP_STATUS             },
@@ -306,6 +312,8 @@ get_usage(zpool_help_t idx)
                return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
                    "\t    [-O file-system-property=value] ... \n"
                    "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
+       case HELP_CHECKPOINT:
+               return (gettext("\tcheckpoint [--discard] <pool> ...\n"));
        case HELP_DESTROY:
                return (gettext("\tdestroy [-f] <pool>\n"));
        case HELP_DETACH:
@@ -316,15 +324,13 @@ get_usage(zpool_help_t idx)
                return (gettext("\thistory [-il] [<pool>] ...\n"));
        case HELP_IMPORT:
                return (gettext("\timport [-d dir] [-D]\n"
-                   "\timport [-d dir | -c cachefile] [-F [-n]] [-l] "
-                   "<pool | id>\n"
                    "\timport [-o mntopts] [-o property=value] ... \n"
                    "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
                    "[-R root] [-F [-n]] -a\n"
                    "\timport [-o mntopts] [-o property=value] ... \n"
                    "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
                    "[-R root] [-F [-n]]\n"
-                   "\t    <pool | id> [newpool]\n"));
+                   "\t    [--rewind-to-checkpoint] <pool | id> [newpool]\n"));
        case HELP_IOSTAT:
                return (gettext("\tiostat [[[-c [script1,script2,...]"
                    "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
@@ -344,7 +350,7 @@ get_usage(zpool_help_t idx)
                return (gettext("\treplace [-f] [-o property=value] "
                    "<pool> <device> [new-device]\n"));
        case HELP_REMOVE:
-               return (gettext("\tremove <pool> <device> ...\n"));
+               return (gettext("\tremove [-nps] <pool> <device> ...\n"));
        case HELP_REOPEN:
                return (gettext("\treopen [-n] <pool>\n"));
        case HELP_SCRUB:
@@ -782,8 +788,7 @@ zpool_do_add(int argc, char **argv)
 /*
  * zpool remove  <pool> <vdev> ...
  *
- * Removes the given vdev from the pool.  Currently, this supports removing
- * spares, cache, and log devices from the pool.
+ * Removes the given vdev from the pool.
  */
 int
 zpool_do_remove(int argc, char **argv)
@@ -791,28 +796,87 @@ zpool_do_remove(int argc, char **argv)
        char *poolname;
        int i, ret = 0;
        zpool_handle_t *zhp = NULL;
+       boolean_t stop = B_FALSE;
+       char c;
+       boolean_t noop = B_FALSE;
+       boolean_t parsable = B_FALSE;
 
-       argc--;
-       argv++;
+       /* check options */
+       while ((c = getopt(argc, argv, "nps")) != -1) {
+               switch (c) {
+               case 'n':
+                       noop = B_TRUE;
+                       break;
+               case 'p':
+                       parsable = B_TRUE;
+                       break;
+               case 's':
+                       stop = B_TRUE;
+                       break;
+               case '?':
+                       (void) fprintf(stderr, gettext("invalid option '%c'\n"),
+                           optopt);
+                       usage(B_FALSE);
+               }
+       }
+
+       argc -= optind;
+       argv += optind;
 
        /* get pool name and check number of arguments */
        if (argc < 1) {
                (void) fprintf(stderr, gettext("missing pool name argument\n"));
                usage(B_FALSE);
        }
-       if (argc < 2) {
-               (void) fprintf(stderr, gettext("missing device\n"));
-               usage(B_FALSE);
-       }
 
        poolname = argv[0];
 
        if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
                return (1);
 
-       for (i = 1; i < argc; i++) {
-               if (zpool_vdev_remove(zhp, argv[i]) != 0)
+       if (stop && noop) {
+               (void) fprintf(stderr, gettext("stop request ignored\n"));
+               return (0);
+       }
+
+       if (stop) {
+               if (argc > 1) {
+                       (void) fprintf(stderr, gettext("too many arguments\n"));
+                       usage(B_FALSE);
+               }
+               if (zpool_vdev_remove_cancel(zhp) != 0)
                        ret = 1;
+       } else {
+               if (argc < 2) {
+                       (void) fprintf(stderr, gettext("missing device\n"));
+                       usage(B_FALSE);
+               }
+
+               for (i = 1; i < argc; i++) {
+                       if (noop) {
+                               uint64_t size;
+
+                               if (zpool_vdev_indirect_size(zhp, argv[i],
+                                   &size) != 0) {
+                                       ret = 1;
+                                       break;
+                               }
+                               if (parsable) {
+                                       (void) printf("%s %llu\n",
+                                           argv[i], (unsigned long long)size);
+                               } else {
+                                       char valstr[32];
+                                       zfs_nicenum(size, valstr,
+                                           sizeof (valstr));
+                                       (void) printf("Memory that will be "
+                                           "used after removing %s: %s\n",
+                                           argv[i], valstr);
+                               }
+                       } else {
+                               if (zpool_vdev_remove(zhp, argv[i]) != 0)
+                                       ret = 1;
+                       }
+               }
        }
        zpool_close(zhp);
 
@@ -1655,6 +1719,7 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
        uint64_t notpresent;
        spare_cbdata_t spare_cb;
        const char *state;
+       char *type;
        char *path = NULL;
 
        if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
@@ -1664,6 +1729,11 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
        verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
            (uint64_t **)&vs, &c) == 0);
 
+       verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
+
+       if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
+               return;
+
        state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
        if (isspare) {
                /*
@@ -1756,6 +1826,10 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
                        (void) printf(gettext("currently in use"));
                        break;
 
+               case VDEV_AUX_CHILDREN_OFFLINE:
+                       (void) printf(gettext("all children offline"));
+                       break;
+
                default:
                        (void) printf(gettext("corrupted data"));
                        break;
@@ -1855,6 +1929,10 @@ print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
                        (void) printf(gettext("currently in use"));
                        break;
 
+               case VDEV_AUX_CHILDREN_OFFLINE:
+                       (void) printf(gettext("all children offline"));
+                       break;
+
                default:
                        (void) printf(gettext("corrupted data"));
                        break;
@@ -2381,6 +2459,79 @@ do_import(nvlist_t *config, const char *newname, const char *mntopts,
        return (ret);
 }
 
+/*
+ * zpool checkpoint <pool>
+ *       checkpoint --discard <pool>
+ *
+ *       -d         Discard the checkpoint from a checkpointed
+ *       --discard  pool.
+ *
+ * Checkpoints the specified pool, by taking a "snapshot" of its
+ * current state. A pool can only have one checkpoint at a time.
+ */
+int
+zpool_do_checkpoint(int argc, char **argv)
+{
+       boolean_t discard;
+       char *pool;
+       zpool_handle_t *zhp;
+       int c, err;
+
+       struct option long_options[] = {
+               {"discard", no_argument, NULL, 'd'},
+               {0, 0, 0, 0}
+       };
+
+       discard = B_FALSE;
+       while ((c = getopt_long(argc, argv, ":d", long_options, NULL)) != -1) {
+               switch (c) {
+               case 'd':
+                       discard = B_TRUE;
+                       break;
+               case '?':
+                       (void) fprintf(stderr, gettext("invalid option '%c'\n"),
+                           optopt);
+                       usage(B_FALSE);
+               }
+       }
+
+       argc -= optind;
+       argv += optind;
+
+       if (argc < 1) {
+               (void) fprintf(stderr, gettext("missing pool argument\n"));
+               usage(B_FALSE);
+       }
+
+       if (argc > 1) {
+               (void) fprintf(stderr, gettext("too many arguments\n"));
+               usage(B_FALSE);
+       }
+
+       pool = argv[0];
+
+       if ((zhp = zpool_open(g_zfs, pool)) == NULL) {
+               /* As a special case, check for use of '/' in the name */
+               if (strchr(pool, '/') != NULL)
+                       (void) fprintf(stderr, gettext("'zpool checkpoint' "
+                           "doesn't work on datasets. To save the state "
+                           "of a dataset from a specific point in time "
+                           "please use 'zfs snapshot'\n"));
+               return (1);
+       }
+
+       if (discard)
+               err = (zpool_discard_checkpoint(zhp) != 0);
+       else
+               err = (zpool_checkpoint(zhp) != 0);
+
+       zpool_close(zhp);
+
+       return (err);
+}
+
+#define        CHECKPOINT_OPT  1024
+
 /*
  * zpool import [-d dir] [-D]
  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
@@ -2427,6 +2578,9 @@ do_import(nvlist_t *config, const char *newname, const char *mntopts,
  *      -s     Scan using the default search path, the libblkid cache will
  *             not be consulted.
  *
+ *       --rewind-to-checkpoint
+ *             Import the pool and revert back to the checkpoint.
+ *
  * The import command scans for pools to import, and import pools based on pool
  * name and GUID.  The pool can also be renamed as part of the import process.
  */
@@ -2462,8 +2616,14 @@ zpool_do_import(int argc, char **argv)
        importargs_t idata = { 0 };
        char *endptr;
 
+       struct option long_options[] = {
+               {"rewind-to-checkpoint", no_argument, NULL, CHECKPOINT_OPT},
+               {0, 0, 0, 0}
+       };
+
        /* check options */
-       while ((c = getopt(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX")) != -1) {
+       while ((c = getopt_long(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX",
+           long_options, NULL)) != -1) {
                switch (c) {
                case 'a':
                        do_all = B_TRUE;
@@ -2550,6 +2710,9 @@ zpool_do_import(int argc, char **argv)
                case 'X':
                        xtreme_rewind = B_TRUE;
                        break;
+               case CHECKPOINT_OPT:
+                       flags |= ZFS_IMPORT_CHECKPOINT;
+                       break;
                case ':':
                        (void) fprintf(stderr, gettext("missing argument for "
                            "'%c' option\n"), optopt);
@@ -2595,8 +2758,9 @@ zpool_do_import(int argc, char **argv)
 
        /* In the future, we can capture further policy and include it here */
        if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
-           nvlist_add_uint64(policy, ZPOOL_REWIND_REQUEST_TXG, txg) != 0 ||
-           nvlist_add_uint32(policy, ZPOOL_REWIND_REQUEST, rewind_policy) != 0)
+           nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 ||
+           nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
+           rewind_policy) != 0)
                goto error;
 
        /* check argument count */
@@ -2688,6 +2852,7 @@ zpool_do_import(int argc, char **argv)
        idata.guid = searchguid;
        idata.cachefile = cachefile;
        idata.scan = do_scan;
+       idata.policy = policy;
 
        pools = zpool_search_import(g_zfs, &idata);
 
@@ -2746,7 +2911,7 @@ zpool_do_import(int argc, char **argv)
                if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
                        continue;
 
-               verify(nvlist_add_nvlist(config, ZPOOL_REWIND_POLICY,
+               verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
                    policy) == 0);
 
                if (argc == 0) {
@@ -3668,6 +3833,9 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
 
        calcvs = safe_malloc(sizeof (*calcvs));
 
+       if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
+               return (ret);
+
        if (oldnv != NULL) {
                verify(nvlist_lookup_uint64_array(oldnv,
                    ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
@@ -4900,6 +5068,7 @@ print_one_column(zpool_prop_t prop, uint64_t value, boolean_t scripted,
 
        switch (prop) {
        case ZPOOL_PROP_EXPANDSZ:
+       case ZPOOL_PROP_CHECKPOINT:
                if (value == 0)
                        (void) strlcpy(propval, "-", sizeof (propval));
                else
@@ -4964,6 +5133,9 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
                else
                        format = ZFS_NICENUM_1024;
 
+               if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
+                       return;
+
                if (scripted)
                        (void) printf("\t%s", name);
                else if (strlen(name) + depth > cb->cb_namewidth)
@@ -4984,6 +5156,8 @@ print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
                    toplevel, format);
                print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
                    scripted, toplevel, format);
+               print_one_column(ZPOOL_PROP_CHECKPOINT,
+                   vs->vs_checkpoint_space, scripted, toplevel, format);
                print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, scripted,
                    B_TRUE, format);
                print_one_column(ZPOOL_PROP_FRAGMENTATION,
@@ -5107,8 +5281,8 @@ zpool_do_list(int argc, char **argv)
        int ret = 0;
        list_cbdata_t cb = { 0 };
        static char default_props[] =
-           "name,size,allocated,free,expandsize,fragmentation,capacity,"
-           "dedupratio,health,altroot";
+           "name,size,allocated,free,checkpoint,expandsize,fragmentation,"
+           "capacity,dedupratio,health,altroot";
        char *props = default_props;
        float interval = 0;
        unsigned long count = 0;
@@ -5799,8 +5973,10 @@ zpool_do_clear(int argc, char **argv)
 
        /* In future, further rewind policy choices can be passed along here */
        if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
-           nvlist_add_uint32(policy, ZPOOL_REWIND_REQUEST, rewind_policy) != 0)
+           nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
+           rewind_policy) != 0) {
                return (1);
+       }
 
        pool = argv[0];
        device = argc == 2 ? argv[1] : NULL;
@@ -5908,6 +6084,32 @@ typedef struct scrub_cbdata {
        pool_scrub_cmd_t cb_scrub_cmd;
 } scrub_cbdata_t;
 
+static boolean_t
+zpool_has_checkpoint(zpool_handle_t *zhp)
+{
+       nvlist_t *config, *nvroot;
+
+       config = zpool_get_config(zhp, NULL);
+
+       if (config != NULL) {
+               pool_checkpoint_stat_t *pcs = NULL;
+               uint_t c;
+
+               nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
+               (void) nvlist_lookup_uint64_array(nvroot,
+                   ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
+
+               if (pcs == NULL || pcs->pcs_state == CS_NONE)
+                       return (B_FALSE);
+
+               assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS ||
+                   pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
+               return (B_TRUE);
+       }
+
+       return (B_FALSE);
+}
+
 int
 scrub_callback(zpool_handle_t *zhp, void *data)
 {
@@ -5925,6 +6127,13 @@ scrub_callback(zpool_handle_t *zhp, void *data)
 
        err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
 
+       if (err == 0 && zpool_has_checkpoint(zhp) &&
+           cb->cb_type == POOL_SCAN_SCRUB) {
+               (void) printf(gettext("warning: will not scrub state that "
+                   "belongs to the checkpoint of pool '%s'\n"),
+                   zpool_get_name(zhp));
+       }
+
        return (err != 0);
 }
 
@@ -5982,7 +6191,7 @@ zpool_do_scrub(int argc, char **argv)
 /*
  * Print out detailed scrub status.
  */
-void
+static void
 print_scan_status(pool_scan_stat_t *ps)
 {
        time_t start, end, pause;
@@ -6129,6 +6338,178 @@ print_scan_status(pool_scan_stat_t *ps)
        }
 }
 
+/*
+ * As we don't scrub checkpointed blocks, we want to warn the
+ * user that we skipped scanning some blocks if a checkpoint exists
+ * or existed at any time during the scan.
+ */
+static void
+print_checkpoint_scan_warning(pool_scan_stat_t *ps, pool_checkpoint_stat_t *pcs)
+{
+       if (ps == NULL || pcs == NULL)
+               return;
+
+       if (pcs->pcs_state == CS_NONE ||
+           pcs->pcs_state == CS_CHECKPOINT_DISCARDING)
+               return;
+
+       assert(pcs->pcs_state == CS_CHECKPOINT_EXISTS);
+
+       if (ps->pss_state == DSS_NONE)
+               return;
+
+       if ((ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) &&
+           ps->pss_end_time < pcs->pcs_start_time)
+               return;
+
+       if (ps->pss_state == DSS_FINISHED || ps->pss_state == DSS_CANCELED) {
+               (void) printf(gettext("    scan warning: skipped blocks "
+                   "that are only referenced by the checkpoint.\n"));
+       } else {
+               assert(ps->pss_state == DSS_SCANNING);
+               (void) printf(gettext("    scan warning: skipping blocks "
+                   "that are only referenced by the checkpoint.\n"));
+       }
+}
+
+/*
+ * Print out detailed removal status.
+ */
+static void
+print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
+{
+       char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
+       time_t start, end;
+       nvlist_t *config, *nvroot;
+       nvlist_t **child;
+       uint_t children;
+       char *vdev_name;
+
+       if (prs == NULL || prs->prs_state == DSS_NONE)
+               return;
+
+       /*
+        * Determine name of vdev.
+        */
+       config = zpool_get_config(zhp, NULL);
+       nvroot = fnvlist_lookup_nvlist(config,
+           ZPOOL_CONFIG_VDEV_TREE);
+       verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
+           &child, &children) == 0);
+       assert(prs->prs_removing_vdev < children);
+       vdev_name = zpool_vdev_name(g_zfs, zhp,
+           child[prs->prs_removing_vdev], B_TRUE);
+
+       (void) printf(gettext("remove: "));
+
+       start = prs->prs_start_time;
+       end = prs->prs_end_time;
+       zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
+
+       /*
+        * Removal is finished or canceled.
+        */
+       if (prs->prs_state == DSS_FINISHED) {
+               uint64_t minutes_taken = (end - start) / 60;
+
+               (void) printf(gettext("Removal of vdev %llu copied %s "
+                   "in %lluh%um, completed on %s"),
+                   (longlong_t)prs->prs_removing_vdev,
+                   copied_buf,
+                   (u_longlong_t)(minutes_taken / 60),
+                   (uint_t)(minutes_taken % 60),
+                   ctime((time_t *)&end));
+       } else if (prs->prs_state == DSS_CANCELED) {
+               (void) printf(gettext("Removal of %s canceled on %s"),
+                   vdev_name, ctime(&end));
+       } else {
+               uint64_t copied, total, elapsed, mins_left, hours_left;
+               double fraction_done;
+               uint_t rate;
+
+               assert(prs->prs_state == DSS_SCANNING);
+
+               /*
+                * Removal is in progress.
+                */
+               (void) printf(gettext(
+                   "Evacuation of %s in progress since %s"),
+                   vdev_name, ctime(&start));
+
+               copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
+               total = prs->prs_to_copy;
+               fraction_done = (double)copied / total;
+
+               /* elapsed time for this pass */
+               elapsed = time(NULL) - prs->prs_start_time;
+               elapsed = elapsed > 0 ? elapsed : 1;
+               rate = copied / elapsed;
+               rate = rate > 0 ? rate : 1;
+               mins_left = ((total - copied) / rate) / 60;
+               hours_left = mins_left / 60;
+
+               zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
+               zfs_nicenum(total, total_buf, sizeof (total_buf));
+               zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
+
+               /*
+                * do not print estimated time if hours_left is more than
+                * 30 days
+                */
+               (void) printf(gettext("    %s copied out of %s at %s/s, "
+                   "%.2f%% done"),
+                   examined_buf, total_buf, rate_buf, 100 * fraction_done);
+               if (hours_left < (30 * 24)) {
+                       (void) printf(gettext(", %lluh%um to go\n"),
+                           (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
+               } else {
+                       (void) printf(gettext(
+                           ", (copy is slow, no estimated time)\n"));
+               }
+       }
+
+       if (prs->prs_mapping_memory > 0) {
+               char mem_buf[7];
+               zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
+               (void) printf(gettext("    %s memory used for "
+                   "removed device mappings\n"),
+                   mem_buf);
+       }
+}
+
+static void
+print_checkpoint_status(pool_checkpoint_stat_t *pcs)
+{
+       time_t start;
+       char space_buf[7];
+
+       if (pcs == NULL || pcs->pcs_state == CS_NONE)
+               return;
+
+       (void) printf(gettext("checkpoint: "));
+
+       start = pcs->pcs_start_time;
+       zfs_nicenum(pcs->pcs_space, space_buf, sizeof (space_buf));
+
+       if (pcs->pcs_state == CS_CHECKPOINT_EXISTS) {
+               char *date = ctime(&start);
+
+               /*
+                * ctime() adds a newline at the end of the generated
+                * string, thus the weird format specifier and the
+                * strlen() call used to chop it off from the output.
+                */
+               (void) printf(gettext("created %.*s, consumes %s\n"),
+                   (int)(strlen(date) - 1), date, space_buf);
+               return;
+       }
+
+       assert(pcs->pcs_state == CS_CHECKPOINT_DISCARDING);
+
+       (void) printf(gettext("discarding, %s remaining.\n"),
+           space_buf);
+}
+
 static void
 print_error_log(zpool_handle_t *zhp)
 {
@@ -6294,11 +6675,11 @@ status_callback(zpool_handle_t *zhp, void *data)
        else
                (void) printf("\n");
 
-       verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
-           &nvroot) == 0);
+       nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
        verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
            (uint64_t **)&vs, &c) == 0);
-       health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
+
+       health = zpool_get_state_str(zhp);
 
        (void) printf(gettext("  pool: %s\n"), zpool_get_name(zhp));
        (void) printf(gettext(" state: %s\n"), health);
@@ -6467,6 +6848,15 @@ status_callback(zpool_handle_t *zhp, void *data)
                    "to be recovered.\n"));
                break;
 
+       case ZPOOL_STATUS_IO_FAILURE_MMP:
+               (void) printf(gettext("status: The pool is suspended because "
+                   "multihost writes failed or were delayed;\n\tanother "
+                   "system could import the pool undetected.\n"));
+               (void) printf(gettext("action: Make sure the pool's devices "
+                   "are connected, then reboot your system and\n\timport the "
+                   "pool.\n"));
+               break;
+
        case ZPOOL_STATUS_IO_FAILURE_WAIT:
        case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
                (void) printf(gettext("status: One or more devices are "
@@ -6545,11 +6935,21 @@ status_callback(zpool_handle_t *zhp, void *data)
                uint64_t nerr;
                nvlist_t **spares, **l2cache;
                uint_t nspares, nl2cache;
+               pool_checkpoint_stat_t *pcs = NULL;
                pool_scan_stat_t *ps = NULL;
+               pool_removal_stat_t *prs = NULL;
 
+               (void) nvlist_lookup_uint64_array(nvroot,
+                   ZPOOL_CONFIG_CHECKPOINT_STATS, (uint64_t **)&pcs, &c);
                (void) nvlist_lookup_uint64_array(nvroot,
                    ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&ps, &c);
+               (void) nvlist_lookup_uint64_array(nvroot,
+                   ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
+
                print_scan_status(ps);
+               print_checkpoint_scan_warning(ps, pcs);
+               print_removal_status(zhp, prs);
+               print_checkpoint_status(pcs);
 
                cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
                    cbp->cb_name_flags | VDEV_NAME_TYPE_ID);