granicus.if.org Git - zfs/blob - cmd/zpool/zpool_main.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
  26  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
  27  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
  28  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
  29  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
  30  * Copyright (c) 2017 Datto Inc.
  31  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  32  */
  33
  34 #include <assert.h>
  35 #include <ctype.h>
  36 #include <dirent.h>
  37 #include <errno.h>
  38 #include <fcntl.h>
  39 #include <libgen.h>
  40 #include <libintl.h>
  41 #include <libuutil.h>
  42 #include <locale.h>
  43 #include <stdio.h>
  44 #include <stdlib.h>
  45 #include <string.h>
  46 #include <strings.h>
  47 #include <unistd.h>
  48 #include <pwd.h>
  49 #include <zone.h>
  50 #include <sys/wait.h>
  51 #include <zfs_prop.h>
  52 #include <sys/fs/zfs.h>
  53 #include <sys/stat.h>
  54 #include <sys/systeminfo.h>
  55 #include <sys/fm/fs/zfs.h>
  56 #include <sys/fm/util.h>
  57 #include <sys/fm/protocol.h>
  58 #include <sys/zfs_ioctl.h>
  59 #include <sys/mount.h>
  60 #include <sys/sysmacros.h>
  61
  62 #include <math.h>
  63
  64 #include <libzfs.h>
  65
  66 #include "zpool_util.h"
  67 #include "zfs_comutil.h"
  68 #include "zfeature_common.h"
  69
  70 #include "statcommon.h"
  71
  72 static int zpool_do_create(int, char **);
  73 static int zpool_do_destroy(int, char **);
  74
  75 static int zpool_do_add(int, char **);
  76 static int zpool_do_remove(int, char **);
  77 static int zpool_do_labelclear(int, char **);
  78
  79 static int zpool_do_list(int, char **);
  80 static int zpool_do_iostat(int, char **);
  81 static int zpool_do_status(int, char **);
  82
  83 static int zpool_do_online(int, char **);
  84 static int zpool_do_offline(int, char **);
  85 static int zpool_do_clear(int, char **);
  86 static int zpool_do_reopen(int, char **);
  87
  88 static int zpool_do_reguid(int, char **);
  89
  90 static int zpool_do_attach(int, char **);
  91 static int zpool_do_detach(int, char **);
  92 static int zpool_do_replace(int, char **);
  93 static int zpool_do_split(int, char **);
  94
  95 static int zpool_do_scrub(int, char **);
  96
  97 static int zpool_do_import(int, char **);
  98 static int zpool_do_export(int, char **);
  99
 100 static int zpool_do_upgrade(int, char **);
 101
 102 static int zpool_do_history(int, char **);
 103 static int zpool_do_events(int, char **);
 104
 105 static int zpool_do_get(int, char **);
 106 static int zpool_do_set(int, char **);
 107
 108 static int zpool_do_sync(int, char **);
 109
 110 /*
 111  * These libumem hooks provide a reasonable set of defaults for the allocator's
 112  * debugging facilities.
 113  */
 114
 115 #ifdef DEBUG
 116 const char *
 117 _umem_debug_init(void)
 118 {
 119         return ("default,verbose"); /* $UMEM_DEBUG setting */
 120 }
 121
 122 const char *
 123 _umem_logging_init(void)
 124 {
 125         return ("fail,contents"); /* $UMEM_LOGGING setting */
 126 }
 127 #endif
 128
 129 typedef enum {
 130         HELP_ADD,
 131         HELP_ATTACH,
 132         HELP_CLEAR,
 133         HELP_CREATE,
 134         HELP_DESTROY,
 135         HELP_DETACH,
 136         HELP_EXPORT,
 137         HELP_HISTORY,
 138         HELP_IMPORT,
 139         HELP_IOSTAT,
 140         HELP_LABELCLEAR,
 141         HELP_LIST,
 142         HELP_OFFLINE,
 143         HELP_ONLINE,
 144         HELP_REPLACE,
 145         HELP_REMOVE,
 146         HELP_SCRUB,
 147         HELP_STATUS,
 148         HELP_UPGRADE,
 149         HELP_EVENTS,
 150         HELP_GET,
 151         HELP_SET,
 152         HELP_SPLIT,
 153         HELP_SYNC,
 154         HELP_REGUID,
 155         HELP_REOPEN
 156 } zpool_help_t;
 157
 158
 159 /*
 160  * Flags for stats to display with "zpool iostats"
 161  */
 162 enum iostat_type {
 163         IOS_DEFAULT = 0,
 164         IOS_LATENCY = 1,
 165         IOS_QUEUES = 2,
 166         IOS_L_HISTO = 3,
 167         IOS_RQ_HISTO = 4,
 168         IOS_COUNT,      /* always last element */
 169 };
 170
 171 /* iostat_type entries as bitmasks */
 172 #define IOS_DEFAULT_M   (1ULL << IOS_DEFAULT)
 173 #define IOS_LATENCY_M   (1ULL << IOS_LATENCY)
 174 #define IOS_QUEUES_M    (1ULL << IOS_QUEUES)
 175 #define IOS_L_HISTO_M   (1ULL << IOS_L_HISTO)
 176 #define IOS_RQ_HISTO_M  (1ULL << IOS_RQ_HISTO)
 177
 178 /* Mask of all the histo bits */
 179 #define IOS_ANYHISTO_M (IOS_L_HISTO_M | IOS_RQ_HISTO_M)
 180
 181 /*
 182  * Lookup table for iostat flags to nvlist names.  Basically a list
 183  * of all the nvlists a flag requires.  Also specifies the order in
 184  * which data gets printed in zpool iostat.
 185  */
 186 static const char *vsx_type_to_nvlist[IOS_COUNT][11] = {
 187         [IOS_L_HISTO] = {
 188             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 189             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 190             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 191             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 192             ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
 193             ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
 194             ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
 195             ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
 196             ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
 197             NULL},
 198         [IOS_LATENCY] = {
 199             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 200             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 201             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 202             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 203             NULL},
 204         [IOS_QUEUES] = {
 205             ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
 206             ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
 207             ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
 208             ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
 209             ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
 210             NULL},
 211         [IOS_RQ_HISTO] = {
 212             ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
 213             ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
 214             ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
 215             ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
 216             ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
 217             ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
 218             ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
 219             ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
 220             ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
 221             ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
 222             NULL},
 223 };
 224
 225
 226 /*
 227  * Given a cb->cb_flags with a histogram bit set, return the iostat_type.
 228  * Right now, only one histo bit is ever set at one time, so we can
 229  * just do a highbit64(a)
 230  */
 231 #define IOS_HISTO_IDX(a)        (highbit64(a & IOS_ANYHISTO_M) - 1)
 232
 233 typedef struct zpool_command {
 234         const char      *name;
 235         int             (*func)(int, char **);
 236         zpool_help_t    usage;
 237 } zpool_command_t;
 238
 239 /*
 240  * Master command table.  Each ZFS command has a name, associated function, and
 241  * usage message.  The usage messages need to be internationalized, so we have
 242  * to have a function to return the usage message based on a command index.
 243  *
 244  * These commands are organized according to how they are displayed in the usage
 245  * message.  An empty command (one with a NULL name) indicates an empty line in
 246  * the generic usage message.
 247  */
 248 static zpool_command_t command_table[] = {
 249         { "create",     zpool_do_create,        HELP_CREATE             },
 250         { "destroy",    zpool_do_destroy,       HELP_DESTROY            },
 251         { NULL },
 252         { "add",        zpool_do_add,           HELP_ADD                },
 253         { "remove",     zpool_do_remove,        HELP_REMOVE             },
 254         { NULL },
 255         { "labelclear", zpool_do_labelclear,    HELP_LABELCLEAR         },
 256         { NULL },
 257         { "list",       zpool_do_list,          HELP_LIST               },
 258         { "iostat",     zpool_do_iostat,        HELP_IOSTAT             },
 259         { "status",     zpool_do_status,        HELP_STATUS             },
 260         { NULL },
 261         { "online",     zpool_do_online,        HELP_ONLINE             },
 262         { "offline",    zpool_do_offline,       HELP_OFFLINE            },
 263         { "clear",      zpool_do_clear,         HELP_CLEAR              },
 264         { "reopen",     zpool_do_reopen,        HELP_REOPEN             },
 265         { NULL },
 266         { "attach",     zpool_do_attach,        HELP_ATTACH             },
 267         { "detach",     zpool_do_detach,        HELP_DETACH             },
 268         { "replace",    zpool_do_replace,       HELP_REPLACE            },
 269         { "split",      zpool_do_split,         HELP_SPLIT              },
 270         { NULL },
 271         { "scrub",      zpool_do_scrub,         HELP_SCRUB              },
 272         { NULL },
 273         { "import",     zpool_do_import,        HELP_IMPORT             },
 274         { "export",     zpool_do_export,        HELP_EXPORT             },
 275         { "upgrade",    zpool_do_upgrade,       HELP_UPGRADE            },
 276         { "reguid",     zpool_do_reguid,        HELP_REGUID             },
 277         { NULL },
 278         { "history",    zpool_do_history,       HELP_HISTORY            },
 279         { "events",     zpool_do_events,        HELP_EVENTS             },
 280         { NULL },
 281         { "get",        zpool_do_get,           HELP_GET                },
 282         { "set",        zpool_do_set,           HELP_SET                },
 283         { "sync",       zpool_do_sync,          HELP_SYNC               },
 284 };
 285
 286 #define NCOMMAND        (ARRAY_SIZE(command_table))
 287
 288 static zpool_command_t *current_command;
 289 static char history_str[HIS_MAX_RECORD_LEN];
 290 static boolean_t log_history = B_TRUE;
 291 static uint_t timestamp_fmt = NODATE;
 292
 293 static const char *
 294 get_usage(zpool_help_t idx)
 295 {
 296         switch (idx) {
 297         case HELP_ADD:
 298                 return (gettext("\tadd [-fgLnP] [-o property=value] "
 299                     "<pool> <vdev> ...\n"));
 300         case HELP_ATTACH:
 301                 return (gettext("\tattach [-f] [-o property=value] "
 302                     "<pool> <device> <new-device>\n"));
 303         case HELP_CLEAR:
 304                 return (gettext("\tclear [-nF] <pool> [device]\n"));
 305         case HELP_CREATE:
 306                 return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
 307                     "\t    [-O file-system-property=value] ... \n"
 308                     "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
 309         case HELP_DESTROY:
 310                 return (gettext("\tdestroy [-f] <pool>\n"));
 311         case HELP_DETACH:
 312                 return (gettext("\tdetach <pool> <device>\n"));
 313         case HELP_EXPORT:
 314                 return (gettext("\texport [-af] <pool> ...\n"));
 315         case HELP_HISTORY:
 316                 return (gettext("\thistory [-il] [<pool>] ...\n"));
 317         case HELP_IMPORT:
 318                 return (gettext("\timport [-d dir] [-D]\n"
 319                     "\timport [-d dir | -c cachefile] [-F [-n]] [-l] "
 320                     "<pool | id>\n"
 321                     "\timport [-o mntopts] [-o property=value] ... \n"
 322                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 323                     "[-R root] [-F [-n]] -a\n"
 324                     "\timport [-o mntopts] [-o property=value] ... \n"
 325                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 326                     "[-R root] [-F [-n]]\n"
 327                     "\t    <pool | id> [newpool]\n"));
 328         case HELP_IOSTAT:
 329                 return (gettext("\tiostat [[[-c [script1,script2,...]"
 330                     "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
 331                     "\t    [[pool ...]|[pool vdev ...]|[vdev ...]]"
 332                     " [interval [count]]\n"));
 333         case HELP_LABELCLEAR:
 334                 return (gettext("\tlabelclear [-f] <vdev>\n"));
 335         case HELP_LIST:
 336                 return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
 337                     "[-T d|u] [pool] ... \n"
 338                     "\t    [interval [count]]\n"));
 339         case HELP_OFFLINE:
 340                 return (gettext("\toffline [-f] [-t] <pool> <device> ...\n"));
 341         case HELP_ONLINE:
 342                 return (gettext("\tonline <pool> <device> ...\n"));
 343         case HELP_REPLACE:
 344                 return (gettext("\treplace [-f] [-o property=value] "
 345                     "<pool> <device> [new-device]\n"));
 346         case HELP_REMOVE:
 347                 return (gettext("\tremove [-nps] <pool> <device> ...\n"));
 348         case HELP_REOPEN:
 349                 return (gettext("\treopen [-n] <pool>\n"));
 350         case HELP_SCRUB:
 351                 return (gettext("\tscrub [-s | -p] <pool> ...\n"));
 352         case HELP_STATUS:
 353                 return (gettext("\tstatus [-c [script1,script2,...]] [-gLPvxD]"
 354                     "[-T d|u] [pool] ... \n"
 355                     "\t    [interval [count]]\n"));
 356         case HELP_UPGRADE:
 357                 return (gettext("\tupgrade\n"
 358                     "\tupgrade -v\n"
 359                     "\tupgrade [-V version] <-a | pool ...>\n"));
 360         case HELP_EVENTS:
 361                 return (gettext("\tevents [-vHf [pool] | -c]\n"));
 362         case HELP_GET:
 363                 return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
 364                     "<\"all\" | property[,...]> <pool> ...\n"));
 365         case HELP_SET:
 366                 return (gettext("\tset <property=value> <pool> \n"));
 367         case HELP_SPLIT:
 368                 return (gettext("\tsplit [-gLnPl] [-R altroot] [-o mntopts]\n"
 369                     "\t    [-o property=value] <pool> <newpool> "
 370                     "[<device> ...]\n"));
 371         case HELP_REGUID:
 372                 return (gettext("\treguid <pool>\n"));
 373         case HELP_SYNC:
 374                 return (gettext("\tsync [pool] ...\n"));
 375         }
 376
 377         abort();
 378         /* NOTREACHED */
 379 }
 380
 381
 382 /*
 383  * Callback routine that will print out a pool property value.
 384  */
 385 static int
 386 print_prop_cb(int prop, void *cb)
 387 {
 388         FILE *fp = cb;
 389
 390         (void) fprintf(fp, "\t%-15s  ", zpool_prop_to_name(prop));
 391
 392         if (zpool_prop_readonly(prop))
 393                 (void) fprintf(fp, "  NO   ");
 394         else
 395                 (void) fprintf(fp, " YES   ");
 396
 397         if (zpool_prop_values(prop) == NULL)
 398                 (void) fprintf(fp, "-\n");
 399         else
 400                 (void) fprintf(fp, "%s\n", zpool_prop_values(prop));
 401
 402         return (ZPROP_CONT);
 403 }
 404
 405 /*
 406  * Display usage message.  If we're inside a command, display only the usage for
 407  * that command.  Otherwise, iterate over the entire command table and display
 408  * a complete usage message.
 409  */
 410 void
 411 usage(boolean_t requested)
 412 {
 413         FILE *fp = requested ? stdout : stderr;
 414
 415         if (current_command == NULL) {
 416                 int i;
 417
 418                 (void) fprintf(fp, gettext("usage: zpool command args ...\n"));
 419                 (void) fprintf(fp,
 420                     gettext("where 'command' is one of the following:\n\n"));
 421
 422                 for (i = 0; i < NCOMMAND; i++) {
 423                         if (command_table[i].name == NULL)
 424                                 (void) fprintf(fp, "\n");
 425                         else
 426                                 (void) fprintf(fp, "%s",
 427                                     get_usage(command_table[i].usage));
 428                 }
 429         } else {
 430                 (void) fprintf(fp, gettext("usage:\n"));
 431                 (void) fprintf(fp, "%s", get_usage(current_command->usage));
 432         }
 433
 434         if (current_command != NULL &&
 435             ((strcmp(current_command->name, "set") == 0) ||
 436             (strcmp(current_command->name, "get") == 0) ||
 437             (strcmp(current_command->name, "list") == 0))) {
 438
 439                 (void) fprintf(fp,
 440                     gettext("\nthe following properties are supported:\n"));
 441
 442                 (void) fprintf(fp, "\n\t%-15s  %s   %s\n\n",
 443                     "PROPERTY", "EDIT", "VALUES");
 444
 445                 /* Iterate over all properties */
 446                 (void) zprop_iter(print_prop_cb, fp, B_FALSE, B_TRUE,
 447                     ZFS_TYPE_POOL);
 448
 449                 (void) fprintf(fp, "\t%-15s   ", "feature@...");
 450                 (void) fprintf(fp, "YES   disabled | enabled | active\n");
 451
 452                 (void) fprintf(fp, gettext("\nThe feature@ properties must be "
 453                     "appended with a feature name.\nSee zpool-features(5).\n"));
 454         }
 455
 456         /*
 457          * See comments at end of main().
 458          */
 459         if (getenv("ZFS_ABORT") != NULL) {
 460                 (void) printf("dumping core by request\n");
 461                 abort();
 462         }
 463
 464         exit(requested ? 0 : 2);
 465 }
 466
 467 void
 468 print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
 469     boolean_t print_logs, int name_flags)
 470 {
 471         nvlist_t **child;
 472         uint_t c, children;
 473         char *vname;
 474
 475         if (name != NULL)
 476                 (void) printf("\t%*s%s\n", indent, "", name);
 477
 478         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
 479             &child, &children) != 0)
 480                 return;
 481
 482         for (c = 0; c < children; c++) {
 483                 uint64_t is_log = B_FALSE;
 484
 485                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
 486                     &is_log);
 487                 if ((is_log && !print_logs) || (!is_log && print_logs))
 488                         continue;
 489
 490                 vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
 491                 print_vdev_tree(zhp, vname, child[c], indent + 2,
 492                     B_FALSE, name_flags);
 493                 free(vname);
 494         }
 495 }
 496
 497 static boolean_t
 498 prop_list_contains_feature(nvlist_t *proplist)
 499 {
 500         nvpair_t *nvp;
 501         for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
 502             nvp = nvlist_next_nvpair(proplist, nvp)) {
 503                 if (zpool_prop_feature(nvpair_name(nvp)))
 504                         return (B_TRUE);
 505         }
 506         return (B_FALSE);
 507 }
 508
 509 /*
 510  * Add a property pair (name, string-value) into a property nvlist.
 511  */
 512 static int
 513 add_prop_list(const char *propname, char *propval, nvlist_t **props,
 514     boolean_t poolprop)
 515 {
 516         zpool_prop_t prop = ZPOOL_PROP_INVAL;
 517         zfs_prop_t fprop;
 518         nvlist_t *proplist;
 519         const char *normnm;
 520         char *strval;
 521
 522         if (*props == NULL &&
 523             nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
 524                 (void) fprintf(stderr,
 525                     gettext("internal error: out of memory\n"));
 526                 return (1);
 527         }
 528
 529         proplist = *props;
 530
 531         if (poolprop) {
 532                 const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
 533
 534                 if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
 535                     !zpool_prop_feature(propname)) {
 536                         (void) fprintf(stderr, gettext("property '%s' is "
 537                             "not a valid pool property\n"), propname);
 538                         return (2);
 539                 }
 540
 541                 /*
 542                  * feature@ properties and version should not be specified
 543                  * at the same time.
 544                  */
 545                 if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
 546                     nvlist_exists(proplist, vname)) ||
 547                     (prop == ZPOOL_PROP_VERSION &&
 548                     prop_list_contains_feature(proplist))) {
 549                         (void) fprintf(stderr, gettext("'feature@' and "
 550                             "'version' properties cannot be specified "
 551                             "together\n"));
 552                         return (2);
 553                 }
 554
 555
 556                 if (zpool_prop_feature(propname))
 557                         normnm = propname;
 558                 else
 559                         normnm = zpool_prop_to_name(prop);
 560         } else {
 561                 if ((fprop = zfs_name_to_prop(propname)) != ZPROP_INVAL) {
 562                         normnm = zfs_prop_to_name(fprop);
 563                 } else {
 564                         normnm = propname;
 565                 }
 566         }
 567
 568         if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
 569             prop != ZPOOL_PROP_CACHEFILE) {
 570                 (void) fprintf(stderr, gettext("property '%s' "
 571                     "specified multiple times\n"), propname);
 572                 return (2);
 573         }
 574
 575         if (nvlist_add_string(proplist, normnm, propval) != 0) {
 576                 (void) fprintf(stderr, gettext("internal "
 577                     "error: out of memory\n"));
 578                 return (1);
 579         }
 580
 581         return (0);
 582 }
 583
 584 /*
 585  * Set a default property pair (name, string-value) in a property nvlist
 586  */
 587 static int
 588 add_prop_list_default(const char *propname, char *propval, nvlist_t **props,
 589     boolean_t poolprop)
 590 {
 591         char *pval;
 592
 593         if (nvlist_lookup_string(*props, propname, &pval) == 0)
 594                 return (0);
 595
 596         return (add_prop_list(propname, propval, props, B_TRUE));
 597 }
 598
 599 /*
 600  * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
 601  *
 602  *      -f      Force addition of devices, even if they appear in use
 603  *      -g      Display guid for individual vdev name.
 604  *      -L      Follow links when resolving vdev path name.
 605  *      -n      Do not add the devices, but display the resulting layout if
 606  *              they were to be added.
 607  *      -o      Set property=value.
 608  *      -P      Display full path for vdev name.
 609  *
 610  * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
 611  * handled by get_vdev_spec(), which constructs the nvlist needed to pass to
 612  * libzfs.
 613  */
 614 int
 615 zpool_do_add(int argc, char **argv)
 616 {
 617         boolean_t force = B_FALSE;
 618         boolean_t dryrun = B_FALSE;
 619         int name_flags = 0;
 620         int c;
 621         nvlist_t *nvroot;
 622         char *poolname;
 623         int ret;
 624         zpool_handle_t *zhp;
 625         nvlist_t *config;
 626         nvlist_t *props = NULL;
 627         char *propval;
 628
 629         /* check options */
 630         while ((c = getopt(argc, argv, "fgLno:P")) != -1) {
 631                 switch (c) {
 632                 case 'f':
 633                         force = B_TRUE;
 634                         break;
 635                 case 'g':
 636                         name_flags |= VDEV_NAME_GUID;
 637                         break;
 638                 case 'L':
 639                         name_flags |= VDEV_NAME_FOLLOW_LINKS;
 640                         break;
 641                 case 'n':
 642                         dryrun = B_TRUE;
 643                         break;
 644                 case 'o':
 645                         if ((propval = strchr(optarg, '=')) == NULL) {
 646                                 (void) fprintf(stderr, gettext("missing "
 647                                     "'=' for -o option\n"));
 648                                 usage(B_FALSE);
 649                         }
 650                         *propval = '\0';
 651                         propval++;
 652
 653                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
 654                             (add_prop_list(optarg, propval, &props, B_TRUE)))
 655                                 usage(B_FALSE);
 656                         break;
 657                 case 'P':
 658                         name_flags |= VDEV_NAME_PATH;
 659                         break;
 660                 case '?':
 661                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
 662                             optopt);
 663                         usage(B_FALSE);
 664                 }
 665         }
 666
 667         argc -= optind;
 668         argv += optind;
 669
 670         /* get pool name and check number of arguments */
 671         if (argc < 1) {
 672                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 673                 usage(B_FALSE);
 674         }
 675         if (argc < 2) {
 676                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
 677                 usage(B_FALSE);
 678         }
 679
 680         poolname = argv[0];
 681
 682         argc--;
 683         argv++;
 684
 685         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
 686                 return (1);
 687
 688         if ((config = zpool_get_config(zhp, NULL)) == NULL) {
 689                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
 690                     poolname);
 691                 zpool_close(zhp);
 692                 return (1);
 693         }
 694
 695         /* unless manually specified use "ashift" pool property (if set) */
 696         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
 697                 int intval;
 698                 zprop_source_t src;
 699                 char strval[ZPOOL_MAXPROPLEN];
 700
 701                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
 702                 if (src != ZPROP_SRC_DEFAULT) {
 703                         (void) sprintf(strval, "%" PRId32, intval);
 704                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
 705                             &props, B_TRUE) == 0);
 706                 }
 707         }
 708
 709         /* pass off to get_vdev_spec for processing */
 710         nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
 711             argc, argv);
 712         if (nvroot == NULL) {
 713                 zpool_close(zhp);
 714                 return (1);
 715         }
 716
 717         if (dryrun) {
 718                 nvlist_t *poolnvroot;
 719                 nvlist_t **l2child;
 720                 uint_t l2children, c;
 721                 char *vname;
 722                 boolean_t hadcache = B_FALSE;
 723
 724                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
 725                     &poolnvroot) == 0);
 726
 727                 (void) printf(gettext("would update '%s' to the following "
 728                     "configuration:\n"), zpool_get_name(zhp));
 729
 730                 /* print original main pool and new tree */
 731                 print_vdev_tree(zhp, poolname, poolnvroot, 0, B_FALSE,
 732                     name_flags);
 733                 print_vdev_tree(zhp, NULL, nvroot, 0, B_FALSE, name_flags);
 734
 735                 /* Do the same for the logs */
 736                 if (num_logs(poolnvroot) > 0) {
 737                         print_vdev_tree(zhp, "logs", poolnvroot, 0, B_TRUE,
 738                             name_flags);
 739                         print_vdev_tree(zhp, NULL, nvroot, 0, B_TRUE,
 740                             name_flags);
 741                 } else if (num_logs(nvroot) > 0) {
 742                         print_vdev_tree(zhp, "logs", nvroot, 0, B_TRUE,
 743                             name_flags);
 744                 }
 745
 746                 /* Do the same for the caches */
 747                 if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_L2CACHE,
 748                     &l2child, &l2children) == 0 && l2children) {
 749                         hadcache = B_TRUE;
 750                         (void) printf(gettext("\tcache\n"));
 751                         for (c = 0; c < l2children; c++) {
 752                                 vname = zpool_vdev_name(g_zfs, NULL,
 753                                     l2child[c], name_flags);
 754                                 (void) printf("\t  %s\n", vname);
 755                                 free(vname);
 756                         }
 757                 }
 758                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
 759                     &l2child, &l2children) == 0 && l2children) {
 760                         if (!hadcache)
 761                                 (void) printf(gettext("\tcache\n"));
 762                         for (c = 0; c < l2children; c++) {
 763                                 vname = zpool_vdev_name(g_zfs, NULL,
 764                                     l2child[c], name_flags);
 765                                 (void) printf("\t  %s\n", vname);
 766                                 free(vname);
 767                         }
 768                 }
 769
 770                 ret = 0;
 771         } else {
 772                 ret = (zpool_add(zhp, nvroot) != 0);
 773         }
 774
 775         nvlist_free(props);
 776         nvlist_free(nvroot);
 777         zpool_close(zhp);
 778
 779         return (ret);
 780 }
 781
 782 /*
 783  * zpool remove  <pool> <vdev> ...
 784  *
 785  * Removes the given vdev from the pool.
 786  */
 787 int
 788 zpool_do_remove(int argc, char **argv)
 789 {
 790         char *poolname;
 791         int i, ret = 0;
 792         zpool_handle_t *zhp = NULL;
 793         boolean_t stop = B_FALSE;
 794         char c;
 795         boolean_t noop = B_FALSE;
 796         boolean_t parsable = B_FALSE;
 797
 798         /* check options */
 799         while ((c = getopt(argc, argv, "nps")) != -1) {
 800                 switch (c) {
 801                 case 'n':
 802                         noop = B_TRUE;
 803                         break;
 804                 case 'p':
 805                         parsable = B_TRUE;
 806                         break;
 807                 case 's':
 808                         stop = B_TRUE;
 809                         break;
 810                 case '?':
 811                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
 812                             optopt);
 813                         usage(B_FALSE);
 814                 }
 815         }
 816
 817         argc -= optind;
 818         argv += optind;
 819
 820         /* get pool name and check number of arguments */
 821         if (argc < 1) {
 822                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 823                 usage(B_FALSE);
 824         }
 825
 826         poolname = argv[0];
 827
 828         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
 829                 return (1);
 830
 831         if (stop && noop) {
 832                 (void) fprintf(stderr, gettext("stop request ignored\n"));
 833                 return (0);
 834         }
 835
 836         if (stop) {
 837                 if (argc > 1) {
 838                         (void) fprintf(stderr, gettext("too many arguments\n"));
 839                         usage(B_FALSE);
 840                 }
 841                 if (zpool_vdev_remove_cancel(zhp) != 0)
 842                         ret = 1;
 843         } else {
 844                 if (argc < 2) {
 845                         (void) fprintf(stderr, gettext("missing device\n"));
 846                         usage(B_FALSE);
 847                 }
 848
 849                 for (i = 1; i < argc; i++) {
 850                         if (noop) {
 851                                 uint64_t size;
 852
 853                                 if (zpool_vdev_indirect_size(zhp, argv[i],
 854                                     &size) != 0) {
 855                                         ret = 1;
 856                                         break;
 857                                 }
 858                                 if (parsable) {
 859                                         (void) printf("%s %llu\n",
 860                                             argv[i], (unsigned long long)size);
 861                                 } else {
 862                                         char valstr[32];
 863                                         zfs_nicenum(size, valstr,
 864                                             sizeof (valstr));
 865                                         (void) printf("Memory that will be "
 866                                             "used after removing %s: %s\n",
 867                                             argv[i], valstr);
 868                                 }
 869                         } else {
 870                                 if (zpool_vdev_remove(zhp, argv[i]) != 0)
 871                                         ret = 1;
 872                         }
 873                 }
 874         }
 875         zpool_close(zhp);
 876
 877         return (ret);
 878 }
 879
 880 /*
 881  * zpool labelclear [-f] <vdev>
 882  *
 883  *      -f      Force clearing the label for the vdevs which are members of
 884  *              the exported or foreign pools.
 885  *
 886  * Verifies that the vdev is not active and zeros out the label information
 887  * on the device.
 888  */
 889 int
 890 zpool_do_labelclear(int argc, char **argv)
 891 {
 892         char vdev[MAXPATHLEN];
 893         char *name = NULL;
 894         struct stat st;
 895         int c, fd = -1, ret = 0;
 896         nvlist_t *config;
 897         pool_state_t state;
 898         boolean_t inuse = B_FALSE;
 899         boolean_t force = B_FALSE;
 900
 901         /* check options */
 902         while ((c = getopt(argc, argv, "f")) != -1) {
 903                 switch (c) {
 904                 case 'f':
 905                         force = B_TRUE;
 906                         break;
 907                 default:
 908                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
 909                             optopt);
 910                         usage(B_FALSE);
 911                 }
 912         }
 913
 914         argc -= optind;
 915         argv += optind;
 916
 917         /* get vdev name */
 918         if (argc < 1) {
 919                 (void) fprintf(stderr, gettext("missing vdev name\n"));
 920                 usage(B_FALSE);
 921         }
 922         if (argc > 1) {
 923                 (void) fprintf(stderr, gettext("too many arguments\n"));
 924                 usage(B_FALSE);
 925         }
 926
 927         /*
 928          * Check if we were given absolute path and use it as is.
 929          * Otherwise if the provided vdev name doesn't point to a file,
 930          * try prepending expected disk paths and partition numbers.
 931          */
 932         (void) strlcpy(vdev, argv[0], sizeof (vdev));
 933         if (vdev[0] != '/' && stat(vdev, &st) != 0) {
 934                 int error;
 935
 936                 error = zfs_resolve_shortname(argv[0], vdev, MAXPATHLEN);
 937                 if (error == 0 && zfs_dev_is_whole_disk(vdev)) {
 938                         if (zfs_append_partition(vdev, MAXPATHLEN) == -1)
 939                                 error = ENOENT;
 940                 }
 941
 942                 if (error || (stat(vdev, &st) != 0)) {
 943                         (void) fprintf(stderr, gettext(
 944                             "failed to find device %s, try specifying absolute "
 945                             "path instead\n"), argv[0]);
 946                         return (1);
 947                 }
 948         }
 949
 950         if ((fd = open(vdev, O_RDWR)) < 0) {
 951                 (void) fprintf(stderr, gettext("failed to open %s: %s\n"),
 952                     vdev, strerror(errno));
 953                 return (1);
 954         }
 955
 956         if (ioctl(fd, BLKFLSBUF) != 0)
 957                 (void) fprintf(stderr, gettext("failed to invalidate "
 958                     "cache for %s: %s\n"), vdev, strerror(errno));
 959
 960         if (zpool_read_label(fd, &config, NULL) != 0 || config == NULL) {
 961                 (void) fprintf(stderr,
 962                     gettext("failed to check state for %s\n"), vdev);
 963                 ret = 1;
 964                 goto errout;
 965         }
 966         nvlist_free(config);
 967
 968         ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
 969         if (ret != 0) {
 970                 (void) fprintf(stderr,
 971                     gettext("failed to check state for %s\n"), vdev);
 972                 ret = 1;
 973                 goto errout;
 974         }
 975
 976         if (!inuse)
 977                 goto wipe_label;
 978
 979         switch (state) {
 980         default:
 981         case POOL_STATE_ACTIVE:
 982         case POOL_STATE_SPARE:
 983         case POOL_STATE_L2CACHE:
 984                 (void) fprintf(stderr, gettext(
 985                     "%s is a member (%s) of pool \"%s\"\n"),
 986                     vdev, zpool_pool_state_to_name(state), name);
 987                 ret = 1;
 988                 goto errout;
 989
 990         case POOL_STATE_EXPORTED:
 991                 if (force)
 992                         break;
 993                 (void) fprintf(stderr, gettext(
 994                     "use '-f' to override the following error:\n"
 995                     "%s is a member of exported pool \"%s\"\n"),
 996                     vdev, name);
 997                 ret = 1;
 998                 goto errout;
 999
1000         case POOL_STATE_POTENTIALLY_ACTIVE:
1001                 if (force)
1002                         break;
1003                 (void) fprintf(stderr, gettext(
1004                     "use '-f' to override the following error:\n"
1005                     "%s is a member of potentially active pool \"%s\"\n"),
1006                     vdev, name);
1007                 ret = 1;
1008                 goto errout;
1009
1010         case POOL_STATE_DESTROYED:
1011                 /* inuse should never be set for a destroyed pool */
1012                 assert(0);
1013                 break;
1014         }
1015
1016 wipe_label:
1017         ret = zpool_clear_label(fd);
1018         if (ret != 0) {
1019                 (void) fprintf(stderr,
1020                     gettext("failed to clear label for %s\n"), vdev);
1021         }
1022
1023 errout:
1024         free(name);
1025         (void) close(fd);
1026
1027         return (ret);
1028 }
1029
1030 /*
1031  * zpool create [-fnd] [-o property=value] ...
1032  *              [-O file-system-property=value] ...
1033  *              [-R root] [-m mountpoint] <pool> <dev> ...
1034  *
1035  *      -f      Force creation, even if devices appear in use
1036  *      -n      Do not create the pool, but display the resulting layout if it
1037  *              were to be created.
1038  *      -R      Create a pool under an alternate root
1039  *      -m      Set default mountpoint for the root dataset.  By default it's
1040  *              '/<pool>'
1041  *      -o      Set property=value.
1042  *      -o      Set feature@feature=enabled|disabled.
1043  *      -d      Don't automatically enable all supported pool features
1044  *              (individual features can be enabled with -o).
1045  *      -O      Set fsproperty=value in the pool's root file system
1046  *
1047  * Creates the named pool according to the given vdev specification.  The
1048  * bulk of the vdev processing is done in get_vdev_spec() in zpool_vdev.c.  Once
1049  * we get the nvlist back from get_vdev_spec(), we either print out the contents
1050  * (if '-n' was specified), or pass it to libzfs to do the creation.
1051  */
1052 int
1053 zpool_do_create(int argc, char **argv)
1054 {
1055         boolean_t force = B_FALSE;
1056         boolean_t dryrun = B_FALSE;
1057         boolean_t enable_all_pool_feat = B_TRUE;
1058         int c;
1059         nvlist_t *nvroot = NULL;
1060         char *poolname;
1061         char *tname = NULL;
1062         int ret = 1;
1063         char *altroot = NULL;
1064         char *mountpoint = NULL;
1065         nvlist_t *fsprops = NULL;
1066         nvlist_t *props = NULL;
1067         char *propval;
1068
1069         /* check options */
1070         while ((c = getopt(argc, argv, ":fndR:m:o:O:t:")) != -1) {
1071                 switch (c) {
1072                 case 'f':
1073                         force = B_TRUE;
1074                         break;
1075                 case 'n':
1076                         dryrun = B_TRUE;
1077                         break;
1078                 case 'd':
1079                         enable_all_pool_feat = B_FALSE;
1080                         break;
1081                 case 'R':
1082                         altroot = optarg;
1083                         if (add_prop_list(zpool_prop_to_name(
1084                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
1085                                 goto errout;
1086                         if (add_prop_list_default(zpool_prop_to_name(
1087                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1088                                 goto errout;
1089                         break;
1090                 case 'm':
1091                         /* Equivalent to -O mountpoint=optarg */
1092                         mountpoint = optarg;
1093                         break;
1094                 case 'o':
1095                         if ((propval = strchr(optarg, '=')) == NULL) {
1096                                 (void) fprintf(stderr, gettext("missing "
1097                                     "'=' for -o option\n"));
1098                                 goto errout;
1099                         }
1100                         *propval = '\0';
1101                         propval++;
1102
1103                         if (add_prop_list(optarg, propval, &props, B_TRUE))
1104                                 goto errout;
1105
1106                         /*
1107                          * If the user is creating a pool that doesn't support
1108                          * feature flags, don't enable any features.
1109                          */
1110                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
1111                                 char *end;
1112                                 u_longlong_t ver;
1113
1114                                 ver = strtoull(propval, &end, 10);
1115                                 if (*end == '\0' &&
1116                                     ver < SPA_VERSION_FEATURES) {
1117                                         enable_all_pool_feat = B_FALSE;
1118                                 }
1119                         }
1120                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
1121                                 altroot = propval;
1122                         break;
1123                 case 'O':
1124                         if ((propval = strchr(optarg, '=')) == NULL) {
1125                                 (void) fprintf(stderr, gettext("missing "
1126                                     "'=' for -O option\n"));
1127                                 goto errout;
1128                         }
1129                         *propval = '\0';
1130                         propval++;
1131
1132                         /*
1133                          * Mountpoints are checked and then added later.
1134                          * Uniquely among properties, they can be specified
1135                          * more than once, to avoid conflict with -m.
1136                          */
1137                         if (0 == strcmp(optarg,
1138                             zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
1139                                 mountpoint = propval;
1140                         } else if (add_prop_list(optarg, propval, &fsprops,
1141                             B_FALSE)) {
1142                                 goto errout;
1143                         }
1144                         break;
1145                 case 't':
1146                         /*
1147                          * Sanity check temporary pool name.
1148                          */
1149                         if (strchr(optarg, '/') != NULL) {
1150                                 (void) fprintf(stderr, gettext("cannot create "
1151                                     "'%s': invalid character '/' in temporary "
1152                                     "name\n"), optarg);
1153                                 (void) fprintf(stderr, gettext("use 'zfs "
1154                                     "create' to create a dataset\n"));
1155                                 goto errout;
1156                         }
1157
1158                         if (add_prop_list(zpool_prop_to_name(
1159                             ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
1160                                 goto errout;
1161                         if (add_prop_list_default(zpool_prop_to_name(
1162                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1163                                 goto errout;
1164                         tname = optarg;
1165                         break;
1166                 case ':':
1167                         (void) fprintf(stderr, gettext("missing argument for "
1168                             "'%c' option\n"), optopt);
1169                         goto badusage;
1170                 case '?':
1171                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1172                             optopt);
1173                         goto badusage;
1174                 }
1175         }
1176
1177         argc -= optind;
1178         argv += optind;
1179
1180         /* get pool name and check number of arguments */
1181         if (argc < 1) {
1182                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1183                 goto badusage;
1184         }
1185         if (argc < 2) {
1186                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
1187                 goto badusage;
1188         }
1189
1190         poolname = argv[0];
1191
1192         /*
1193          * As a special case, check for use of '/' in the name, and direct the
1194          * user to use 'zfs create' instead.
1195          */
1196         if (strchr(poolname, '/') != NULL) {
1197                 (void) fprintf(stderr, gettext("cannot create '%s': invalid "
1198                     "character '/' in pool name\n"), poolname);
1199                 (void) fprintf(stderr, gettext("use 'zfs create' to "
1200                     "create a dataset\n"));
1201                 goto errout;
1202         }
1203
1204         /* pass off to get_vdev_spec for bulk processing */
1205         nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
1206             argc - 1, argv + 1);
1207         if (nvroot == NULL)
1208                 goto errout;
1209
1210         /* make_root_vdev() allows 0 toplevel children if there are spares */
1211         if (!zfs_allocatable_devs(nvroot)) {
1212                 (void) fprintf(stderr, gettext("invalid vdev "
1213                     "specification: at least one toplevel vdev must be "
1214                     "specified\n"));
1215                 goto errout;
1216         }
1217
1218         if (altroot != NULL && altroot[0] != '/') {
1219                 (void) fprintf(stderr, gettext("invalid alternate root '%s': "
1220                     "must be an absolute path\n"), altroot);
1221                 goto errout;
1222         }
1223
1224         /*
1225          * Check the validity of the mountpoint and direct the user to use the
1226          * '-m' mountpoint option if it looks like its in use.
1227          */
1228         if (mountpoint == NULL ||
1229             (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1230             strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1231                 char buf[MAXPATHLEN];
1232                 DIR *dirp;
1233
1234                 if (mountpoint && mountpoint[0] != '/') {
1235                         (void) fprintf(stderr, gettext("invalid mountpoint "
1236                             "'%s': must be an absolute path, 'legacy', or "
1237                             "'none'\n"), mountpoint);
1238                         goto errout;
1239                 }
1240
1241                 if (mountpoint == NULL) {
1242                         if (altroot != NULL)
1243                                 (void) snprintf(buf, sizeof (buf), "%s/%s",
1244                                     altroot, poolname);
1245                         else
1246                                 (void) snprintf(buf, sizeof (buf), "/%s",
1247                                     poolname);
1248                 } else {
1249                         if (altroot != NULL)
1250                                 (void) snprintf(buf, sizeof (buf), "%s%s",
1251                                     altroot, mountpoint);
1252                         else
1253                                 (void) snprintf(buf, sizeof (buf), "%s",
1254                                     mountpoint);
1255                 }
1256
1257                 if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1258                         (void) fprintf(stderr, gettext("mountpoint '%s' : "
1259                             "%s\n"), buf, strerror(errno));
1260                         (void) fprintf(stderr, gettext("use '-m' "
1261                             "option to provide a different default\n"));
1262                         goto errout;
1263                 } else if (dirp) {
1264                         int count = 0;
1265
1266                         while (count < 3 && readdir(dirp) != NULL)
1267                                 count++;
1268                         (void) closedir(dirp);
1269
1270                         if (count > 2) {
1271                                 (void) fprintf(stderr, gettext("mountpoint "
1272                                     "'%s' exists and is not empty\n"), buf);
1273                                 (void) fprintf(stderr, gettext("use '-m' "
1274                                     "option to provide a "
1275                                     "different default\n"));
1276                                 goto errout;
1277                         }
1278                 }
1279         }
1280
1281         /*
1282          * Now that the mountpoint's validity has been checked, ensure that
1283          * the property is set appropriately prior to creating the pool.
1284          */
1285         if (mountpoint != NULL) {
1286                 ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1287                     mountpoint, &fsprops, B_FALSE);
1288                 if (ret != 0)
1289                         goto errout;
1290         }
1291
1292         ret = 1;
1293         if (dryrun) {
1294                 /*
1295                  * For a dry run invocation, print out a basic message and run
1296                  * through all the vdevs in the list and print out in an
1297                  * appropriate hierarchy.
1298                  */
1299                 (void) printf(gettext("would create '%s' with the "
1300                     "following layout:\n\n"), poolname);
1301
1302                 print_vdev_tree(NULL, poolname, nvroot, 0, B_FALSE, 0);
1303                 if (num_logs(nvroot) > 0)
1304                         print_vdev_tree(NULL, "logs", nvroot, 0, B_TRUE, 0);
1305
1306                 ret = 0;
1307         } else {
1308                 /*
1309                  * Hand off to libzfs.
1310                  */
1311                 spa_feature_t i;
1312                 for (i = 0; i < SPA_FEATURES; i++) {
1313                         char propname[MAXPATHLEN];
1314                         char *propval;
1315                         zfeature_info_t *feat = &spa_feature_table[i];
1316
1317                         (void) snprintf(propname, sizeof (propname),
1318                             "feature@%s", feat->fi_uname);
1319
1320                         /*
1321                          * Only features contained in props will be enabled:
1322                          * remove from the nvlist every ZFS_FEATURE_DISABLED
1323                          * value and add every missing ZFS_FEATURE_ENABLED if
1324                          * enable_all_pool_feat is set.
1325                          */
1326                         if (!nvlist_lookup_string(props, propname, &propval)) {
1327                                 if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
1328                                         (void) nvlist_remove_all(props,
1329                                             propname);
1330                         } else if (enable_all_pool_feat) {
1331                                 ret = add_prop_list(propname,
1332                                     ZFS_FEATURE_ENABLED, &props, B_TRUE);
1333                                 if (ret != 0)
1334                                         goto errout;
1335                         }
1336                 }
1337
1338                 ret = 1;
1339                 if (zpool_create(g_zfs, poolname,
1340                     nvroot, props, fsprops) == 0) {
1341                         zfs_handle_t *pool = zfs_open(g_zfs,
1342                             tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
1343                         if (pool != NULL) {
1344                                 if (zfs_mount(pool, NULL, 0) == 0)
1345                                         ret = zfs_shareall(pool);
1346                                 zfs_close(pool);
1347                         }
1348                 } else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1349                         (void) fprintf(stderr, gettext("pool name may have "
1350                             "been omitted\n"));
1351                 }
1352         }
1353
1354 errout:
1355         nvlist_free(nvroot);
1356         nvlist_free(fsprops);
1357         nvlist_free(props);
1358         return (ret);
1359 badusage:
1360         nvlist_free(fsprops);
1361         nvlist_free(props);
1362         usage(B_FALSE);
1363         return (2);
1364 }
1365
1366 /*
1367  * zpool destroy <pool>
1368  *
1369  *      -f      Forcefully unmount any datasets
1370  *
1371  * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1372  */
1373 int
1374 zpool_do_destroy(int argc, char **argv)
1375 {
1376         boolean_t force = B_FALSE;
1377         int c;
1378         char *pool;
1379         zpool_handle_t *zhp;
1380         int ret;
1381
1382         /* check options */
1383         while ((c = getopt(argc, argv, "f")) != -1) {
1384                 switch (c) {
1385                 case 'f':
1386                         force = B_TRUE;
1387                         break;
1388                 case '?':
1389                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1390                             optopt);
1391                         usage(B_FALSE);
1392                 }
1393         }
1394
1395         argc -= optind;
1396         argv += optind;
1397
1398         /* check arguments */
1399         if (argc < 1) {
1400                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1401                 usage(B_FALSE);
1402         }
1403         if (argc > 1) {
1404                 (void) fprintf(stderr, gettext("too many arguments\n"));
1405                 usage(B_FALSE);
1406         }
1407
1408         pool = argv[0];
1409
1410         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1411                 /*
1412                  * As a special case, check for use of '/' in the name, and
1413                  * direct the user to use 'zfs destroy' instead.
1414                  */
1415                 if (strchr(pool, '/') != NULL)
1416                         (void) fprintf(stderr, gettext("use 'zfs destroy' to "
1417                             "destroy a dataset\n"));
1418                 return (1);
1419         }
1420
1421         if (zpool_disable_datasets(zhp, force) != 0) {
1422                 (void) fprintf(stderr, gettext("could not destroy '%s': "
1423                     "could not unmount datasets\n"), zpool_get_name(zhp));
1424                 zpool_close(zhp);
1425                 return (1);
1426         }
1427
1428         /* The history must be logged as part of the export */
1429         log_history = B_FALSE;
1430
1431         ret = (zpool_destroy(zhp, history_str) != 0);
1432
1433         zpool_close(zhp);
1434
1435         return (ret);
1436 }
1437
1438 typedef struct export_cbdata {
1439         boolean_t force;
1440         boolean_t hardforce;
1441 } export_cbdata_t;
1442
1443 /*
1444  * Export one pool
1445  */
1446 int
1447 zpool_export_one(zpool_handle_t *zhp, void *data)
1448 {
1449         export_cbdata_t *cb = data;
1450
1451         if (zpool_disable_datasets(zhp, cb->force) != 0)
1452                 return (1);
1453
1454         /* The history must be logged as part of the export */
1455         log_history = B_FALSE;
1456
1457         if (cb->hardforce) {
1458                 if (zpool_export_force(zhp, history_str) != 0)
1459                         return (1);
1460         } else if (zpool_export(zhp, cb->force, history_str) != 0) {
1461                 return (1);
1462         }
1463
1464         return (0);
1465 }
1466
1467 /*
1468  * zpool export [-f] <pool> ...
1469  *
1470  *      -a      Export all pools
1471  *      -f      Forcefully unmount datasets
1472  *
1473  * Export the given pools.  By default, the command will attempt to cleanly
1474  * unmount any active datasets within the pool.  If the '-f' flag is specified,
1475  * then the datasets will be forcefully unmounted.
1476  */
1477 int
1478 zpool_do_export(int argc, char **argv)
1479 {
1480         export_cbdata_t cb;
1481         boolean_t do_all = B_FALSE;
1482         boolean_t force = B_FALSE;
1483         boolean_t hardforce = B_FALSE;
1484         int c, ret;
1485
1486         /* check options */
1487         while ((c = getopt(argc, argv, "afF")) != -1) {
1488                 switch (c) {
1489                 case 'a':
1490                         do_all = B_TRUE;
1491                         break;
1492                 case 'f':
1493                         force = B_TRUE;
1494                         break;
1495                 case 'F':
1496                         hardforce = B_TRUE;
1497                         break;
1498                 case '?':
1499                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1500                             optopt);
1501                         usage(B_FALSE);
1502                 }
1503         }
1504
1505         cb.force = force;
1506         cb.hardforce = hardforce;
1507         argc -= optind;
1508         argv += optind;
1509
1510         if (do_all) {
1511                 if (argc != 0) {
1512                         (void) fprintf(stderr, gettext("too many arguments\n"));
1513                         usage(B_FALSE);
1514                 }
1515
1516                 return (for_each_pool(argc, argv, B_TRUE, NULL,
1517                     zpool_export_one, &cb));
1518         }
1519
1520         /* check arguments */
1521         if (argc < 1) {
1522                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1523                 usage(B_FALSE);
1524         }
1525
1526         ret = for_each_pool(argc, argv, B_TRUE, NULL, zpool_export_one, &cb);
1527
1528         return (ret);
1529 }
1530
1531 /*
1532  * Given a vdev configuration, determine the maximum width needed for the device
1533  * name column.
1534  */
1535 static int
1536 max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max,
1537     int name_flags)
1538 {
1539         char *name;
1540         nvlist_t **child;
1541         uint_t c, children;
1542         int ret;
1543
1544         name = zpool_vdev_name(g_zfs, zhp, nv, name_flags);
1545         if (strlen(name) + depth > max)
1546                 max = strlen(name) + depth;
1547
1548         free(name);
1549
1550         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1551             &child, &children) == 0) {
1552                 for (c = 0; c < children; c++)
1553                         if ((ret = max_width(zhp, child[c], depth + 2,
1554                             max, name_flags)) > max)
1555                                 max = ret;
1556         }
1557
1558         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1559             &child, &children) == 0) {
1560                 for (c = 0; c < children; c++)
1561                         if ((ret = max_width(zhp, child[c], depth + 2,
1562                             max, name_flags)) > max)
1563                                 max = ret;
1564         }
1565
1566         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1567             &child, &children) == 0) {
1568                 for (c = 0; c < children; c++)
1569                         if ((ret = max_width(zhp, child[c], depth + 2,
1570                             max, name_flags)) > max)
1571                                 max = ret;
1572         }
1573
1574         return (max);
1575 }
1576
1577 typedef struct spare_cbdata {
1578         uint64_t        cb_guid;
1579         zpool_handle_t  *cb_zhp;
1580 } spare_cbdata_t;
1581
1582 static boolean_t
1583 find_vdev(nvlist_t *nv, uint64_t search)
1584 {
1585         uint64_t guid;
1586         nvlist_t **child;
1587         uint_t c, children;
1588
1589         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
1590             search == guid)
1591                 return (B_TRUE);
1592
1593         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1594             &child, &children) == 0) {
1595                 for (c = 0; c < children; c++)
1596                         if (find_vdev(child[c], search))
1597                                 return (B_TRUE);
1598         }
1599
1600         return (B_FALSE);
1601 }
1602
1603 static int
1604 find_spare(zpool_handle_t *zhp, void *data)
1605 {
1606         spare_cbdata_t *cbp = data;
1607         nvlist_t *config, *nvroot;
1608
1609         config = zpool_get_config(zhp, NULL);
1610         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1611             &nvroot) == 0);
1612
1613         if (find_vdev(nvroot, cbp->cb_guid)) {
1614                 cbp->cb_zhp = zhp;
1615                 return (1);
1616         }
1617
1618         zpool_close(zhp);
1619         return (0);
1620 }
1621
1622 typedef struct status_cbdata {
1623         int             cb_count;
1624         int             cb_name_flags;
1625         int             cb_namewidth;
1626         boolean_t       cb_allpools;
1627         boolean_t       cb_verbose;
1628         boolean_t       cb_explain;
1629         boolean_t       cb_first;
1630         boolean_t       cb_dedup_stats;
1631         boolean_t       cb_print_status;
1632         vdev_cmd_data_list_t    *vcdl;
1633 } status_cbdata_t;
1634
1635 /* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
1636 static int
1637 is_blank_str(char *str)
1638 {
1639         while (str != NULL && *str != '\0') {
1640                 if (!isblank(*str))
1641                         return (0);
1642                 str++;
1643         }
1644         return (1);
1645 }
1646
1647 /* Print command output lines for specific vdev in a specific pool */
1648 static void
1649 zpool_print_cmd(vdev_cmd_data_list_t *vcdl, const char *pool, char *path)
1650 {
1651         vdev_cmd_data_t *data;
1652         int i, j;
1653         char *val;
1654
1655         for (i = 0; i < vcdl->count; i++) {
1656                 if ((strcmp(vcdl->data[i].path, path) != 0) ||
1657                     (strcmp(vcdl->data[i].pool, pool) != 0)) {
1658                         /* Not the vdev we're looking for */
1659                         continue;
1660                 }
1661
1662                 data = &vcdl->data[i];
1663                 /* Print out all the output values for this vdev */
1664                 for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
1665                         val = NULL;
1666                         /* Does this vdev have values for this column? */
1667                         for (int k = 0; k < data->cols_cnt; k++) {
1668                                 if (strcmp(data->cols[k],
1669                                     vcdl->uniq_cols[j]) == 0) {
1670                                         /* yes it does, record the value */
1671                                         val = data->lines[k];
1672                                         break;
1673                                 }
1674                         }
1675                         /*
1676                          * Mark empty values with dashes to make output
1677                          * awk-able.
1678                          */
1679                         if (is_blank_str(val))
1680                                 val = "-";
1681
1682                         printf("%*s", vcdl->uniq_cols_width[j], val);
1683                         if (j < vcdl->uniq_cols_cnt - 1)
1684                                 printf("  ");
1685                 }
1686
1687                 /* Print out any values that aren't in a column at the end */
1688                 for (j = data->cols_cnt; j < data->lines_cnt; j++) {
1689                         /* Did we have any columns?  If so print a spacer. */
1690                         if (vcdl->uniq_cols_cnt > 0)
1691                                 printf("  ");
1692
1693                         val = data->lines[j];
1694                         printf("%s", val ? val : "");
1695                 }
1696                 break;
1697         }
1698 }
1699
1700 /*
1701  * Print out configuration state as requested by status_callback.
1702  */
1703 static void
1704 print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
1705     nvlist_t *nv, int depth, boolean_t isspare)
1706 {
1707         nvlist_t **child;
1708         uint_t c, children;
1709         pool_scan_stat_t *ps = NULL;
1710         vdev_stat_t *vs;
1711         char rbuf[6], wbuf[6], cbuf[6];
1712         char *vname;
1713         uint64_t notpresent;
1714         spare_cbdata_t spare_cb;
1715         const char *state;
1716         char *type;
1717         char *path = NULL;
1718
1719         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1720             &child, &children) != 0)
1721                 children = 0;
1722
1723         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
1724             (uint64_t **)&vs, &c) == 0);
1725
1726         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1727
1728         if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
1729                 return;
1730
1731         state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
1732         if (isspare) {
1733                 /*
1734                  * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
1735                  * online drives.
1736                  */
1737                 if (vs->vs_aux == VDEV_AUX_SPARED)
1738                         state = "INUSE";
1739                 else if (vs->vs_state == VDEV_STATE_HEALTHY)
1740                         state = "AVAIL";
1741         }
1742
1743         (void) printf("\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
1744             name, state);
1745
1746         if (!isspare) {
1747                 zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
1748                 zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
1749                 zfs_nicenum(vs->vs_checksum_errors, cbuf, sizeof (cbuf));
1750                 (void) printf(" %5s %5s %5s", rbuf, wbuf, cbuf);
1751         }
1752
1753         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
1754             &notpresent) == 0) {
1755                 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
1756                 (void) printf("  was %s", path);
1757         } else if (vs->vs_aux != 0) {
1758                 (void) printf("  ");
1759
1760                 switch (vs->vs_aux) {
1761                 case VDEV_AUX_OPEN_FAILED:
1762                         (void) printf(gettext("cannot open"));
1763                         break;
1764
1765                 case VDEV_AUX_BAD_GUID_SUM:
1766                         (void) printf(gettext("missing device"));
1767                         break;
1768
1769                 case VDEV_AUX_NO_REPLICAS:
1770                         (void) printf(gettext("insufficient replicas"));
1771                         break;
1772
1773                 case VDEV_AUX_VERSION_NEWER:
1774                         (void) printf(gettext("newer version"));
1775                         break;
1776
1777                 case VDEV_AUX_UNSUP_FEAT:
1778                         (void) printf(gettext("unsupported feature(s)"));
1779                         break;
1780
1781                 case VDEV_AUX_SPARED:
1782                         verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
1783                             &spare_cb.cb_guid) == 0);
1784                         if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
1785                                 if (strcmp(zpool_get_name(spare_cb.cb_zhp),
1786                                     zpool_get_name(zhp)) == 0)
1787                                         (void) printf(gettext("currently in "
1788                                             "use"));
1789                                 else
1790                                         (void) printf(gettext("in use by "
1791                                             "pool '%s'"),
1792                                             zpool_get_name(spare_cb.cb_zhp));
1793                                 zpool_close(spare_cb.cb_zhp);
1794                         } else {
1795                                 (void) printf(gettext("currently in use"));
1796                         }
1797                         break;
1798
1799                 case VDEV_AUX_ERR_EXCEEDED:
1800                         (void) printf(gettext("too many errors"));
1801                         break;
1802
1803                 case VDEV_AUX_IO_FAILURE:
1804                         (void) printf(gettext("experienced I/O failures"));
1805                         break;
1806
1807                 case VDEV_AUX_BAD_LOG:
1808                         (void) printf(gettext("bad intent log"));
1809                         break;
1810
1811                 case VDEV_AUX_EXTERNAL:
1812                         (void) printf(gettext("external device fault"));
1813                         break;
1814
1815                 case VDEV_AUX_SPLIT_POOL:
1816                         (void) printf(gettext("split into new pool"));
1817                         break;
1818
1819                 case VDEV_AUX_ACTIVE:
1820                         (void) printf(gettext("currently in use"));
1821                         break;
1822
1823                 case VDEV_AUX_CHILDREN_OFFLINE:
1824                         (void) printf(gettext("all children offline"));
1825                         break;
1826
1827                 default:
1828                         (void) printf(gettext("corrupted data"));
1829                         break;
1830                 }
1831         }
1832
1833         (void) nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_SCAN_STATS,
1834             (uint64_t **)&ps, &c);
1835
1836         if (ps != NULL && ps->pss_state == DSS_SCANNING &&
1837             vs->vs_scan_processed != 0 && children == 0) {
1838                 (void) printf(gettext("  (%s)"),
1839                     (ps->pss_func == POOL_SCAN_RESILVER) ?
1840                     "resilvering" : "repairing");
1841         }
1842
1843         if (cb->vcdl != NULL) {
1844                 if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0) {
1845                         printf("  ");
1846                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
1847                 }
1848         }
1849
1850         (void) printf("\n");
1851
1852         for (c = 0; c < children; c++) {
1853                 uint64_t islog = B_FALSE, ishole = B_FALSE;
1854
1855                 /* Don't print logs or holes here */
1856                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1857                     &islog);
1858                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
1859                     &ishole);
1860                 if (islog || ishole)
1861                         continue;
1862                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
1863                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
1864                 print_status_config(zhp, cb, vname, child[c], depth + 2,
1865                     isspare);
1866                 free(vname);
1867         }
1868 }
1869
1870 /*
1871  * Print the configuration of an exported pool.  Iterate over all vdevs in the
1872  * pool, printing out the name and status for each one.
1873  */
1874 static void
1875 print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
1876     int depth)
1877 {
1878         nvlist_t **child;
1879         uint_t c, children;
1880         vdev_stat_t *vs;
1881         char *type, *vname;
1882
1883         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1884         if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
1885             strcmp(type, VDEV_TYPE_HOLE) == 0)
1886                 return;
1887
1888         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
1889             (uint64_t **)&vs, &c) == 0);
1890
1891         (void) printf("\t%*s%-*s", depth, "", cb->cb_namewidth - depth, name);
1892         (void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
1893
1894         if (vs->vs_aux != 0) {
1895                 (void) printf("  ");
1896
1897                 switch (vs->vs_aux) {
1898                 case VDEV_AUX_OPEN_FAILED:
1899                         (void) printf(gettext("cannot open"));
1900                         break;
1901
1902                 case VDEV_AUX_BAD_GUID_SUM:
1903                         (void) printf(gettext("missing device"));
1904                         break;
1905
1906                 case VDEV_AUX_NO_REPLICAS:
1907                         (void) printf(gettext("insufficient replicas"));
1908                         break;
1909
1910                 case VDEV_AUX_VERSION_NEWER:
1911                         (void) printf(gettext("newer version"));
1912                         break;
1913
1914                 case VDEV_AUX_UNSUP_FEAT:
1915                         (void) printf(gettext("unsupported feature(s)"));
1916                         break;
1917
1918                 case VDEV_AUX_ERR_EXCEEDED:
1919                         (void) printf(gettext("too many errors"));
1920                         break;
1921
1922                 case VDEV_AUX_ACTIVE:
1923                         (void) printf(gettext("currently in use"));
1924                         break;
1925
1926                 case VDEV_AUX_CHILDREN_OFFLINE:
1927                         (void) printf(gettext("all children offline"));
1928                         break;
1929
1930                 default:
1931                         (void) printf(gettext("corrupted data"));
1932                         break;
1933                 }
1934         }
1935         (void) printf("\n");
1936
1937         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1938             &child, &children) != 0)
1939                 return;
1940
1941         for (c = 0; c < children; c++) {
1942                 uint64_t is_log = B_FALSE;
1943
1944                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1945                     &is_log);
1946                 if (is_log)
1947                         continue;
1948
1949                 vname = zpool_vdev_name(g_zfs, NULL, child[c],
1950                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
1951                 print_import_config(cb, vname, child[c], depth + 2);
1952                 free(vname);
1953         }
1954
1955         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1956             &child, &children) == 0) {
1957                 (void) printf(gettext("\tcache\n"));
1958                 for (c = 0; c < children; c++) {
1959                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
1960                             cb->cb_name_flags);
1961                         (void) printf("\t  %s\n", vname);
1962                         free(vname);
1963                 }
1964         }
1965
1966         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1967             &child, &children) == 0) {
1968                 (void) printf(gettext("\tspares\n"));
1969                 for (c = 0; c < children; c++) {
1970                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
1971                             cb->cb_name_flags);
1972                         (void) printf("\t  %s\n", vname);
1973                         free(vname);
1974                 }
1975         }
1976 }
1977
1978 /*
1979  * Print log vdevs.
1980  * Logs are recorded as top level vdevs in the main pool child array
1981  * but with "is_log" set to 1. We use either print_status_config() or
1982  * print_import_config() to print the top level logs then any log
1983  * children (eg mirrored slogs) are printed recursively - which
1984  * works because only the top level vdev is marked "is_log"
1985  */
1986 static void
1987 print_logs(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv)
1988 {
1989         uint_t c, children;
1990         nvlist_t **child;
1991
1992         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
1993             &children) != 0)
1994                 return;
1995
1996         (void) printf(gettext("\tlogs\n"));
1997
1998         for (c = 0; c < children; c++) {
1999                 uint64_t is_log = B_FALSE;
2000                 char *name;
2001
2002                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2003                     &is_log);
2004                 if (!is_log)
2005                         continue;
2006                 name = zpool_vdev_name(g_zfs, zhp, child[c],
2007                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2008                 if (cb->cb_print_status)
2009                         print_status_config(zhp, cb, name, child[c], 2,
2010                             B_FALSE);
2011                 else
2012                         print_import_config(cb, name, child[c], 2);
2013                 free(name);
2014         }
2015 }
2016
2017 /*
2018  * Display the status for the given pool.
2019  */
2020 static void
2021 show_import(nvlist_t *config)
2022 {
2023         uint64_t pool_state;
2024         vdev_stat_t *vs;
2025         char *name;
2026         uint64_t guid;
2027         uint64_t hostid = 0;
2028         char *msgid;
2029         char *hostname = "unknown";
2030         nvlist_t *nvroot, *nvinfo;
2031         zpool_status_t reason;
2032         zpool_errata_t errata;
2033         const char *health;
2034         uint_t vsc;
2035         char *comment;
2036         status_cbdata_t cb = { 0 };
2037
2038         verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2039             &name) == 0);
2040         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2041             &guid) == 0);
2042         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2043             &pool_state) == 0);
2044         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2045             &nvroot) == 0);
2046
2047         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
2048             (uint64_t **)&vs, &vsc) == 0);
2049         health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2050
2051         reason = zpool_import_status(config, &msgid, &errata);
2052
2053         (void) printf(gettext("   pool: %s\n"), name);
2054         (void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
2055         (void) printf(gettext("  state: %s"), health);
2056         if (pool_state == POOL_STATE_DESTROYED)
2057                 (void) printf(gettext(" (DESTROYED)"));
2058         (void) printf("\n");
2059
2060         switch (reason) {
2061         case ZPOOL_STATUS_MISSING_DEV_R:
2062         case ZPOOL_STATUS_MISSING_DEV_NR:
2063         case ZPOOL_STATUS_BAD_GUID_SUM:
2064                 (void) printf(gettext(" status: One or more devices are "
2065                     "missing from the system.\n"));
2066                 break;
2067
2068         case ZPOOL_STATUS_CORRUPT_LABEL_R:
2069         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
2070                 (void) printf(gettext(" status: One or more devices contains "
2071                     "corrupted data.\n"));
2072                 break;
2073
2074         case ZPOOL_STATUS_CORRUPT_DATA:
2075                 (void) printf(
2076                     gettext(" status: The pool data is corrupted.\n"));
2077                 break;
2078
2079         case ZPOOL_STATUS_OFFLINE_DEV:
2080                 (void) printf(gettext(" status: One or more devices "
2081                     "are offlined.\n"));
2082                 break;
2083
2084         case ZPOOL_STATUS_CORRUPT_POOL:
2085                 (void) printf(gettext(" status: The pool metadata is "
2086                     "corrupted.\n"));
2087                 break;
2088
2089         case ZPOOL_STATUS_VERSION_OLDER:
2090                 (void) printf(gettext(" status: The pool is formatted using a "
2091                     "legacy on-disk version.\n"));
2092                 break;
2093
2094         case ZPOOL_STATUS_VERSION_NEWER:
2095                 (void) printf(gettext(" status: The pool is formatted using an "
2096                     "incompatible version.\n"));
2097                 break;
2098
2099         case ZPOOL_STATUS_FEAT_DISABLED:
2100                 (void) printf(gettext(" status: Some supported features are "
2101                     "not enabled on the pool.\n"));
2102                 break;
2103
2104         case ZPOOL_STATUS_UNSUP_FEAT_READ:
2105                 (void) printf(gettext("status: The pool uses the following "
2106                     "feature(s) not supported on this system:\n"));
2107                 zpool_print_unsup_feat(config);
2108                 break;
2109
2110         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2111                 (void) printf(gettext("status: The pool can only be accessed "
2112                     "in read-only mode on this system. It\n\tcannot be "
2113                     "accessed in read-write mode because it uses the "
2114                     "following\n\tfeature(s) not supported on this system:\n"));
2115                 zpool_print_unsup_feat(config);
2116                 break;
2117
2118         case ZPOOL_STATUS_HOSTID_ACTIVE:
2119                 (void) printf(gettext(" status: The pool is currently "
2120                     "imported by another system.\n"));
2121                 break;
2122
2123         case ZPOOL_STATUS_HOSTID_REQUIRED:
2124                 (void) printf(gettext(" status: The pool has the "
2125                     "multihost property on.  It cannot\n\tbe safely imported "
2126                     "when the system hostid is not set.\n"));
2127                 break;
2128
2129         case ZPOOL_STATUS_HOSTID_MISMATCH:
2130                 (void) printf(gettext(" status: The pool was last accessed by "
2131                     "another system.\n"));
2132                 break;
2133
2134         case ZPOOL_STATUS_FAULTED_DEV_R:
2135         case ZPOOL_STATUS_FAULTED_DEV_NR:
2136                 (void) printf(gettext(" status: One or more devices are "
2137                     "faulted.\n"));
2138                 break;
2139
2140         case ZPOOL_STATUS_BAD_LOG:
2141                 (void) printf(gettext(" status: An intent log record cannot be "
2142                     "read.\n"));
2143                 break;
2144
2145         case ZPOOL_STATUS_RESILVERING:
2146                 (void) printf(gettext(" status: One or more devices were being "
2147                     "resilvered.\n"));
2148                 break;
2149
2150         case ZPOOL_STATUS_ERRATA:
2151                 (void) printf(gettext(" status: Errata #%d detected.\n"),
2152                     errata);
2153                 break;
2154
2155         default:
2156                 /*
2157                  * No other status can be seen when importing pools.
2158                  */
2159                 assert(reason == ZPOOL_STATUS_OK);
2160         }
2161
2162         /*
2163          * Print out an action according to the overall state of the pool.
2164          */
2165         if (vs->vs_state == VDEV_STATE_HEALTHY) {
2166                 if (reason == ZPOOL_STATUS_VERSION_OLDER ||
2167                     reason == ZPOOL_STATUS_FEAT_DISABLED) {
2168                         (void) printf(gettext(" action: The pool can be "
2169                             "imported using its name or numeric identifier, "
2170                             "though\n\tsome features will not be available "
2171                             "without an explicit 'zpool upgrade'.\n"));
2172                 } else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
2173                         (void) printf(gettext(" action: The pool can be "
2174                             "imported using its name or numeric "
2175                             "identifier and\n\tthe '-f' flag.\n"));
2176                 } else if (reason == ZPOOL_STATUS_ERRATA) {
2177                         switch (errata) {
2178                         case ZPOOL_ERRATA_NONE:
2179                                 break;
2180
2181                         case ZPOOL_ERRATA_ZOL_2094_SCRUB:
2182                                 (void) printf(gettext(" action: The pool can "
2183                                     "be imported using its name or numeric "
2184                                     "identifier,\n\thowever there is a compat"
2185                                     "ibility issue which should be corrected"
2186                                     "\n\tby running 'zpool scrub'\n"));
2187                                 break;
2188
2189                         case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
2190                                 (void) printf(gettext(" action: The pool can"
2191                                     "not be imported with this version of ZFS "
2192                                     "due to\n\tan active asynchronous destroy. "
2193                                     "Revert to an earlier version\n\tand "
2194                                     "allow the destroy to complete before "
2195                                     "updating.\n"));
2196                                 break;
2197
2198                         case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
2199                                 (void) printf(gettext(" action: Existing "
2200                                     "encrypted datasets contain an on-disk "
2201                                     "incompatibility, which\n\tneeds to be "
2202                                     "corrected. Backup these datasets to new "
2203                                     "encrypted datasets\n\tand destroy the "
2204                                     "old ones.\n"));
2205                                 break;
2206
2207                         default:
2208                                 /*
2209                                  * All errata must contain an action message.
2210                                  */
2211                                 assert(0);
2212                         }
2213                 } else {
2214                         (void) printf(gettext(" action: The pool can be "
2215                             "imported using its name or numeric "
2216                             "identifier.\n"));
2217                 }
2218         } else if (vs->vs_state == VDEV_STATE_DEGRADED) {
2219                 (void) printf(gettext(" action: The pool can be imported "
2220                     "despite missing or damaged devices.  The\n\tfault "
2221                     "tolerance of the pool may be compromised if imported.\n"));
2222         } else {
2223                 switch (reason) {
2224                 case ZPOOL_STATUS_VERSION_NEWER:
2225                         (void) printf(gettext(" action: The pool cannot be "
2226                             "imported.  Access the pool on a system running "
2227                             "newer\n\tsoftware, or recreate the pool from "
2228                             "backup.\n"));
2229                         break;
2230                 case ZPOOL_STATUS_UNSUP_FEAT_READ:
2231                         (void) printf(gettext("action: The pool cannot be "
2232                             "imported. Access the pool on a system that "
2233                             "supports\n\tthe required feature(s), or recreate "
2234                             "the pool from backup.\n"));
2235                         break;
2236                 case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2237                         (void) printf(gettext("action: The pool cannot be "
2238                             "imported in read-write mode. Import the pool "
2239                             "with\n"
2240                             "\t\"-o readonly=on\", access the pool on a system "
2241                             "that supports the\n\trequired feature(s), or "
2242                             "recreate the pool from backup.\n"));
2243                         break;
2244                 case ZPOOL_STATUS_MISSING_DEV_R:
2245                 case ZPOOL_STATUS_MISSING_DEV_NR:
2246                 case ZPOOL_STATUS_BAD_GUID_SUM:
2247                         (void) printf(gettext(" action: The pool cannot be "
2248                             "imported. Attach the missing\n\tdevices and try "
2249                             "again.\n"));
2250                         break;
2251                 case ZPOOL_STATUS_HOSTID_ACTIVE:
2252                         VERIFY0(nvlist_lookup_nvlist(config,
2253                             ZPOOL_CONFIG_LOAD_INFO, &nvinfo));
2254
2255                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2256                                 hostname = fnvlist_lookup_string(nvinfo,
2257                                     ZPOOL_CONFIG_MMP_HOSTNAME);
2258
2259                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2260                                 hostid = fnvlist_lookup_uint64(nvinfo,
2261                                     ZPOOL_CONFIG_MMP_HOSTID);
2262
2263                         (void) printf(gettext(" action: The pool must be "
2264                             "exported from %s (hostid=%lx)\n\tbefore it "
2265                             "can be safely imported.\n"), hostname,
2266                             (unsigned long) hostid);
2267                         break;
2268                 case ZPOOL_STATUS_HOSTID_REQUIRED:
2269                         (void) printf(gettext(" action: Set a unique system "
2270                             "hostid with the zgenhostid(8) command.\n"));
2271                         break;
2272                 default:
2273                         (void) printf(gettext(" action: The pool cannot be "
2274                             "imported due to damaged devices or data.\n"));
2275                 }
2276         }
2277
2278         /* Print the comment attached to the pool. */
2279         if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
2280                 (void) printf(gettext("comment: %s\n"), comment);
2281
2282         /*
2283          * If the state is "closed" or "can't open", and the aux state
2284          * is "corrupt data":
2285          */
2286         if (((vs->vs_state == VDEV_STATE_CLOSED) ||
2287             (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
2288             (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
2289                 if (pool_state == POOL_STATE_DESTROYED)
2290                         (void) printf(gettext("\tThe pool was destroyed, "
2291                             "but can be imported using the '-Df' flags.\n"));
2292                 else if (pool_state != POOL_STATE_EXPORTED)
2293                         (void) printf(gettext("\tThe pool may be active on "
2294                             "another system, but can be imported using\n\t"
2295                             "the '-f' flag.\n"));
2296         }
2297
2298         if (msgid != NULL)
2299                 (void) printf(gettext("   see: http://zfsonlinux.org/msg/%s\n"),
2300                     msgid);
2301
2302         (void) printf(gettext(" config:\n\n"));
2303
2304         cb.cb_namewidth = max_width(NULL, nvroot, 0, strlen(name),
2305             VDEV_NAME_TYPE_ID);
2306         if (cb.cb_namewidth < 10)
2307                 cb.cb_namewidth = 10;
2308
2309         print_import_config(&cb, name, nvroot, 0);
2310         if (num_logs(nvroot) > 0)
2311                 print_logs(NULL, &cb, nvroot);
2312
2313         if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
2314                 (void) printf(gettext("\n\tAdditional devices are known to "
2315                     "be part of this pool, though their\n\texact "
2316                     "configuration cannot be determined.\n"));
2317         }
2318 }
2319
2320 static boolean_t
2321 zfs_force_import_required(nvlist_t *config)
2322 {
2323         uint64_t state;
2324         uint64_t hostid = 0;
2325         nvlist_t *nvinfo;
2326
2327         state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
2328         (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
2329
2330         if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
2331                 return (B_TRUE);
2332
2333         nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2334         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
2335                 mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
2336                     ZPOOL_CONFIG_MMP_STATE);
2337
2338                 if (mmp_state != MMP_STATE_INACTIVE)
2339                         return (B_TRUE);
2340         }
2341
2342         return (B_FALSE);
2343 }
2344
2345 /*
2346  * Perform the import for the given configuration.  This passes the heavy
2347  * lifting off to zpool_import_props(), and then mounts the datasets contained
2348  * within the pool.
2349  */
2350 static int
2351 do_import(nvlist_t *config, const char *newname, const char *mntopts,
2352     nvlist_t *props, int flags)
2353 {
2354         int ret = 0;
2355         zpool_handle_t *zhp;
2356         char *name;
2357         uint64_t version;
2358
2359         name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
2360         version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
2361
2362         if (!SPA_VERSION_IS_SUPPORTED(version)) {
2363                 (void) fprintf(stderr, gettext("cannot import '%s': pool "
2364                     "is formatted using an unsupported ZFS version\n"), name);
2365                 return (1);
2366         } else if (zfs_force_import_required(config) &&
2367             !(flags & ZFS_IMPORT_ANY_HOST)) {
2368                 mmp_state_t mmp_state = MMP_STATE_INACTIVE;
2369                 nvlist_t *nvinfo;
2370
2371                 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2372                 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
2373                         mmp_state = fnvlist_lookup_uint64(nvinfo,
2374                             ZPOOL_CONFIG_MMP_STATE);
2375
2376                 if (mmp_state == MMP_STATE_ACTIVE) {
2377                         char *hostname = "<unknown>";
2378                         uint64_t hostid = 0;
2379
2380                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2381                                 hostname = fnvlist_lookup_string(nvinfo,
2382                                     ZPOOL_CONFIG_MMP_HOSTNAME);
2383
2384                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2385                                 hostid = fnvlist_lookup_uint64(nvinfo,
2386                                     ZPOOL_CONFIG_MMP_HOSTID);
2387
2388                         (void) fprintf(stderr, gettext("cannot import '%s': "
2389                             "pool is imported on %s (hostid: "
2390                             "0x%lx)\nExport the pool on the other system, "
2391                             "then run 'zpool import'.\n"),
2392                             name, hostname, (unsigned long) hostid);
2393                 } else if (mmp_state == MMP_STATE_NO_HOSTID) {
2394                         (void) fprintf(stderr, gettext("Cannot import '%s': "
2395                             "pool has the multihost property on and the\n"
2396                             "system's hostid is not set. Set a unique hostid "
2397                             "with the zgenhostid(8) command.\n"), name);
2398                 } else {
2399                         char *hostname = "<unknown>";
2400                         uint64_t timestamp = 0;
2401                         uint64_t hostid = 0;
2402
2403                         if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
2404                                 hostname = fnvlist_lookup_string(config,
2405                                     ZPOOL_CONFIG_HOSTNAME);
2406
2407                         if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
2408                                 timestamp = fnvlist_lookup_uint64(config,
2409                                     ZPOOL_CONFIG_TIMESTAMP);
2410
2411                         if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
2412                                 hostid = fnvlist_lookup_uint64(config,
2413                                     ZPOOL_CONFIG_HOSTID);
2414
2415                         (void) fprintf(stderr, gettext("cannot import '%s': "
2416                             "pool was previously in use from another system.\n"
2417                             "Last accessed by %s (hostid=%lx) at %s"
2418                             "The pool can be imported, use 'zpool import -f' "
2419                             "to import the pool.\n"), name, hostname,
2420                             (unsigned long)hostid, ctime((time_t *)&timestamp));
2421                 }
2422
2423                 return (1);
2424         }
2425
2426         if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
2427                 return (1);
2428
2429         if (newname != NULL)
2430                 name = (char *)newname;
2431
2432         if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
2433                 return (1);
2434
2435         /*
2436          * Loading keys is best effort. We don't want to return immediately
2437          * if it fails but we do want to give the error to the caller.
2438          */
2439         if (flags & ZFS_IMPORT_LOAD_KEYS) {
2440                 ret = zfs_crypto_attempt_load_keys(g_zfs, name);
2441                 if (ret != 0)
2442                         ret = 1;
2443         }
2444
2445         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
2446             !(flags & ZFS_IMPORT_ONLY) &&
2447             zpool_enable_datasets(zhp, mntopts, 0) != 0) {
2448                 zpool_close(zhp);
2449                 return (1);
2450         }
2451
2452         zpool_close(zhp);
2453         return (ret);
2454 }
2455
2456 /*
2457  * zpool import [-d dir] [-D]
2458  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
2459  *              [-d dir | -c cachefile] [-f] -a
2460  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
2461  *              [-d dir | -c cachefile] [-f] [-n] [-F] <pool | id> [newpool]
2462  *
2463  *       -c     Read pool information from a cachefile instead of searching
2464  *              devices.
2465  *
2466  *       -d     Scan in a specific directory, other than /dev/.  More than
2467  *              one directory can be specified using multiple '-d' options.
2468  *
2469  *       -D     Scan for previously destroyed pools or import all or only
2470  *              specified destroyed pools.
2471  *
2472  *       -R     Temporarily import the pool, with all mountpoints relative to
2473  *              the given root.  The pool will remain exported when the machine
2474  *              is rebooted.
2475  *
2476  *       -V     Import even in the presence of faulted vdevs.  This is an
2477  *              intentionally undocumented option for testing purposes, and
2478  *              treats the pool configuration as complete, leaving any bad
2479  *              vdevs in the FAULTED state. In other words, it does verbatim
2480  *              import.
2481  *
2482  *       -f     Force import, even if it appears that the pool is active.
2483  *
2484  *       -F     Attempt rewind if necessary.
2485  *
2486  *       -n     See if rewind would work, but don't actually rewind.
2487  *
2488  *       -N     Import the pool but don't mount datasets.
2489  *
2490  *       -T     Specify a starting txg to use for import. This option is
2491  *              intentionally undocumented option for testing purposes.
2492  *
2493  *       -a     Import all pools found.
2494  *
2495  *       -l     Load encryption keys while importing.
2496  *
2497  *       -o     Set property=value and/or temporary mount options (without '=').
2498  *
2499  *       -s     Scan using the default search path, the libblkid cache will
2500  *              not be consulted.
2501  *
2502  * The import command scans for pools to import, and import pools based on pool
2503  * name and GUID.  The pool can also be renamed as part of the import process.
2504  */
2505 int
2506 zpool_do_import(int argc, char **argv)
2507 {
2508         char **searchdirs = NULL;
2509         char *env, *envdup = NULL;
2510         int nsearch = 0;
2511         int c;
2512         int err = 0;
2513         nvlist_t *pools = NULL;
2514         boolean_t do_all = B_FALSE;
2515         boolean_t do_destroyed = B_FALSE;
2516         char *mntopts = NULL;
2517         nvpair_t *elem;
2518         nvlist_t *config;
2519         uint64_t searchguid = 0;
2520         char *searchname = NULL;
2521         char *propval;
2522         nvlist_t *found_config;
2523         nvlist_t *policy = NULL;
2524         nvlist_t *props = NULL;
2525         boolean_t first;
2526         int flags = ZFS_IMPORT_NORMAL;
2527         uint32_t rewind_policy = ZPOOL_NO_REWIND;
2528         boolean_t dryrun = B_FALSE;
2529         boolean_t do_rewind = B_FALSE;
2530         boolean_t xtreme_rewind = B_FALSE;
2531         boolean_t do_scan = B_FALSE;
2532         uint64_t pool_state, txg = -1ULL;
2533         char *cachefile = NULL;
2534         importargs_t idata = { 0 };
2535         char *endptr;
2536
2537         /* check options */
2538         while ((c = getopt(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX")) != -1) {
2539                 switch (c) {
2540                 case 'a':
2541                         do_all = B_TRUE;
2542                         break;
2543                 case 'c':
2544                         cachefile = optarg;
2545                         break;
2546                 case 'd':
2547                         if (searchdirs == NULL) {
2548                                 searchdirs = safe_malloc(sizeof (char *));
2549                         } else {
2550                                 char **tmp = safe_malloc((nsearch + 1) *
2551                                     sizeof (char *));
2552                                 bcopy(searchdirs, tmp, nsearch *
2553                                     sizeof (char *));
2554                                 free(searchdirs);
2555                                 searchdirs = tmp;
2556                         }
2557                         searchdirs[nsearch++] = optarg;
2558                         break;
2559                 case 'D':
2560                         do_destroyed = B_TRUE;
2561                         break;
2562                 case 'f':
2563                         flags |= ZFS_IMPORT_ANY_HOST;
2564                         break;
2565                 case 'F':
2566                         do_rewind = B_TRUE;
2567                         break;
2568                 case 'l':
2569                         flags |= ZFS_IMPORT_LOAD_KEYS;
2570                         break;
2571                 case 'm':
2572                         flags |= ZFS_IMPORT_MISSING_LOG;
2573                         break;
2574                 case 'n':
2575                         dryrun = B_TRUE;
2576                         break;
2577                 case 'N':
2578                         flags |= ZFS_IMPORT_ONLY;
2579                         break;
2580                 case 'o':
2581                         if ((propval = strchr(optarg, '=')) != NULL) {
2582                                 *propval = '\0';
2583                                 propval++;
2584                                 if (add_prop_list(optarg, propval,
2585                                     &props, B_TRUE))
2586                                         goto error;
2587                         } else {
2588                                 mntopts = optarg;
2589                         }
2590                         break;
2591                 case 'R':
2592                         if (add_prop_list(zpool_prop_to_name(
2593                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
2594                                 goto error;
2595                         if (add_prop_list_default(zpool_prop_to_name(
2596                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
2597                                 goto error;
2598                         break;
2599                 case 's':
2600                         do_scan = B_TRUE;
2601                         break;
2602                 case 't':
2603                         flags |= ZFS_IMPORT_TEMP_NAME;
2604                         if (add_prop_list_default(zpool_prop_to_name(
2605                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
2606                                 goto error;
2607                         break;
2608
2609                 case 'T':
2610                         errno = 0;
2611                         txg = strtoull(optarg, &endptr, 0);
2612                         if (errno != 0 || *endptr != '\0') {
2613                                 (void) fprintf(stderr,
2614                                     gettext("invalid txg value\n"));
2615                                 usage(B_FALSE);
2616                         }
2617                         rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
2618                         break;
2619                 case 'V':
2620                         flags |= ZFS_IMPORT_VERBATIM;
2621                         break;
2622                 case 'X':
2623                         xtreme_rewind = B_TRUE;
2624                         break;
2625                 case ':':
2626                         (void) fprintf(stderr, gettext("missing argument for "
2627                             "'%c' option\n"), optopt);
2628                         usage(B_FALSE);
2629                         break;
2630                 case '?':
2631                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
2632                             optopt);
2633                         usage(B_FALSE);
2634                 }
2635         }
2636
2637         argc -= optind;
2638         argv += optind;
2639
2640         if (cachefile && nsearch != 0) {
2641                 (void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
2642                 usage(B_FALSE);
2643         }
2644
2645         if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
2646                 (void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
2647                 usage(B_FALSE);
2648         }
2649
2650         if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
2651                 (void) fprintf(stderr, gettext("-l is only meaningful during "
2652                     "an import\n"));
2653                 usage(B_FALSE);
2654         }
2655
2656         if ((dryrun || xtreme_rewind) && !do_rewind) {
2657                 (void) fprintf(stderr,
2658                     gettext("-n or -X only meaningful with -F\n"));
2659                 usage(B_FALSE);
2660         }
2661         if (dryrun)
2662                 rewind_policy = ZPOOL_TRY_REWIND;
2663         else if (do_rewind)
2664                 rewind_policy = ZPOOL_DO_REWIND;
2665         if (xtreme_rewind)
2666                 rewind_policy |= ZPOOL_EXTREME_REWIND;
2667
2668         /* In the future, we can capture further policy and include it here */
2669         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
2670             nvlist_add_uint64(policy, ZPOOL_REWIND_REQUEST_TXG, txg) != 0 ||
2671             nvlist_add_uint32(policy, ZPOOL_REWIND_REQUEST, rewind_policy) != 0)
2672                 goto error;
2673
2674         /* check argument count */
2675         if (do_all) {
2676                 if (argc != 0) {
2677                         (void) fprintf(stderr, gettext("too many arguments\n"));
2678                         usage(B_FALSE);
2679                 }
2680         } else {
2681                 if (argc > 2) {
2682                         (void) fprintf(stderr, gettext("too many arguments\n"));
2683                         usage(B_FALSE);
2684                 }
2685         }
2686
2687         /*
2688          * Check for the effective uid.  We do this explicitly here because
2689          * otherwise any attempt to discover pools will silently fail.
2690          */
2691         if (argc == 0 && geteuid() != 0) {
2692                 (void) fprintf(stderr, gettext("cannot "
2693                     "discover pools: permission denied\n"));
2694                 if (searchdirs != NULL)
2695                         free(searchdirs);
2696
2697                 nvlist_free(props);
2698                 nvlist_free(policy);
2699                 return (1);
2700         }
2701
2702         /*
2703          * Depending on the arguments given, we do one of the following:
2704          *
2705          *      <none>  Iterate through all pools and display information about
2706          *              each one.
2707          *
2708          *      -a      Iterate through all pools and try to import each one.
2709          *
2710          *      <id>    Find the pool that corresponds to the given GUID/pool
2711          *              name and import that one.
2712          *
2713          *      -D      Above options applies only to destroyed pools.
2714          */
2715         if (argc != 0) {
2716                 char *endptr;
2717
2718                 errno = 0;
2719                 searchguid = strtoull(argv[0], &endptr, 10);
2720                 if (errno != 0 || *endptr != '\0') {
2721                         searchname = argv[0];
2722                         searchguid = 0;
2723                 }
2724                 found_config = NULL;
2725
2726                 /*
2727                  * User specified a name or guid.  Ensure it's unique.
2728                  */
2729                 idata.unique = B_TRUE;
2730         }
2731
2732         /*
2733          * Check the environment for the preferred search path.
2734          */
2735         if ((searchdirs == NULL) && (env = getenv("ZPOOL_IMPORT_PATH"))) {
2736                 char *dir;
2737
2738                 envdup = strdup(env);
2739
2740                 dir = strtok(envdup, ":");
2741                 while (dir != NULL) {
2742                         if (searchdirs == NULL) {
2743                                 searchdirs = safe_malloc(sizeof (char *));
2744                         } else {
2745                                 char **tmp = safe_malloc((nsearch + 1) *
2746                                     sizeof (char *));
2747                                 bcopy(searchdirs, tmp, nsearch *
2748                                     sizeof (char *));
2749                                 free(searchdirs);
2750                                 searchdirs = tmp;
2751                         }
2752                         searchdirs[nsearch++] = dir;
2753                         dir = strtok(NULL, ":");
2754                 }
2755         }
2756
2757         idata.path = searchdirs;
2758         idata.paths = nsearch;
2759         idata.poolname = searchname;
2760         idata.guid = searchguid;
2761         idata.cachefile = cachefile;
2762         idata.scan = do_scan;
2763         idata.policy = policy;
2764
2765         pools = zpool_search_import(g_zfs, &idata);
2766
2767         if (pools != NULL && idata.exists &&
2768             (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
2769                 (void) fprintf(stderr, gettext("cannot import '%s': "
2770                     "a pool with that name already exists\n"),
2771                     argv[0]);
2772                 (void) fprintf(stderr, gettext("use the form '%s "
2773                     "<pool | id> <newpool>' to give it a new name\n"),
2774                     "zpool import");
2775                 err = 1;
2776         } else if (pools == NULL && idata.exists) {
2777                 (void) fprintf(stderr, gettext("cannot import '%s': "
2778                     "a pool with that name is already created/imported,\n"),
2779                     argv[0]);
2780                 (void) fprintf(stderr, gettext("and no additional pools "
2781                     "with that name were found\n"));
2782                 err = 1;
2783         } else if (pools == NULL) {
2784                 if (argc != 0) {
2785                         (void) fprintf(stderr, gettext("cannot import '%s': "
2786                             "no such pool available\n"), argv[0]);
2787                 }
2788                 err = 1;
2789         }
2790
2791         if (err == 1) {
2792                 if (searchdirs != NULL)
2793                         free(searchdirs);
2794                 if (envdup != NULL)
2795                         free(envdup);
2796                 nvlist_free(policy);
2797                 nvlist_free(pools);
2798                 nvlist_free(props);
2799                 return (1);
2800         }
2801
2802         /*
2803          * At this point we have a list of import candidate configs. Even if
2804          * we were searching by pool name or guid, we still need to
2805          * post-process the list to deal with pool state and possible
2806          * duplicate names.
2807          */
2808         err = 0;
2809         elem = NULL;
2810         first = B_TRUE;
2811         while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
2812
2813                 verify(nvpair_value_nvlist(elem, &config) == 0);
2814
2815                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2816                     &pool_state) == 0);
2817                 if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
2818                         continue;
2819                 if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
2820                         continue;
2821
2822                 verify(nvlist_add_nvlist(config, ZPOOL_REWIND_POLICY,
2823                     policy) == 0);
2824
2825                 if (argc == 0) {
2826                         if (first)
2827                                 first = B_FALSE;
2828                         else if (!do_all)
2829                                 (void) printf("\n");
2830
2831                         if (do_all) {
2832                                 err |= do_import(config, NULL, mntopts,
2833                                     props, flags);
2834                         } else {
2835                                 show_import(config);
2836                         }
2837                 } else if (searchname != NULL) {
2838                         char *name;
2839
2840                         /*
2841                          * We are searching for a pool based on name.
2842                          */
2843                         verify(nvlist_lookup_string(config,
2844                             ZPOOL_CONFIG_POOL_NAME, &name) == 0);
2845
2846                         if (strcmp(name, searchname) == 0) {
2847                                 if (found_config != NULL) {
2848                                         (void) fprintf(stderr, gettext(
2849                                             "cannot import '%s': more than "
2850                                             "one matching pool\n"), searchname);
2851                                         (void) fprintf(stderr, gettext(
2852                                             "import by numeric ID instead\n"));
2853                                         err = B_TRUE;
2854                                 }
2855                                 found_config = config;
2856                         }
2857                 } else {
2858                         uint64_t guid;
2859
2860                         /*
2861                          * Search for a pool by guid.
2862                          */
2863                         verify(nvlist_lookup_uint64(config,
2864                             ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
2865
2866                         if (guid == searchguid)
2867                                 found_config = config;
2868                 }
2869         }
2870
2871         /*
2872          * If we were searching for a specific pool, verify that we found a
2873          * pool, and then do the import.
2874          */
2875         if (argc != 0 && err == 0) {
2876                 if (found_config == NULL) {
2877                         (void) fprintf(stderr, gettext("cannot import '%s': "
2878                             "no such pool available\n"), argv[0]);
2879                         err = B_TRUE;
2880                 } else {
2881                         err |= do_import(found_config, argc == 1 ? NULL :
2882                             argv[1], mntopts, props, flags);
2883                 }
2884         }
2885
2886         /*
2887          * If we were just looking for pools, report an error if none were
2888          * found.
2889          */
2890         if (argc == 0 && first)
2891                 (void) fprintf(stderr,
2892                     gettext("no pools available to import\n"));
2893
2894 error:
2895         nvlist_free(props);
2896         nvlist_free(pools);
2897         nvlist_free(policy);
2898         if (searchdirs != NULL)
2899                 free(searchdirs);
2900         if (envdup != NULL)
2901                 free(envdup);
2902
2903         return (err ? 1 : 0);
2904 }
2905
2906 /*
2907  * zpool sync [-f] [pool] ...
2908  *
2909  * -f (undocumented) force uberblock (and config including zpool cache file)
2910  *    update.
2911  *
2912  * Sync the specified pool(s).
2913  * Without arguments "zpool sync" will sync all pools.
2914  * This command initiates TXG sync(s) and will return after the TXG(s) commit.
2915  *
2916  */
2917 static int
2918 zpool_do_sync(int argc, char **argv)
2919 {
2920         int ret;
2921         boolean_t force = B_FALSE;
2922
2923         /* check options */
2924         while ((ret  = getopt(argc, argv, "f")) != -1) {
2925                 switch (ret) {
2926                 case 'f':
2927                         force = B_TRUE;
2928                         break;
2929                 case '?':
2930                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
2931                             optopt);
2932                         usage(B_FALSE);
2933                 }
2934         }
2935
2936         argc -= optind;
2937         argv += optind;
2938
2939         /* if argc == 0 we will execute zpool_sync_one on all pools */
2940         ret = for_each_pool(argc, argv, B_FALSE, NULL, zpool_sync_one, &force);
2941
2942         return (ret);
2943 }
2944
2945 typedef struct iostat_cbdata {
2946         uint64_t cb_flags;
2947         int cb_name_flags;
2948         int cb_namewidth;
2949         int cb_iteration;
2950         char **cb_vdev_names; /* Only show these vdevs */
2951         unsigned int cb_vdev_names_count;
2952         boolean_t cb_verbose;
2953         boolean_t cb_literal;
2954         boolean_t cb_scripted;
2955         zpool_list_t *cb_list;
2956         vdev_cmd_data_list_t *vcdl;
2957 } iostat_cbdata_t;
2958
2959 /*  iostat labels */
2960 typedef struct name_and_columns {
2961         const char *name;       /* Column name */
2962         unsigned int columns;   /* Center name to this number of columns */
2963 } name_and_columns_t;
2964
2965 #define IOSTAT_MAX_LABELS       11      /* Max number of labels on one line */
2966
2967 static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] =
2968 {
2969         [IOS_DEFAULT] = {{"capacity", 2}, {"operations", 2}, {"bandwidth", 2},
2970             {NULL}},
2971         [IOS_LATENCY] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
2972             {"asyncq_wait", 2}, {"scrub"}},
2973         [IOS_QUEUES] = {{"syncq_read", 2}, {"syncq_write", 2},
2974             {"asyncq_read", 2}, {"asyncq_write", 2}, {"scrubq_read", 2},
2975             {NULL}},
2976         [IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2},
2977             {"sync_queue", 2}, {"async_queue", 2}, {NULL}},
2978         [IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2},
2979             {"async_read", 2}, {"async_write", 2}, {"scrub", 2}, {NULL}},
2980
2981 };
2982
2983 /* Shorthand - if "columns" field not set, default to 1 column */
2984 static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] =
2985 {
2986         [IOS_DEFAULT] = {{"alloc"}, {"free"}, {"read"}, {"write"}, {"read"},
2987             {"write"}, {NULL}},
2988         [IOS_LATENCY] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
2989             {"write"}, {"read"}, {"write"}, {"wait"}, {NULL}},
2990         [IOS_QUEUES] = {{"pend"}, {"activ"}, {"pend"}, {"activ"}, {"pend"},
2991             {"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"}, {NULL}},
2992         [IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
2993             {"write"}, {"read"}, {"write"}, {"scrub"}, {NULL}},
2994         [IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
2995             {"ind"}, {"agg"}, {"ind"}, {"agg"}, {NULL}},
2996 };
2997
2998 static const char *histo_to_title[] = {
2999         [IOS_L_HISTO] = "latency",
3000         [IOS_RQ_HISTO] = "req_size",
3001 };
3002
3003 /*
3004  * Return the number of labels in a null-terminated name_and_columns_t
3005  * array.
3006  *
3007  */
3008 static unsigned int
3009 label_array_len(const name_and_columns_t *labels)
3010 {
3011         int i = 0;
3012
3013         while (labels[i].name)
3014                 i++;
3015
3016         return (i);
3017 }
3018
3019 /*
3020  * Return the number of strings in a null-terminated string array.
3021  * For example:
3022  *
3023  *     const char foo[] = {"bar", "baz", NULL}
3024  *
3025  * returns 2
3026  */
3027 static uint64_t
3028 str_array_len(const char *array[])
3029 {
3030         uint64_t i = 0;
3031         while (array[i])
3032                 i++;
3033
3034         return (i);
3035 }
3036
3037
3038 /*
3039  * Return a default column width for default/latency/queue columns. This does
3040  * not include histograms, which have their columns autosized.
3041  */
3042 static unsigned int
3043 default_column_width(iostat_cbdata_t *cb, enum iostat_type type)
3044 {
3045         unsigned long column_width = 5; /* Normal niceprint */
3046         static unsigned long widths[] = {
3047                 /*
3048                  * Choose some sane default column sizes for printing the
3049                  * raw numbers.
3050                  */
3051                 [IOS_DEFAULT] = 15, /* 1PB capacity */
3052                 [IOS_LATENCY] = 10, /* 1B ns = 10sec */
3053                 [IOS_QUEUES] = 6,   /* 1M queue entries */
3054         };
3055
3056         if (cb->cb_literal)
3057                 column_width = widths[type];
3058
3059         return (column_width);
3060 }
3061
3062 /*
3063  * Print the column labels, i.e:
3064  *
3065  *   capacity     operations     bandwidth
3066  * alloc   free   read  write   read  write  ...
3067  *
3068  * If force_column_width is set, use it for the column width.  If not set, use
3069  * the default column width.
3070  */
3071 void
3072 print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
3073     const name_and_columns_t labels[][IOSTAT_MAX_LABELS])
3074 {
3075         int i, idx, s;
3076         unsigned int text_start, rw_column_width, spaces_to_end;
3077         uint64_t flags = cb->cb_flags;
3078         uint64_t f;
3079         unsigned int column_width = force_column_width;
3080
3081         /* For each bit set in flags */
3082         for (f = flags; f; f &= ~(1ULL << idx)) {
3083                 idx = lowbit64(f) - 1;
3084                 if (!force_column_width)
3085                         column_width = default_column_width(cb, idx);
3086                 /* Print our top labels centered over "read  write" label. */
3087                 for (i = 0; i < label_array_len(labels[idx]); i++) {
3088                         const char *name = labels[idx][i].name;
3089                         /*
3090                          * We treat labels[][].columns == 0 as shorthand
3091                          * for one column.  It makes writing out the label
3092                          * tables more concise.
3093                          */
3094                         unsigned int columns = MAX(1, labels[idx][i].columns);
3095                         unsigned int slen = strlen(name);
3096
3097                         rw_column_width = (column_width * columns) +
3098                             (2 * (columns - 1));
3099
3100                         text_start = (int)((rw_column_width)/columns -
3101                             slen/columns);
3102
3103                         printf("  ");   /* Two spaces between columns */
3104
3105                         /* Space from beginning of column to label */
3106                         for (s = 0; s < text_start; s++)
3107                                 printf(" ");
3108
3109                         printf("%s", name);
3110
3111                         /* Print space after label to end of column */
3112                         spaces_to_end = rw_column_width - text_start - slen;
3113                         for (s = 0; s < spaces_to_end; s++)
3114                                 printf(" ");
3115
3116                 }
3117         }
3118 }
3119
3120
3121 /*
3122  * print_cmd_columns - Print custom column titles from -c
3123  *
3124  * If the user specified the "zpool status|iostat -c" then print their custom
3125  * column titles in the header.  For example, print_cmd_columns() would print
3126  * the "  col1  col2" part of this:
3127  *
3128  * $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
3129  * ...
3130  *            capacity     operations     bandwidth
3131  * pool        alloc   free   read  write   read  write  col1  col2
3132  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
3133  * mypool       269K  1008M      0      0    107    946
3134  *   mirror     269K  1008M      0      0    107    946
3135  *     sdb         -      -      0      0    102    473  val1  val2
3136  *     sdc         -      -      0      0      5    473  val1  val2
3137  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
3138  */
3139 void
3140 print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
3141 {
3142         int i, j;
3143         vdev_cmd_data_t *data = &vcdl->data[0];
3144
3145         if (vcdl->count == 0 || data == NULL)
3146                 return;
3147
3148         /*
3149          * Each vdev cmd should have the same column names unless the user did
3150          * something weird with their cmd.  Just take the column names from the
3151          * first vdev and assume it works for all of them.
3152          */
3153         for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
3154                 printf("  ");
3155                 if (use_dashes) {
3156                         for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
3157                                 printf("-");
3158                 } else {
3159                         printf("%*s", vcdl->uniq_cols_width[i],
3160                             vcdl->uniq_cols[i]);
3161                 }
3162         }
3163 }
3164
3165
3166 /*
3167  * Utility function to print out a line of dashes like:
3168  *
3169  *      --------------------------------  -----  -----  -----  -----  -----
3170  *
3171  * ...or a dashed named-row line like:
3172  *
3173  *      logs                                  -      -      -      -      -
3174  *
3175  * @cb:                         iostat data
3176  *
3177  * @force_column_width          If non-zero, use the value as the column width.
3178  *                              Otherwise use the default column widths.
3179  *
3180  * @name:                       Print a dashed named-row line starting
3181  *                              with @name.  Otherwise, print a regular
3182  *                              dashed line.
3183  */
3184 static void
3185 print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
3186     const char *name)
3187 {
3188         int i;
3189         unsigned int namewidth;
3190         uint64_t flags = cb->cb_flags;
3191         uint64_t f;
3192         int idx;
3193         const name_and_columns_t *labels;
3194         const char *title;
3195
3196
3197         if (cb->cb_flags & IOS_ANYHISTO_M) {
3198                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3199         } else if (cb->cb_vdev_names_count) {
3200                 title = "vdev";
3201         } else  {
3202                 title = "pool";
3203         }
3204
3205         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3206             name ? strlen(name) : 0);
3207
3208
3209         if (name) {
3210                 printf("%-*s", namewidth, name);
3211         } else {
3212                 for (i = 0; i < namewidth; i++)
3213                         (void) printf("-");
3214         }
3215
3216         /* For each bit in flags */
3217         for (f = flags; f; f &= ~(1ULL << idx)) {
3218                 unsigned int column_width;
3219                 idx = lowbit64(f) - 1;
3220                 if (force_column_width)
3221                         column_width = force_column_width;
3222                 else
3223                         column_width = default_column_width(cb, idx);
3224
3225                 labels = iostat_bottom_labels[idx];
3226                 for (i = 0; i < label_array_len(labels); i++) {
3227                         if (name)
3228                                 printf("  %*s-", column_width - 1, " ");
3229                         else
3230                                 printf("  %.*s", column_width,
3231                                     "--------------------");
3232                 }
3233         }
3234 }
3235
3236
3237 static void
3238 print_iostat_separator_impl(iostat_cbdata_t *cb,
3239     unsigned int force_column_width)
3240 {
3241         print_iostat_dashes(cb, force_column_width, NULL);
3242 }
3243
3244 static void
3245 print_iostat_separator(iostat_cbdata_t *cb)
3246 {
3247         print_iostat_separator_impl(cb, 0);
3248 }
3249
3250 static void
3251 print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
3252     const char *histo_vdev_name)
3253 {
3254         unsigned int namewidth;
3255         const char *title;
3256
3257         if (cb->cb_flags & IOS_ANYHISTO_M) {
3258                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3259         } else if (cb->cb_vdev_names_count) {
3260                 title = "vdev";
3261         } else  {
3262                 title = "pool";
3263         }
3264
3265         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3266             histo_vdev_name ? strlen(histo_vdev_name) : 0);
3267
3268         if (histo_vdev_name)
3269                 printf("%-*s", namewidth, histo_vdev_name);
3270         else
3271                 printf("%*s", namewidth, "");
3272
3273
3274         print_iostat_labels(cb, force_column_width, iostat_top_labels);
3275         printf("\n");
3276
3277         printf("%-*s", namewidth, title);
3278
3279         print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
3280         if (cb->vcdl != NULL)
3281                 print_cmd_columns(cb->vcdl, 0);
3282
3283         printf("\n");
3284
3285         print_iostat_separator_impl(cb, force_column_width);
3286
3287         if (cb->vcdl != NULL)
3288                 print_cmd_columns(cb->vcdl, 1);
3289
3290         printf("\n");
3291 }
3292
3293 static void
3294 print_iostat_header(iostat_cbdata_t *cb)
3295 {
3296         print_iostat_header_impl(cb, 0, NULL);
3297 }
3298
3299
3300 /*
3301  * Display a single statistic.
3302  */
3303 static void
3304 print_one_stat(uint64_t value, enum zfs_nicenum_format format,
3305     unsigned int column_size, boolean_t scripted)
3306 {
3307         char buf[64];
3308
3309         zfs_nicenum_format(value, buf, sizeof (buf), format);
3310
3311         if (scripted)
3312                 printf("\t%s", buf);
3313         else
3314                 printf("  %*s", column_size, buf);
3315 }
3316
3317 /*
3318  * Calculate the default vdev stats
3319  *
3320  * Subtract oldvs from newvs, apply a scaling factor, and save the resulting
3321  * stats into calcvs.
3322  */
3323 static void
3324 calc_default_iostats(vdev_stat_t *oldvs, vdev_stat_t *newvs,
3325     vdev_stat_t *calcvs)
3326 {
3327         int i;
3328
3329         memcpy(calcvs, newvs, sizeof (*calcvs));
3330         for (i = 0; i < ARRAY_SIZE(calcvs->vs_ops); i++)
3331                 calcvs->vs_ops[i] = (newvs->vs_ops[i] - oldvs->vs_ops[i]);
3332
3333         for (i = 0; i < ARRAY_SIZE(calcvs->vs_bytes); i++)
3334                 calcvs->vs_bytes[i] = (newvs->vs_bytes[i] - oldvs->vs_bytes[i]);
3335 }
3336
3337 /*
3338  * Internal representation of the extended iostats data.
3339  *
3340  * The extended iostat stats are exported in nvlists as either uint64_t arrays
3341  * or single uint64_t's.  We make both look like arrays to make them easier
3342  * to process.  In order to make single uint64_t's look like arrays, we set
3343  * __data to the stat data, and then set *data = &__data with count = 1.  Then,
3344  * we can just use *data and count.
3345  */
3346 struct stat_array {
3347         uint64_t *data;
3348         uint_t count;   /* Number of entries in data[] */
3349         uint64_t __data; /* Only used when data is a single uint64_t */
3350 };
3351
3352 static uint64_t
3353 stat_histo_max(struct stat_array *nva, unsigned int len)
3354 {
3355         uint64_t max = 0;
3356         int i;
3357         for (i = 0; i < len; i++)
3358                 max = MAX(max, array64_max(nva[i].data, nva[i].count));
3359
3360         return (max);
3361 }
3362
3363 /*
3364  * Helper function to lookup a uint64_t array or uint64_t value and store its
3365  * data as a stat_array.  If the nvpair is a single uint64_t value, then we make
3366  * it look like a one element array to make it easier to process.
3367  */
3368 static int
3369 nvpair64_to_stat_array(nvlist_t *nvl, const char *name,
3370     struct stat_array *nva)
3371 {
3372         nvpair_t *tmp;
3373         int ret;
3374
3375         verify(nvlist_lookup_nvpair(nvl, name, &tmp) == 0);
3376         switch (nvpair_type(tmp)) {
3377         case DATA_TYPE_UINT64_ARRAY:
3378                 ret = nvpair_value_uint64_array(tmp, &nva->data, &nva->count);
3379                 break;
3380         case DATA_TYPE_UINT64:
3381                 ret = nvpair_value_uint64(tmp, &nva->__data);
3382                 nva->data = &nva->__data;
3383                 nva->count = 1;
3384                 break;
3385         default:
3386                 /* Not a uint64_t */
3387                 ret = EINVAL;
3388                 break;
3389         }
3390
3391         return (ret);
3392 }
3393
3394 /*
3395  * Given a list of nvlist names, look up the extended stats in newnv and oldnv,
3396  * subtract them, and return the results in a newly allocated stat_array.
3397  * You must free the returned array after you are done with it with
3398  * free_calc_stats().
3399  *
3400  * Additionally, you can set "oldnv" to NULL if you simply want the newnv
3401  * values.
3402  */
3403 static struct stat_array *
3404 calc_and_alloc_stats_ex(const char **names, unsigned int len, nvlist_t *oldnv,
3405     nvlist_t *newnv)
3406 {
3407         nvlist_t *oldnvx = NULL, *newnvx;
3408         struct stat_array *oldnva, *newnva, *calcnva;
3409         int i, j;
3410         unsigned int alloc_size = (sizeof (struct stat_array)) * len;
3411
3412         /* Extract our extended stats nvlist from the main list */
3413         verify(nvlist_lookup_nvlist(newnv, ZPOOL_CONFIG_VDEV_STATS_EX,
3414             &newnvx) == 0);
3415         if (oldnv) {
3416                 verify(nvlist_lookup_nvlist(oldnv, ZPOOL_CONFIG_VDEV_STATS_EX,
3417                     &oldnvx) == 0);
3418         }
3419
3420         newnva = safe_malloc(alloc_size);
3421         oldnva = safe_malloc(alloc_size);
3422         calcnva = safe_malloc(alloc_size);
3423
3424         for (j = 0; j < len; j++) {
3425                 verify(nvpair64_to_stat_array(newnvx, names[j],
3426                     &newnva[j]) == 0);
3427                 calcnva[j].count = newnva[j].count;
3428                 alloc_size = calcnva[j].count * sizeof (calcnva[j].data[0]);
3429                 calcnva[j].data = safe_malloc(alloc_size);
3430                 memcpy(calcnva[j].data, newnva[j].data, alloc_size);
3431
3432                 if (oldnvx) {
3433                         verify(nvpair64_to_stat_array(oldnvx, names[j],
3434                             &oldnva[j]) == 0);
3435                         for (i = 0; i < oldnva[j].count; i++)
3436                                 calcnva[j].data[i] -= oldnva[j].data[i];
3437                 }
3438         }
3439         free(newnva);
3440         free(oldnva);
3441         return (calcnva);
3442 }
3443
3444 static void
3445 free_calc_stats(struct stat_array *nva, unsigned int len)
3446 {
3447         int i;
3448         for (i = 0; i < len; i++)
3449                 free(nva[i].data);
3450
3451         free(nva);
3452 }
3453
3454 static void
3455 print_iostat_histo(struct stat_array *nva, unsigned int len,
3456     iostat_cbdata_t *cb, unsigned int column_width, unsigned int namewidth,
3457     double scale)
3458 {
3459         int i, j;
3460         char buf[6];
3461         uint64_t val;
3462         enum zfs_nicenum_format format;
3463         unsigned int buckets;
3464         unsigned int start_bucket;
3465
3466         if (cb->cb_literal)
3467                 format = ZFS_NICENUM_RAW;
3468         else
3469                 format = ZFS_NICENUM_1024;
3470
3471         /* All these histos are the same size, so just use nva[0].count */
3472         buckets = nva[0].count;
3473
3474         if (cb->cb_flags & IOS_RQ_HISTO_M) {
3475                 /* Start at 512 - req size should never be lower than this */
3476                 start_bucket = 9;
3477         } else {
3478                 start_bucket = 0;
3479         }
3480
3481         for (j = start_bucket; j < buckets; j++) {
3482                 /* Print histogram bucket label */
3483                 if (cb->cb_flags & IOS_L_HISTO_M) {
3484                         /* Ending range of this bucket */
3485                         val = (1UL << (j + 1)) - 1;
3486                         zfs_nicetime(val, buf, sizeof (buf));
3487                 } else {
3488                         /* Request size (starting range of bucket) */
3489                         val = (1UL << j);
3490                         zfs_nicenum(val, buf, sizeof (buf));
3491                 }
3492
3493                 if (cb->cb_scripted)
3494                         printf("%llu", (u_longlong_t)val);
3495                 else
3496                         printf("%-*s", namewidth, buf);
3497
3498                 /* Print the values on the line */
3499                 for (i = 0; i < len; i++) {
3500                         print_one_stat(nva[i].data[j] * scale, format,
3501                             column_width, cb->cb_scripted);
3502                 }
3503                 printf("\n");
3504         }
3505 }
3506
3507 static void
3508 print_solid_separator(unsigned int length)
3509 {
3510         while (length--)
3511                 printf("-");
3512         printf("\n");
3513 }
3514
3515 static void
3516 print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv,
3517     nvlist_t *newnv, double scale, const char *name)
3518 {
3519         unsigned int column_width;
3520         unsigned int namewidth;
3521         unsigned int entire_width;
3522         enum iostat_type type;
3523         struct stat_array *nva;
3524         const char **names;
3525         unsigned int names_len;
3526
3527         /* What type of histo are we? */
3528         type = IOS_HISTO_IDX(cb->cb_flags);
3529
3530         /* Get NULL-terminated array of nvlist names for our histo */
3531         names = vsx_type_to_nvlist[type];
3532         names_len = str_array_len(names); /* num of names */
3533
3534         nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv);
3535
3536         if (cb->cb_literal) {
3537                 column_width = MAX(5,
3538                     (unsigned int) log10(stat_histo_max(nva, names_len)) + 1);
3539         } else {
3540                 column_width = 5;
3541         }
3542
3543         namewidth = MAX(cb->cb_namewidth,
3544             strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]));
3545
3546         /*
3547          * Calculate the entire line width of what we're printing.  The
3548          * +2 is for the two spaces between columns:
3549          */
3550         /*       read  write                            */
3551         /*      -----  -----                            */
3552         /*      |___|  <---------- column_width         */
3553         /*                                              */
3554         /*      |__________|  <--- entire_width         */
3555         /*                                              */
3556         entire_width = namewidth + (column_width + 2) *
3557             label_array_len(iostat_bottom_labels[type]);
3558
3559         if (cb->cb_scripted)
3560                 printf("%s\n", name);
3561         else
3562                 print_iostat_header_impl(cb, column_width, name);
3563
3564         print_iostat_histo(nva, names_len, cb, column_width,
3565             namewidth, scale);
3566
3567         free_calc_stats(nva, names_len);
3568         if (!cb->cb_scripted)
3569                 print_solid_separator(entire_width);
3570 }
3571
3572 /*
3573  * Calculate the average latency of a power-of-two latency histogram
3574  */
3575 static uint64_t
3576 single_histo_average(uint64_t *histo, unsigned int buckets)
3577 {
3578         int i;
3579         uint64_t count = 0, total = 0;
3580
3581         for (i = 0; i < buckets; i++) {
3582                 /*
3583                  * Our buckets are power-of-two latency ranges.  Use the
3584                  * midpoint latency of each bucket to calculate the average.
3585                  * For example:
3586                  *
3587                  * Bucket          Midpoint
3588                  * 8ns-15ns:       12ns
3589                  * 16ns-31ns:      24ns
3590                  * ...
3591                  */
3592                 if (histo[i] != 0) {
3593                         total += histo[i] * (((1UL << i) + ((1UL << i)/2)));
3594                         count += histo[i];
3595                 }
3596         }
3597
3598         /* Prevent divide by zero */
3599         return (count == 0 ? 0 : total / count);
3600 }
3601
3602 static void
3603 print_iostat_queues(iostat_cbdata_t *cb, nvlist_t *oldnv,
3604     nvlist_t *newnv, double scale)
3605 {
3606         int i;
3607         uint64_t val;
3608         const char *names[] = {
3609                 ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
3610                 ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
3611                 ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
3612                 ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
3613                 ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
3614                 ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
3615                 ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
3616                 ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
3617                 ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
3618                 ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
3619         };
3620
3621         struct stat_array *nva;
3622
3623         unsigned int column_width = default_column_width(cb, IOS_QUEUES);
3624         enum zfs_nicenum_format format;
3625
3626         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), NULL, newnv);
3627
3628         if (cb->cb_literal)
3629                 format = ZFS_NICENUM_RAW;
3630         else
3631                 format = ZFS_NICENUM_1024;
3632
3633         for (i = 0; i < ARRAY_SIZE(names); i++) {
3634                 val = nva[i].data[0] * scale;
3635                 print_one_stat(val, format, column_width, cb->cb_scripted);
3636         }
3637
3638         free_calc_stats(nva, ARRAY_SIZE(names));
3639 }
3640
3641 static void
3642 print_iostat_latency(iostat_cbdata_t *cb, nvlist_t *oldnv,
3643     nvlist_t *newnv, double scale)
3644 {
3645         int i;
3646         uint64_t val;
3647         const char *names[] = {
3648                 ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
3649                 ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
3650                 ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
3651                 ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
3652                 ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
3653                 ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
3654                 ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
3655                 ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
3656                 ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
3657         };
3658         struct stat_array *nva;
3659
3660         unsigned int column_width = default_column_width(cb, IOS_LATENCY);
3661         enum zfs_nicenum_format format;
3662
3663         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv);
3664
3665         if (cb->cb_literal)
3666                 format = ZFS_NICENUM_RAWTIME;
3667         else
3668                 format = ZFS_NICENUM_TIME;
3669
3670         /* Print our avg latencies on the line */
3671         for (i = 0; i < ARRAY_SIZE(names); i++) {
3672                 /* Compute average latency for a latency histo */
3673                 val = single_histo_average(nva[i].data, nva[i].count) * scale;
3674                 print_one_stat(val, format, column_width, cb->cb_scripted);
3675         }
3676         free_calc_stats(nva, ARRAY_SIZE(names));
3677 }
3678
3679 /*
3680  * Print default statistics (capacity/operations/bandwidth)
3681  */
3682 static void
3683 print_iostat_default(vdev_stat_t *vs, iostat_cbdata_t *cb, double scale)
3684 {
3685         unsigned int column_width = default_column_width(cb, IOS_DEFAULT);
3686         enum zfs_nicenum_format format;
3687         char na;        /* char to print for "not applicable" values */
3688
3689         if (cb->cb_literal) {
3690                 format = ZFS_NICENUM_RAW;
3691                 na = '0';
3692         } else {
3693                 format = ZFS_NICENUM_1024;
3694                 na = '-';
3695         }
3696
3697         /* only toplevel vdevs have capacity stats */
3698         if (vs->vs_space == 0) {
3699                 if (cb->cb_scripted)
3700                         printf("\t%c\t%c", na, na);
3701                 else
3702                         printf("  %*c  %*c", column_width, na, column_width,
3703                             na);
3704         } else {
3705                 print_one_stat(vs->vs_alloc, format, column_width,
3706                     cb->cb_scripted);
3707                 print_one_stat(vs->vs_space - vs->vs_alloc, format,
3708                     column_width, cb->cb_scripted);
3709         }
3710
3711         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_READ] * scale),
3712             format, column_width, cb->cb_scripted);
3713         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_WRITE] * scale),
3714             format, column_width, cb->cb_scripted);
3715         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_READ] * scale),
3716             format, column_width, cb->cb_scripted);
3717         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_WRITE] * scale),
3718             format, column_width, cb->cb_scripted);
3719 }
3720
3721 /*
3722  * Print out all the statistics for the given vdev.  This can either be the
3723  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
3724  * is a verbose output, and we don't want to display the toplevel pool stats.
3725  *
3726  * Returns the number of stat lines printed.
3727  */
3728 unsigned int
3729 print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
3730     nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
3731 {
3732         nvlist_t **oldchild, **newchild;
3733         uint_t c, children, oldchildren;
3734         vdev_stat_t *oldvs, *newvs, *calcvs;
3735         vdev_stat_t zerovs = { 0 };
3736         char *vname;
3737         int i;
3738         int ret = 0;
3739         uint64_t tdelta;
3740         double scale;
3741
3742         calcvs = safe_malloc(sizeof (*calcvs));
3743
3744         if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
3745                 return (ret);
3746
3747         if (oldnv != NULL) {
3748                 verify(nvlist_lookup_uint64_array(oldnv,
3749                     ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
3750         } else {
3751                 oldvs = &zerovs;
3752         }
3753
3754         /* Do we only want to see a specific vdev? */
3755         for (i = 0; i < cb->cb_vdev_names_count; i++) {
3756                 /* Yes we do.  Is this the vdev? */
3757                 if (strcmp(name, cb->cb_vdev_names[i]) == 0) {
3758                         /*
3759                          * This is our vdev.  Since it is the only vdev we
3760                          * will be displaying, make depth = 0 so that it
3761                          * doesn't get indented.
3762                          */
3763                         depth = 0;
3764                         break;
3765                 }
3766         }
3767
3768         if (cb->cb_vdev_names_count && (i == cb->cb_vdev_names_count)) {
3769                 /* Couldn't match the name */
3770                 goto children;
3771         }
3772
3773
3774         verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
3775             (uint64_t **)&newvs, &c) == 0);
3776
3777         /*
3778          * Print the vdev name unless it's is a histogram.  Histograms
3779          * display the vdev name in the header itself.
3780          */
3781         if (!(cb->cb_flags & IOS_ANYHISTO_M)) {
3782                 if (cb->cb_scripted) {
3783                         printf("%s", name);
3784                 } else {
3785                         if (strlen(name) + depth > cb->cb_namewidth)
3786                                 (void) printf("%*s%s", depth, "", name);
3787                         else
3788                                 (void) printf("%*s%s%*s", depth, "", name,
3789                                     (int)(cb->cb_namewidth - strlen(name) -
3790                                     depth), "");
3791                 }
3792         }
3793
3794         /* Calculate our scaling factor */
3795         tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
3796         if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) {
3797                 /*
3798                  * If we specify printing histograms with no time interval, then
3799                  * print the histogram numbers over the entire lifetime of the
3800                  * vdev.
3801                  */
3802                 scale = 1;
3803         } else {
3804                 if (tdelta == 0)
3805                         scale = 1.0;
3806                 else
3807                         scale = (double)NANOSEC / tdelta;
3808         }
3809
3810         if (cb->cb_flags & IOS_DEFAULT_M) {
3811                 calc_default_iostats(oldvs, newvs, calcvs);
3812                 print_iostat_default(calcvs, cb, scale);
3813         }
3814         if (cb->cb_flags & IOS_LATENCY_M)
3815                 print_iostat_latency(cb, oldnv, newnv, scale);
3816         if (cb->cb_flags & IOS_QUEUES_M)
3817                 print_iostat_queues(cb, oldnv, newnv, scale);
3818         if (cb->cb_flags & IOS_ANYHISTO_M) {
3819                 printf("\n");
3820                 print_iostat_histos(cb, oldnv, newnv, scale, name);
3821         }
3822
3823         if (cb->vcdl != NULL) {
3824                 char *path;
3825                 if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
3826                     &path) == 0) {
3827                         printf("  ");
3828                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
3829                 }
3830         }
3831
3832         if (!(cb->cb_flags & IOS_ANYHISTO_M))
3833                 printf("\n");
3834
3835         ret++;
3836
3837 children:
3838
3839         free(calcvs);
3840
3841         if (!cb->cb_verbose)
3842                 return (ret);
3843
3844         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
3845             &newchild, &children) != 0)
3846                 return (ret);
3847
3848         if (oldnv) {
3849                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
3850                     &oldchild, &oldchildren) != 0)
3851                         return (ret);
3852
3853                 children = MIN(oldchildren, children);
3854         }
3855
3856         for (c = 0; c < children; c++) {
3857                 uint64_t ishole = B_FALSE, islog = B_FALSE;
3858
3859                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
3860                     &ishole);
3861
3862                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
3863                     &islog);
3864
3865                 if (ishole || islog)
3866                         continue;
3867
3868                 vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
3869                     cb->cb_name_flags);
3870                 ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
3871                     newchild[c], cb, depth + 2);
3872                 free(vname);
3873         }
3874
3875         /*
3876          * Log device section
3877          */
3878
3879         if (num_logs(newnv) > 0) {
3880                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
3881                     !cb->cb_vdev_names) {
3882                         print_iostat_dashes(cb, 0, "logs");
3883                 }
3884                 printf("\n");
3885
3886                 for (c = 0; c < children; c++) {
3887                         uint64_t islog = B_FALSE;
3888                         (void) nvlist_lookup_uint64(newchild[c],
3889                             ZPOOL_CONFIG_IS_LOG, &islog);
3890
3891                         if (islog) {
3892                                 vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
3893                                     cb->cb_name_flags);
3894                                 ret += print_vdev_stats(zhp, vname, oldnv ?
3895                                     oldchild[c] : NULL, newchild[c],
3896                                     cb, depth + 2);
3897                                 free(vname);
3898                         }
3899                 }
3900
3901         }
3902
3903         /*
3904          * Include level 2 ARC devices in iostat output
3905          */
3906         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
3907             &newchild, &children) != 0)
3908                 return (ret);
3909
3910         if (oldnv) {
3911                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
3912                     &oldchild, &oldchildren) != 0)
3913                         return (ret);
3914
3915                 children = MIN(oldchildren, children);
3916         }
3917
3918         if (children > 0) {
3919                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
3920                     !cb->cb_vdev_names) {
3921                         print_iostat_dashes(cb, 0, "cache");
3922                 }
3923                 printf("\n");
3924
3925                 for (c = 0; c < children; c++) {
3926                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
3927                             cb->cb_name_flags);
3928                         ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c]
3929                             : NULL, newchild[c], cb, depth + 2);
3930                         free(vname);
3931                 }
3932         }
3933
3934         return (ret);
3935 }
3936
3937 static int
3938 refresh_iostat(zpool_handle_t *zhp, void *data)
3939 {
3940         iostat_cbdata_t *cb = data;
3941         boolean_t missing;
3942
3943         /*
3944          * If the pool has disappeared, remove it from the list and continue.
3945          */
3946         if (zpool_refresh_stats(zhp, &missing) != 0)
3947                 return (-1);
3948
3949         if (missing)
3950                 pool_list_remove(cb->cb_list, zhp);
3951
3952         return (0);
3953 }
3954
3955 /*
3956  * Callback to print out the iostats for the given pool.
3957  */
3958 int
3959 print_iostat(zpool_handle_t *zhp, void *data)
3960 {
3961         iostat_cbdata_t *cb = data;
3962         nvlist_t *oldconfig, *newconfig;
3963         nvlist_t *oldnvroot, *newnvroot;
3964         int ret;
3965
3966         newconfig = zpool_get_config(zhp, &oldconfig);
3967
3968         if (cb->cb_iteration == 1)
3969                 oldconfig = NULL;
3970
3971         verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
3972             &newnvroot) == 0);
3973
3974         if (oldconfig == NULL)
3975                 oldnvroot = NULL;
3976         else
3977                 verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
3978                     &oldnvroot) == 0);
3979
3980         ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot,
3981             cb, 0);
3982         if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
3983             !cb->cb_scripted && cb->cb_verbose && !cb->cb_vdev_names_count) {
3984                 print_iostat_separator(cb);
3985                 if (cb->vcdl != NULL) {
3986                         print_cmd_columns(cb->vcdl, 1);
3987                 }
3988                 printf("\n");
3989         }
3990
3991         return (ret);
3992 }
3993
3994 static int
3995 get_columns(void)
3996 {
3997         struct winsize ws;
3998         int columns = 80;
3999         int error;
4000
4001         if (isatty(STDOUT_FILENO)) {
4002                 error = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws);
4003                 if (error == 0)
4004                         columns = ws.ws_col;
4005         } else {
4006                 columns = 999;
4007         }
4008
4009         return (columns);
4010 }
4011
4012 int
4013 get_namewidth(zpool_handle_t *zhp, void *data)
4014 {
4015         iostat_cbdata_t *cb = data;
4016         nvlist_t *config, *nvroot;
4017         int columns;
4018
4019         if ((config = zpool_get_config(zhp, NULL)) != NULL) {
4020                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4021                     &nvroot) == 0);
4022                 unsigned int poolname_len = strlen(zpool_get_name(zhp));
4023                 if (!cb->cb_verbose)
4024                         cb->cb_namewidth = MAX(poolname_len, cb->cb_namewidth);
4025                 else
4026                         cb->cb_namewidth = MAX(poolname_len,
4027                             max_width(zhp, nvroot, 0, cb->cb_namewidth,
4028                             cb->cb_name_flags));
4029         }
4030         /*
4031          * The width must be at least 10, but may be as large as the
4032          * column width - 42 so that we can still fit in one line.
4033          */
4034         columns = get_columns();
4035
4036         if (cb->cb_namewidth < 10)
4037                 cb->cb_namewidth = 10;
4038         if (cb->cb_namewidth > columns - 42)
4039                 cb->cb_namewidth = columns - 42;
4040
4041         return (0);
4042 }
4043
4044 /*
4045  * Parse the input string, get the 'interval' and 'count' value if there is one.
4046  */
4047 static void
4048 get_interval_count(int *argcp, char **argv, float *iv,
4049     unsigned long *cnt)
4050 {
4051         float interval = 0;
4052         unsigned long count = 0;
4053         int argc = *argcp;
4054
4055         /*
4056          * Determine if the last argument is an integer or a pool name
4057          */
4058         if (argc > 0 && isnumber(argv[argc - 1])) {
4059                 char *end;
4060
4061                 errno = 0;
4062                 interval = strtof(argv[argc - 1], &end);
4063
4064                 if (*end == '\0' && errno == 0) {
4065                         if (interval == 0) {
4066                                 (void) fprintf(stderr, gettext("interval "
4067                                     "cannot be zero\n"));
4068                                 usage(B_FALSE);
4069                         }
4070                         /*
4071                          * Ignore the last parameter
4072                          */
4073                         argc--;
4074                 } else {
4075                         /*
4076                          * If this is not a valid number, just plow on.  The
4077                          * user will get a more informative error message later
4078                          * on.
4079                          */
4080                         interval = 0;
4081                 }
4082         }
4083
4084         /*
4085          * If the last argument is also an integer, then we have both a count
4086          * and an interval.
4087          */
4088         if (argc > 0 && isnumber(argv[argc - 1])) {
4089                 char *end;
4090
4091                 errno = 0;
4092                 count = interval;
4093                 interval = strtof(argv[argc - 1], &end);
4094
4095                 if (*end == '\0' && errno == 0) {
4096                         if (interval == 0) {
4097                                 (void) fprintf(stderr, gettext("interval "
4098                                     "cannot be zero\n"));
4099                                 usage(B_FALSE);
4100                         }
4101
4102                         /*
4103                          * Ignore the last parameter
4104                          */
4105                         argc--;
4106                 } else {
4107                         interval = 0;
4108                 }
4109         }
4110
4111         *iv = interval;
4112         *cnt = count;
4113         *argcp = argc;
4114 }
4115
4116 static void
4117 get_timestamp_arg(char c)
4118 {
4119         if (c == 'u')
4120                 timestamp_fmt = UDATE;
4121         else if (c == 'd')
4122                 timestamp_fmt = DDATE;
4123         else
4124                 usage(B_FALSE);
4125 }
4126
4127 /*
4128  * Return stat flags that are supported by all pools by both the module and
4129  * zpool iostat.  "*data" should be initialized to all 0xFFs before running.
4130  * It will get ANDed down until only the flags that are supported on all pools
4131  * remain.
4132  */
4133 static int
4134 get_stat_flags_cb(zpool_handle_t *zhp, void *data)
4135 {
4136         uint64_t *mask = data;
4137         nvlist_t *config, *nvroot, *nvx;
4138         uint64_t flags = 0;
4139         int i, j;
4140
4141         config = zpool_get_config(zhp, NULL);
4142         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4143             &nvroot) == 0);
4144
4145         /* Default stats are always supported, but for completeness.. */
4146         if (nvlist_exists(nvroot, ZPOOL_CONFIG_VDEV_STATS))
4147                 flags |= IOS_DEFAULT_M;
4148
4149         /* Get our extended stats nvlist from the main list */
4150         if (nvlist_lookup_nvlist(nvroot, ZPOOL_CONFIG_VDEV_STATS_EX,
4151             &nvx) != 0) {
4152                 /*
4153                  * No extended stats; they're probably running an older
4154                  * module.  No big deal, we support that too.
4155                  */
4156                 goto end;
4157         }
4158
4159         /* For each extended stat, make sure all its nvpairs are supported */
4160         for (j = 0; j < ARRAY_SIZE(vsx_type_to_nvlist); j++) {
4161                 if (!vsx_type_to_nvlist[j][0])
4162                         continue;
4163
4164                 /* Start off by assuming the flag is supported, then check */
4165                 flags |= (1ULL << j);
4166                 for (i = 0; vsx_type_to_nvlist[j][i]; i++) {
4167                         if (!nvlist_exists(nvx, vsx_type_to_nvlist[j][i])) {
4168                                 /* flag isn't supported */
4169                                 flags = flags & ~(1ULL  << j);
4170                                 break;
4171                         }
4172                 }
4173         }
4174 end:
4175         *mask = *mask & flags;
4176         return (0);
4177 }
4178
4179 /*
4180  * Return a bitmask of stats that are supported on all pools by both the module
4181  * and zpool iostat.
4182  */
4183 static uint64_t
4184 get_stat_flags(zpool_list_t *list)
4185 {
4186         uint64_t mask = -1;
4187
4188         /*
4189          * get_stat_flags_cb() will lop off bits from "mask" until only the
4190          * flags that are supported on all pools remain.
4191          */
4192         pool_list_iter(list, B_FALSE, get_stat_flags_cb, &mask);
4193         return (mask);
4194 }
4195
4196 /*
4197  * Return 1 if cb_data->cb_vdev_names[0] is this vdev's name, 0 otherwise.
4198  */
4199 static int
4200 is_vdev_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_data)
4201 {
4202         iostat_cbdata_t *cb = cb_data;
4203         char *name = NULL;
4204         int ret = 0;
4205
4206         name = zpool_vdev_name(g_zfs, zhp, nv, cb->cb_name_flags);
4207
4208         if (strcmp(name, cb->cb_vdev_names[0]) == 0)
4209                 ret = 1; /* match */
4210         free(name);
4211
4212         return (ret);
4213 }
4214
4215 /*
4216  * Returns 1 if cb_data->cb_vdev_names[0] is a vdev name, 0 otherwise.
4217  */
4218 static int
4219 is_vdev(zpool_handle_t *zhp, void *cb_data)
4220 {
4221         return (for_each_vdev(zhp, is_vdev_cb, cb_data));
4222 }
4223
4224 /*
4225  * Check if vdevs are in a pool
4226  *
4227  * Return 1 if all argv[] strings are vdev names in pool "pool_name". Otherwise
4228  * return 0.  If pool_name is NULL, then search all pools.
4229  */
4230 static int
4231 are_vdevs_in_pool(int argc, char **argv, char *pool_name,
4232     iostat_cbdata_t *cb)
4233 {
4234         char **tmp_name;
4235         int ret = 0;
4236         int i;
4237         int pool_count = 0;
4238
4239         if ((argc == 0) || !*argv)
4240                 return (0);
4241
4242         if (pool_name)
4243                 pool_count = 1;
4244
4245         /* Temporarily hijack cb_vdev_names for a second... */
4246         tmp_name = cb->cb_vdev_names;
4247
4248         /* Go though our list of prospective vdev names */
4249         for (i = 0; i < argc; i++) {
4250                 cb->cb_vdev_names = argv + i;
4251
4252                 /* Is this name a vdev in our pools? */
4253                 ret = for_each_pool(pool_count, &pool_name, B_TRUE, NULL,
4254                     is_vdev, cb);
4255                 if (!ret) {
4256                         /* No match */
4257                         break;
4258                 }
4259         }
4260
4261         cb->cb_vdev_names = tmp_name;
4262
4263         return (ret);
4264 }
4265
4266 static int
4267 is_pool_cb(zpool_handle_t *zhp, void *data)
4268 {
4269         char *name = data;
4270         if (strcmp(name, zpool_get_name(zhp)) == 0)
4271                 return (1);
4272
4273         return (0);
4274 }
4275
4276 /*
4277  * Do we have a pool named *name?  If so, return 1, otherwise 0.
4278  */
4279 static int
4280 is_pool(char *name)
4281 {
4282         return (for_each_pool(0, NULL, B_TRUE, NULL,  is_pool_cb, name));
4283 }
4284
4285 /* Are all our argv[] strings pool names?  If so return 1, 0 otherwise. */
4286 static int
4287 are_all_pools(int argc, char **argv)
4288 {
4289         if ((argc == 0) || !*argv)
4290                 return (0);
4291
4292         while (--argc >= 0)
4293                 if (!is_pool(argv[argc]))
4294                         return (0);
4295
4296         return (1);
4297 }
4298
4299 /*
4300  * Helper function to print out vdev/pool names we can't resolve.  Used for an
4301  * error message.
4302  */
4303 static void
4304 error_list_unresolved_vdevs(int argc, char **argv, char *pool_name,
4305     iostat_cbdata_t *cb)
4306 {
4307         int i;
4308         char *name;
4309         char *str;
4310         for (i = 0; i < argc; i++) {
4311                 name = argv[i];
4312
4313                 if (is_pool(name))
4314                         str = gettext("pool");
4315                 else if (are_vdevs_in_pool(1, &name, pool_name, cb))
4316                         str = gettext("vdev in this pool");
4317                 else if (are_vdevs_in_pool(1, &name, NULL, cb))
4318                         str = gettext("vdev in another pool");
4319                 else
4320                         str = gettext("unknown");
4321
4322                 fprintf(stderr, "\t%s (%s)\n", name, str);
4323         }
4324 }
4325
4326 /*
4327  * Same as get_interval_count(), but with additional checks to not misinterpret
4328  * guids as interval/count values.  Assumes VDEV_NAME_GUID is set in
4329  * cb.cb_name_flags.
4330  */
4331 static void
4332 get_interval_count_filter_guids(int *argc, char **argv, float *interval,
4333     unsigned long *count, iostat_cbdata_t *cb)
4334 {
4335         char **tmpargv = argv;
4336         int argc_for_interval = 0;
4337
4338         /* Is the last arg an interval value?  Or a guid? */
4339         if (*argc >= 1 && !are_vdevs_in_pool(1, &argv[*argc - 1], NULL, cb)) {
4340                 /*
4341                  * The last arg is not a guid, so it's probably an
4342                  * interval value.
4343                  */
4344                 argc_for_interval++;
4345
4346                 if (*argc >= 2 &&
4347                     !are_vdevs_in_pool(1, &argv[*argc - 2], NULL, cb)) {
4348                         /*
4349                          * The 2nd to last arg is not a guid, so it's probably
4350                          * an interval value.
4351                          */
4352                         argc_for_interval++;
4353                 }
4354         }
4355
4356         /* Point to our list of possible intervals */
4357         tmpargv = &argv[*argc - argc_for_interval];
4358
4359         *argc = *argc - argc_for_interval;
4360         get_interval_count(&argc_for_interval, tmpargv,
4361             interval, count);
4362 }
4363
4364 /*
4365  * Floating point sleep().  Allows you to pass in a floating point value for
4366  * seconds.
4367  */
4368 static void
4369 fsleep(float sec)
4370 {
4371         struct timespec req;
4372         req.tv_sec = floor(sec);
4373         req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
4374         nanosleep(&req, NULL);
4375 }
4376
4377 /*
4378  * Run one of the zpool status/iostat -c scripts with the help (-h) option and
4379  * print the result.
4380  *
4381  * name:        Short name of the script ('iostat').
4382  * path:        Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
4383  */
4384 static void
4385 print_zpool_script_help(char *name, char *path)
4386 {
4387         char *argv[] = {path, "-h", NULL};
4388         char **lines = NULL;
4389         int lines_cnt = 0;
4390         int rc;
4391
4392         rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
4393             &lines_cnt);
4394         if (rc != 0 || lines == NULL || lines_cnt <= 0) {
4395                 if (lines != NULL)
4396                         libzfs_free_str_array(lines, lines_cnt);
4397                 return;
4398         }
4399
4400         for (int i = 0; i < lines_cnt; i++)
4401                 if (!is_blank_str(lines[i]))
4402                         printf("  %-14s  %s\n", name, lines[i]);
4403
4404         libzfs_free_str_array(lines, lines_cnt);
4405 }
4406
4407 /*
4408  * Go though the zpool status/iostat -c scripts in the user's path, run their
4409  * help option (-h), and print out the results.
4410  */
4411 static void
4412 print_zpool_dir_scripts(char *dirpath)
4413 {
4414         DIR *dir;
4415         struct dirent *ent;
4416         char fullpath[MAXPATHLEN];
4417         struct stat dir_stat;
4418
4419         if ((dir = opendir(dirpath)) != NULL) {
4420                 /* print all the files and directories within directory */
4421                 while ((ent = readdir(dir)) != NULL) {
4422                         sprintf(fullpath, "%s/%s", dirpath, ent->d_name);
4423
4424                         /* Print the scripts */
4425                         if (stat(fullpath, &dir_stat) == 0)
4426                                 if (dir_stat.st_mode & S_IXUSR &&
4427                                     S_ISREG(dir_stat.st_mode))
4428                                         print_zpool_script_help(ent->d_name,
4429                                             fullpath);
4430                 }
4431                 closedir(dir);
4432         }
4433 }
4434
4435 /*
4436  * Print out help text for all zpool status/iostat -c scripts.
4437  */
4438 static void
4439 print_zpool_script_list(char *subcommand)
4440 {
4441         char *dir, *sp;
4442
4443         printf(gettext("Available 'zpool %s -c' commands:\n"), subcommand);
4444
4445         sp = zpool_get_cmd_search_path();
4446         if (sp == NULL)
4447                 return;
4448
4449         dir = strtok(sp, ":");
4450         while (dir != NULL) {
4451                 print_zpool_dir_scripts(dir);
4452                 dir = strtok(NULL, ":");
4453         }
4454
4455         free(sp);
4456 }
4457
4458 /*
4459  * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
4460  *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
4461  *              [interval [count]]
4462  *
4463  *      -c CMD  For each vdev, run command CMD
4464  *      -g      Display guid for individual vdev name.
4465  *      -L      Follow links when resolving vdev path name.
4466  *      -P      Display full path for vdev name.
4467  *      -v      Display statistics for individual vdevs
4468  *      -h      Display help
4469  *      -p      Display values in parsable (exact) format.
4470  *      -H      Scripted mode.  Don't display headers, and separate properties
4471  *              by a single tab.
4472  *      -l      Display average latency
4473  *      -q      Display queue depths
4474  *      -w      Display latency histograms
4475  *      -r      Display request size histogram
4476  *      -T      Display a timestamp in date(1) or Unix format
4477  *
4478  * This command can be tricky because we want to be able to deal with pool
4479  * creation/destruction as well as vdev configuration changes.  The bulk of this
4480  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
4481  * on pool_list_update() to detect the addition of new pools.  Configuration
4482  * changes are all handled within libzfs.
4483  */
4484 int
4485 zpool_do_iostat(int argc, char **argv)
4486 {
4487         int c;
4488         int ret;
4489         int npools;
4490         float interval = 0;
4491         unsigned long count = 0;
4492         zpool_list_t *list;
4493         boolean_t verbose = B_FALSE;
4494         boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE;
4495         boolean_t queues = B_FALSE, parsable = B_FALSE, scripted = B_FALSE;
4496         boolean_t omit_since_boot = B_FALSE;
4497         boolean_t guid = B_FALSE;
4498         boolean_t follow_links = B_FALSE;
4499         boolean_t full_name = B_FALSE;
4500         iostat_cbdata_t cb = { 0 };
4501         char *cmd = NULL;
4502
4503         /* Used for printing error message */
4504         const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q',
4505             [IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'};
4506
4507         uint64_t unsupported_flags;
4508
4509         /* check options */
4510         while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwH")) != -1) {
4511                 switch (c) {
4512                 case 'c':
4513                         if (cmd != NULL) {
4514                                 fprintf(stderr,
4515                                     gettext("Can't set -c flag twice\n"));
4516                                 exit(1);
4517                         }
4518
4519                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
4520                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
4521                                 fprintf(stderr, gettext(
4522                                     "Can't run -c, disabled by "
4523                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
4524                                 exit(1);
4525                         }
4526
4527                         if ((getuid() <= 0 || geteuid() <= 0) &&
4528                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
4529                                 fprintf(stderr, gettext(
4530                                     "Can't run -c with root privileges "
4531                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
4532                                 exit(1);
4533                         }
4534                         cmd = optarg;
4535                         verbose = B_TRUE;
4536                         break;
4537                 case 'g':
4538                         guid = B_TRUE;
4539                         break;
4540                 case 'L':
4541                         follow_links = B_TRUE;
4542                         break;
4543                 case 'P':
4544                         full_name = B_TRUE;
4545                         break;
4546                 case 'T':
4547                         get_timestamp_arg(*optarg);
4548                         break;
4549                 case 'v':
4550                         verbose = B_TRUE;
4551                         break;
4552                 case 'p':
4553                         parsable = B_TRUE;
4554                         break;
4555                 case 'l':
4556                         latency = B_TRUE;
4557                         break;
4558                 case 'q':
4559                         queues = B_TRUE;
4560                         break;
4561                 case 'H':
4562                         scripted = B_TRUE;
4563                         break;
4564                 case 'w':
4565                         l_histo = B_TRUE;
4566                         break;
4567                 case 'r':
4568                         rq_histo = B_TRUE;
4569                         break;
4570                 case 'y':
4571                         omit_since_boot = B_TRUE;
4572                         break;
4573                 case 'h':
4574                         usage(B_FALSE);
4575                         break;
4576                 case '?':
4577                         if (optopt == 'c') {
4578                                 print_zpool_script_list("iostat");
4579                                 exit(0);
4580                         } else {
4581                                 fprintf(stderr,
4582                                     gettext("invalid option '%c'\n"), optopt);
4583                         }
4584                         usage(B_FALSE);
4585                 }
4586         }
4587
4588         argc -= optind;
4589         argv += optind;
4590
4591         cb.cb_literal = parsable;
4592         cb.cb_scripted = scripted;
4593
4594         if (guid)
4595                 cb.cb_name_flags |= VDEV_NAME_GUID;
4596         if (follow_links)
4597                 cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
4598         if (full_name)
4599                 cb.cb_name_flags |= VDEV_NAME_PATH;
4600         cb.cb_iteration = 0;
4601         cb.cb_namewidth = 0;
4602         cb.cb_verbose = verbose;
4603
4604         /* Get our interval and count values (if any) */
4605         if (guid) {
4606                 get_interval_count_filter_guids(&argc, argv, &interval,
4607                     &count, &cb);
4608         } else {
4609                 get_interval_count(&argc, argv, &interval, &count);
4610         }
4611
4612         if (argc == 0) {
4613                 /* No args, so just print the defaults. */
4614         } else if (are_all_pools(argc, argv)) {
4615                 /* All the args are pool names */
4616         } else if (are_vdevs_in_pool(argc, argv, NULL, &cb)) {
4617                 /* All the args are vdevs */
4618                 cb.cb_vdev_names = argv;
4619                 cb.cb_vdev_names_count = argc;
4620                 argc = 0; /* No pools to process */
4621         } else if (are_all_pools(1, argv)) {
4622                 /* The first arg is a pool name */
4623                 if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0], &cb)) {
4624                         /* ...and the rest are vdev names */
4625                         cb.cb_vdev_names = argv + 1;
4626                         cb.cb_vdev_names_count = argc - 1;
4627                         argc = 1; /* One pool to process */
4628                 } else {
4629                         fprintf(stderr, gettext("Expected either a list of "));
4630                         fprintf(stderr, gettext("pools, or list of vdevs in"));
4631                         fprintf(stderr, " \"%s\", ", argv[0]);
4632                         fprintf(stderr, gettext("but got:\n"));
4633                         error_list_unresolved_vdevs(argc - 1, argv + 1,
4634                             argv[0], &cb);
4635                         fprintf(stderr, "\n");
4636                         usage(B_FALSE);
4637                         return (1);
4638                 }
4639         } else {
4640                 /*
4641                  * The args don't make sense. The first arg isn't a pool name,
4642                  * nor are all the args vdevs.
4643                  */
4644                 fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n"));
4645                 fprintf(stderr, "\n");
4646                 return (1);
4647         }
4648
4649         if (cb.cb_vdev_names_count != 0) {
4650                 /*
4651                  * If user specified vdevs, it implies verbose.
4652                  */
4653                 cb.cb_verbose = B_TRUE;
4654         }
4655
4656         /*
4657          * Construct the list of all interesting pools.
4658          */
4659         ret = 0;
4660         if ((list = pool_list_get(argc, argv, NULL, &ret)) == NULL)
4661                 return (1);
4662
4663         if (pool_list_count(list) == 0 && argc != 0) {
4664                 pool_list_free(list);
4665                 return (1);
4666         }
4667
4668         if (pool_list_count(list) == 0 && interval == 0) {
4669                 pool_list_free(list);
4670                 (void) fprintf(stderr, gettext("no pools available\n"));
4671                 return (1);
4672         }
4673
4674         if ((l_histo || rq_histo) && (cmd != NULL || latency || queues)) {
4675                 pool_list_free(list);
4676                 (void) fprintf(stderr,
4677                     gettext("[-r|-w] isn't allowed with [-c|-l|-q]\n"));
4678                 usage(B_FALSE);
4679                 return (1);
4680         }
4681
4682         if (l_histo && rq_histo) {
4683                 pool_list_free(list);
4684                 (void) fprintf(stderr,
4685                     gettext("Only one of [-r|-w] can be passed at a time\n"));
4686                 usage(B_FALSE);
4687                 return (1);
4688         }
4689
4690         /*
4691          * Enter the main iostat loop.
4692          */
4693         cb.cb_list = list;
4694
4695         if (l_histo) {
4696                 /*
4697                  * Histograms tables look out of place when you try to display
4698                  * them with the other stats, so make a rule that you can only
4699                  * print histograms by themselves.
4700                  */
4701                 cb.cb_flags = IOS_L_HISTO_M;
4702         } else if (rq_histo) {
4703                 cb.cb_flags = IOS_RQ_HISTO_M;
4704         } else {
4705                 cb.cb_flags = IOS_DEFAULT_M;
4706                 if (latency)
4707                         cb.cb_flags |= IOS_LATENCY_M;
4708                 if (queues)
4709                         cb.cb_flags |= IOS_QUEUES_M;
4710         }
4711
4712         /*
4713          * See if the module supports all the stats we want to display.
4714          */
4715         unsupported_flags = cb.cb_flags & ~get_stat_flags(list);
4716         if (unsupported_flags) {
4717                 uint64_t f;
4718                 int idx;
4719                 fprintf(stderr,
4720                     gettext("The loaded zfs module doesn't support:"));
4721
4722                 /* for each bit set in unsupported_flags */
4723                 for (f = unsupported_flags; f; f &= ~(1ULL << idx)) {
4724                         idx = lowbit64(f) - 1;
4725                         fprintf(stderr, " -%c", flag_to_arg[idx]);
4726                 }
4727
4728                 fprintf(stderr, ".  Try running a newer module.\n");
4729                 pool_list_free(list);
4730
4731                 return (1);
4732         }
4733
4734         for (;;) {
4735                 if ((npools = pool_list_count(list)) == 0)
4736                         (void) fprintf(stderr, gettext("no pools available\n"));
4737                 else {
4738                         /*
4739                          * If this is the first iteration and -y was supplied
4740                          * we skip any printing.
4741                          */
4742                         boolean_t skip = (omit_since_boot &&
4743                             cb.cb_iteration == 0);
4744
4745                         /*
4746                          * Refresh all statistics.  This is done as an
4747                          * explicit step before calculating the maximum name
4748                          * width, so that any * configuration changes are
4749                          * properly accounted for.
4750                          */
4751                         (void) pool_list_iter(list, B_FALSE, refresh_iostat,
4752                             &cb);
4753
4754                         /*
4755                          * Iterate over all pools to determine the maximum width
4756                          * for the pool / device name column across all pools.
4757                          */
4758                         cb.cb_namewidth = 0;
4759                         (void) pool_list_iter(list, B_FALSE, get_namewidth,
4760                             &cb);
4761
4762                         if (timestamp_fmt != NODATE)
4763                                 print_timestamp(timestamp_fmt);
4764
4765                         if (cmd != NULL && cb.cb_verbose &&
4766                             !(cb.cb_flags & IOS_ANYHISTO_M)) {
4767                                 cb.vcdl = all_pools_for_each_vdev_run(argc,
4768                                     argv, cmd, g_zfs, cb.cb_vdev_names,
4769                                     cb.cb_vdev_names_count, cb.cb_name_flags);
4770                         } else {
4771                                 cb.vcdl = NULL;
4772                         }
4773
4774                         /*
4775                          * If it's the first time and we're not skipping it,
4776                          * or either skip or verbose mode, print the header.
4777                          *
4778                          * The histogram code explicitly prints its header on
4779                          * every vdev, so skip this for histograms.
4780                          */
4781                         if (((++cb.cb_iteration == 1 && !skip) ||
4782                             (skip != verbose)) &&
4783                             (!(cb.cb_flags & IOS_ANYHISTO_M)) &&
4784                             !cb.cb_scripted)
4785                                 print_iostat_header(&cb);
4786
4787                         if (skip) {
4788                                 (void) fsleep(interval);
4789                                 continue;
4790                         }
4791
4792
4793                         pool_list_iter(list, B_FALSE, print_iostat, &cb);
4794
4795                         /*
4796                          * If there's more than one pool, and we're not in
4797                          * verbose mode (which prints a separator for us),
4798                          * then print a separator.
4799                          *
4800                          * In addition, if we're printing specific vdevs then
4801                          * we also want an ending separator.
4802                          */
4803                         if (((npools > 1 && !verbose &&
4804                             !(cb.cb_flags & IOS_ANYHISTO_M)) ||
4805                             (!(cb.cb_flags & IOS_ANYHISTO_M) &&
4806                             cb.cb_vdev_names_count)) &&
4807                             !cb.cb_scripted) {
4808                                 print_iostat_separator(&cb);
4809                                 if (cb.vcdl != NULL)
4810                                         print_cmd_columns(cb.vcdl, 1);
4811                                 printf("\n");
4812                         }
4813
4814                         if (cb.vcdl != NULL)
4815                                 free_vdev_cmd_data_list(cb.vcdl);
4816
4817                 }
4818
4819                 /*
4820                  * Flush the output so that redirection to a file isn't buffered
4821                  * indefinitely.
4822                  */
4823                 (void) fflush(stdout);
4824
4825                 if (interval == 0)
4826                         break;
4827
4828                 if (count != 0 && --count == 0)
4829                         break;
4830
4831                 (void) fsleep(interval);
4832         }
4833
4834         pool_list_free(list);
4835
4836         return (ret);
4837 }
4838
4839 typedef struct list_cbdata {
4840         boolean_t       cb_verbose;
4841         int             cb_name_flags;
4842         int             cb_namewidth;
4843         boolean_t       cb_scripted;
4844         zprop_list_t    *cb_proplist;
4845         boolean_t       cb_literal;
4846 } list_cbdata_t;
4847
4848 /*
4849  * Given a list of columns to display, output appropriate headers for each one.
4850  */
4851 static void
4852 print_header(list_cbdata_t *cb)
4853 {
4854         zprop_list_t *pl = cb->cb_proplist;
4855         char headerbuf[ZPOOL_MAXPROPLEN];
4856         const char *header;
4857         boolean_t first = B_TRUE;
4858         boolean_t right_justify;
4859         size_t width = 0;
4860
4861         for (; pl != NULL; pl = pl->pl_next) {
4862                 width = pl->pl_width;
4863                 if (first && cb->cb_verbose) {
4864                         /*
4865                          * Reset the width to accommodate the verbose listing
4866                          * of devices.
4867                          */
4868                         width = cb->cb_namewidth;
4869                 }
4870
4871                 if (!first)
4872                         (void) printf("  ");
4873                 else
4874                         first = B_FALSE;
4875
4876                 right_justify = B_FALSE;
4877                 if (pl->pl_prop != ZPROP_INVAL) {
4878                         header = zpool_prop_column_name(pl->pl_prop);
4879                         right_justify = zpool_prop_align_right(pl->pl_prop);
4880                 } else {
4881                         int i;
4882
4883                         for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
4884                                 headerbuf[i] = toupper(pl->pl_user_prop[i]);
4885                         headerbuf[i] = '\0';
4886                         header = headerbuf;
4887                 }
4888
4889                 if (pl->pl_next == NULL && !right_justify)
4890                         (void) printf("%s", header);
4891                 else if (right_justify)
4892                         (void) printf("%*s", (int)width, header);
4893                 else
4894                         (void) printf("%-*s", (int)width, header);
4895         }
4896
4897         (void) printf("\n");
4898 }
4899
4900 /*
4901  * Given a pool and a list of properties, print out all the properties according
4902  * to the described layout.
4903  */
4904 static void
4905 print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
4906 {
4907         zprop_list_t *pl = cb->cb_proplist;
4908         boolean_t first = B_TRUE;
4909         char property[ZPOOL_MAXPROPLEN];
4910         char *propstr;
4911         boolean_t right_justify;
4912         size_t width;
4913
4914         for (; pl != NULL; pl = pl->pl_next) {
4915
4916                 width = pl->pl_width;
4917                 if (first && cb->cb_verbose) {
4918                         /*
4919                          * Reset the width to accommodate the verbose listing
4920                          * of devices.
4921                          */
4922                         width = cb->cb_namewidth;
4923                 }
4924
4925                 if (!first) {
4926                         if (cb->cb_scripted)
4927                                 (void) printf("\t");
4928                         else
4929                                 (void) printf("  ");
4930                 } else {
4931                         first = B_FALSE;
4932                 }
4933
4934                 right_justify = B_FALSE;
4935                 if (pl->pl_prop != ZPROP_INVAL) {
4936                         if (zpool_get_prop(zhp, pl->pl_prop, property,
4937                             sizeof (property), NULL, cb->cb_literal) != 0)
4938                                 propstr = "-";
4939                         else
4940                                 propstr = property;
4941
4942                         right_justify = zpool_prop_align_right(pl->pl_prop);
4943                 } else if ((zpool_prop_feature(pl->pl_user_prop) ||
4944                     zpool_prop_unsupported(pl->pl_user_prop)) &&
4945                     zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
4946                     sizeof (property)) == 0) {
4947                         propstr = property;
4948                 } else {
4949                         propstr = "-";
4950                 }
4951
4952
4953                 /*
4954                  * If this is being called in scripted mode, or if this is the
4955                  * last column and it is left-justified, don't include a width
4956                  * format specifier.
4957                  */
4958                 if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
4959                         (void) printf("%s", propstr);
4960                 else if (right_justify)
4961                         (void) printf("%*s", (int)width, propstr);
4962                 else
4963                         (void) printf("%-*s", (int)width, propstr);
4964         }
4965
4966         (void) printf("\n");
4967 }
4968
4969 static void
4970 print_one_column(zpool_prop_t prop, uint64_t value, boolean_t scripted,
4971     boolean_t valid, enum zfs_nicenum_format format)
4972 {
4973         char propval[64];
4974         boolean_t fixed;
4975         size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
4976
4977         switch (prop) {
4978         case ZPOOL_PROP_EXPANDSZ:
4979                 if (value == 0)
4980                         (void) strlcpy(propval, "-", sizeof (propval));
4981                 else
4982                         zfs_nicenum_format(value, propval, sizeof (propval),
4983                             format);
4984                 break;
4985         case ZPOOL_PROP_FRAGMENTATION:
4986                 if (value == ZFS_FRAG_INVALID) {
4987                         (void) strlcpy(propval, "-", sizeof (propval));
4988                 } else if (format == ZFS_NICENUM_RAW) {
4989                         (void) snprintf(propval, sizeof (propval), "%llu",
4990                             (unsigned long long)value);
4991                 } else {
4992                         (void) snprintf(propval, sizeof (propval), "%llu%%",
4993                             (unsigned long long)value);
4994                 }
4995                 break;
4996         case ZPOOL_PROP_CAPACITY:
4997                 if (format == ZFS_NICENUM_RAW)
4998                         (void) snprintf(propval, sizeof (propval), "%llu",
4999                             (unsigned long long)value);
5000                 else
5001                         (void) snprintf(propval, sizeof (propval), "%llu%%",
5002                             (unsigned long long)value);
5003                 break;
5004         default:
5005                 zfs_nicenum_format(value, propval, sizeof (propval), format);
5006         }
5007
5008         if (!valid)
5009                 (void) strlcpy(propval, "-", sizeof (propval));
5010
5011         if (scripted)
5012                 (void) printf("\t%s", propval);
5013         else
5014                 (void) printf("  %*s", (int)width, propval);
5015 }
5016
5017 void
5018 print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
5019     list_cbdata_t *cb, int depth)
5020 {
5021         nvlist_t **child;
5022         vdev_stat_t *vs;
5023         uint_t c, children;
5024         char *vname;
5025         boolean_t scripted = cb->cb_scripted;
5026         uint64_t islog = B_FALSE;
5027         boolean_t haslog = B_FALSE;
5028         char *dashes = "%-*s      -      -      -         -      -      -\n";
5029
5030         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
5031             (uint64_t **)&vs, &c) == 0);
5032
5033         if (name != NULL) {
5034                 boolean_t toplevel = (vs->vs_space != 0);
5035                 uint64_t cap;
5036                 enum zfs_nicenum_format format;
5037
5038                 if (cb->cb_literal)
5039                         format = ZFS_NICENUM_RAW;
5040                 else
5041                         format = ZFS_NICENUM_1024;
5042
5043                 if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
5044                         return;
5045
5046                 if (scripted)
5047                         (void) printf("\t%s", name);
5048                 else if (strlen(name) + depth > cb->cb_namewidth)
5049                         (void) printf("%*s%s", depth, "", name);
5050                 else
5051                         (void) printf("%*s%s%*s", depth, "", name,
5052                             (int)(cb->cb_namewidth - strlen(name) - depth), "");
5053
5054                 /*
5055                  * Print the properties for the individual vdevs. Some
5056                  * properties are only applicable to toplevel vdevs. The
5057                  * 'toplevel' boolean value is passed to the print_one_column()
5058                  * to indicate that the value is valid.
5059                  */
5060                 print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, scripted,
5061                     toplevel, format);
5062                 print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, scripted,
5063                     toplevel, format);
5064                 print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
5065                     scripted, toplevel, format);
5066                 print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, scripted,
5067                     B_TRUE, format);
5068                 print_one_column(ZPOOL_PROP_FRAGMENTATION,
5069                     vs->vs_fragmentation, scripted,
5070                     (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel),
5071                     format);
5072                 cap = (vs->vs_space == 0) ? 0 :
5073                     (vs->vs_alloc * 100 / vs->vs_space);
5074                 print_one_column(ZPOOL_PROP_CAPACITY, cap, scripted, toplevel,
5075                     format);
5076                 (void) printf("\n");
5077         }
5078
5079         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
5080             &child, &children) != 0)
5081                 return;
5082
5083         for (c = 0; c < children; c++) {
5084                 uint64_t ishole = B_FALSE;
5085
5086                 if (nvlist_lookup_uint64(child[c],
5087                     ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
5088                         continue;
5089
5090                 if (nvlist_lookup_uint64(child[c],
5091                     ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog) {
5092                         haslog = B_TRUE;
5093                         continue;
5094                 }
5095
5096                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
5097                     cb->cb_name_flags);
5098                 print_list_stats(zhp, vname, child[c], cb, depth + 2);
5099                 free(vname);
5100         }
5101
5102         if (haslog == B_TRUE) {
5103                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5104                 (void) printf(dashes, cb->cb_namewidth, "log");
5105                 for (c = 0; c < children; c++) {
5106                         if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
5107                             &islog) != 0 || !islog)
5108                                 continue;
5109                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5110                             cb->cb_name_flags);
5111                         print_list_stats(zhp, vname, child[c], cb, depth + 2);
5112                         free(vname);
5113                 }
5114         }
5115
5116         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
5117             &child, &children) == 0 && children > 0) {
5118                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5119                 (void) printf(dashes, cb->cb_namewidth, "cache");
5120                 for (c = 0; c < children; c++) {
5121                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5122                             cb->cb_name_flags);
5123                         print_list_stats(zhp, vname, child[c], cb, depth + 2);
5124                         free(vname);
5125                 }
5126         }
5127
5128         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
5129             &children) == 0 && children > 0) {
5130                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5131                 (void) printf(dashes, cb->cb_namewidth, "spare");
5132                 for (c = 0; c < children; c++) {
5133                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5134                             cb->cb_name_flags);
5135                         print_list_stats(zhp, vname, child[c], cb, depth + 2);
5136                         free(vname);
5137                 }
5138         }
5139 }
5140
5141
5142 /*
5143  * Generic callback function to list a pool.
5144  */
5145 int
5146 list_callback(zpool_handle_t *zhp, void *data)
5147 {
5148         list_cbdata_t *cbp = data;
5149         nvlist_t *config;
5150         nvlist_t *nvroot;
5151
5152         config = zpool_get_config(zhp, NULL);
5153
5154         print_pool(zhp, cbp);
5155         if (!cbp->cb_verbose)
5156                 return (0);
5157
5158         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5159             &nvroot) == 0);
5160         print_list_stats(zhp, NULL, nvroot, cbp, 0);
5161
5162         return (0);
5163 }
5164
5165 /*
5166  * zpool list [-gHLpP] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
5167  *
5168  *      -g      Display guid for individual vdev name.
5169  *      -H      Scripted mode.  Don't display headers, and separate properties
5170  *              by a single tab.
5171  *      -L      Follow links when resolving vdev path name.
5172  *      -o      List of properties to display.  Defaults to
5173  *              "name,size,allocated,free,expandsize,fragmentation,capacity,"
5174  *              "dedupratio,health,altroot"
5175  *      -p      Display values in parsable (exact) format.
5176  *      -P      Display full path for vdev name.
5177  *      -T      Display a timestamp in date(1) or Unix format
5178  *
5179  * List all pools in the system, whether or not they're healthy.  Output space
5180  * statistics for each one, as well as health status summary.
5181  */
5182 int
5183 zpool_do_list(int argc, char **argv)
5184 {
5185         int c;
5186         int ret = 0;
5187         list_cbdata_t cb = { 0 };
5188         static char default_props[] =
5189             "name,size,allocated,free,expandsize,fragmentation,capacity,"
5190             "dedupratio,health,altroot";
5191         char *props = default_props;
5192         float interval = 0;
5193         unsigned long count = 0;
5194         zpool_list_t *list;
5195         boolean_t first = B_TRUE;
5196
5197         /* check options */
5198         while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
5199                 switch (c) {
5200                 case 'g':
5201                         cb.cb_name_flags |= VDEV_NAME_GUID;
5202                         break;
5203                 case 'H':
5204                         cb.cb_scripted = B_TRUE;
5205                         break;
5206                 case 'L':
5207                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
5208                         break;
5209                 case 'o':
5210                         props = optarg;
5211                         break;
5212                 case 'P':
5213                         cb.cb_name_flags |= VDEV_NAME_PATH;
5214                         break;
5215                 case 'p':
5216                         cb.cb_literal = B_TRUE;
5217                         break;
5218                 case 'T':
5219                         get_timestamp_arg(*optarg);
5220                         break;
5221                 case 'v':
5222                         cb.cb_verbose = B_TRUE;
5223                         break;
5224                 case ':':
5225                         (void) fprintf(stderr, gettext("missing argument for "
5226                             "'%c' option\n"), optopt);
5227                         usage(B_FALSE);
5228                         break;
5229                 case '?':
5230                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5231                             optopt);
5232                         usage(B_FALSE);
5233                 }
5234         }
5235
5236         argc -= optind;
5237         argv += optind;
5238
5239         get_interval_count(&argc, argv, &interval, &count);
5240
5241         if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
5242                 usage(B_FALSE);
5243
5244         for (;;) {
5245                 if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
5246                     &ret)) == NULL)
5247                         return (1);
5248
5249                 if (pool_list_count(list) == 0)
5250                         break;
5251
5252                 if (timestamp_fmt != NODATE)
5253                         print_timestamp(timestamp_fmt);
5254
5255                 if (!cb.cb_scripted && (first || cb.cb_verbose)) {
5256                         print_header(&cb);
5257                         first = B_FALSE;
5258                 }
5259                 ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
5260
5261                 if (interval == 0)
5262                         break;
5263
5264                 if (count != 0 && --count == 0)
5265                         break;
5266
5267                 pool_list_free(list);
5268                 (void) fsleep(interval);
5269         }
5270
5271         if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
5272                 (void) printf(gettext("no pools available\n"));
5273                 ret = 0;
5274         }
5275
5276         pool_list_free(list);
5277         zprop_free_list(cb.cb_proplist);
5278         return (ret);
5279 }
5280
5281 static int
5282 zpool_do_attach_or_replace(int argc, char **argv, int replacing)
5283 {
5284         boolean_t force = B_FALSE;
5285         int c;
5286         nvlist_t *nvroot;
5287         char *poolname, *old_disk, *new_disk;
5288         zpool_handle_t *zhp;
5289         nvlist_t *props = NULL;
5290         char *propval;
5291         int ret;
5292
5293         /* check options */
5294         while ((c = getopt(argc, argv, "fo:")) != -1) {
5295                 switch (c) {
5296                 case 'f':
5297                         force = B_TRUE;
5298                         break;
5299                 case 'o':
5300                         if ((propval = strchr(optarg, '=')) == NULL) {
5301                                 (void) fprintf(stderr, gettext("missing "
5302                                     "'=' for -o option\n"));
5303                                 usage(B_FALSE);
5304                         }
5305                         *propval = '\0';
5306                         propval++;
5307
5308                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
5309                             (add_prop_list(optarg, propval, &props, B_TRUE)))
5310                                 usage(B_FALSE);
5311                         break;
5312                 case '?':
5313                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5314                             optopt);
5315                         usage(B_FALSE);
5316                 }
5317         }
5318
5319         argc -= optind;
5320         argv += optind;
5321
5322         /* get pool name and check number of arguments */
5323         if (argc < 1) {
5324                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
5325                 usage(B_FALSE);
5326         }
5327
5328         poolname = argv[0];
5329
5330         if (argc < 2) {
5331                 (void) fprintf(stderr,
5332                     gettext("missing <device> specification\n"));
5333                 usage(B_FALSE);
5334         }
5335
5336         old_disk = argv[1];
5337
5338         if (argc < 3) {
5339                 if (!replacing) {
5340                         (void) fprintf(stderr,
5341                             gettext("missing <new_device> specification\n"));
5342                         usage(B_FALSE);
5343                 }
5344                 new_disk = old_disk;
5345                 argc -= 1;
5346                 argv += 1;
5347         } else {
5348                 new_disk = argv[2];
5349                 argc -= 2;
5350                 argv += 2;
5351         }
5352
5353         if (argc > 1) {
5354                 (void) fprintf(stderr, gettext("too many arguments\n"));
5355                 usage(B_FALSE);
5356         }
5357
5358         if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
5359                 nvlist_free(props);
5360                 return (1);
5361         }
5362
5363         if (zpool_get_config(zhp, NULL) == NULL) {
5364                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
5365                     poolname);
5366                 zpool_close(zhp);
5367                 nvlist_free(props);
5368                 return (1);
5369         }
5370
5371         /* unless manually specified use "ashift" pool property (if set) */
5372         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
5373                 int intval;
5374                 zprop_source_t src;
5375                 char strval[ZPOOL_MAXPROPLEN];
5376
5377                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
5378                 if (src != ZPROP_SRC_DEFAULT) {
5379                         (void) sprintf(strval, "%" PRId32, intval);
5380                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
5381                             &props, B_TRUE) == 0);
5382                 }
5383         }
5384
5385         nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
5386             argc, argv);
5387         if (nvroot == NULL) {
5388                 zpool_close(zhp);
5389                 nvlist_free(props);
5390                 return (1);
5391         }
5392
5393         ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing);
5394
5395         nvlist_free(props);
5396         nvlist_free(nvroot);
5397         zpool_close(zhp);
5398
5399         return (ret);
5400 }
5401
5402 /*
5403  * zpool replace [-f] <pool> <device> <new_device>
5404  *
5405  *      -f      Force attach, even if <new_device> appears to be in use.
5406  *
5407  * Replace <device> with <new_device>.
5408  */
5409 /* ARGSUSED */
5410 int
5411 zpool_do_replace(int argc, char **argv)
5412 {
5413         return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
5414 }
5415
5416 /*
5417  * zpool attach [-f] [-o property=value] <pool> <device> <new_device>
5418  *
5419  *      -f      Force attach, even if <new_device> appears to be in use.
5420  *      -o      Set property=value.
5421  *
5422  * Attach <new_device> to the mirror containing <device>.  If <device> is not
5423  * part of a mirror, then <device> will be transformed into a mirror of
5424  * <device> and <new_device>.  In either case, <new_device> will begin life
5425  * with a DTL of [0, now], and will immediately begin to resilver itself.
5426  */
5427 int
5428 zpool_do_attach(int argc, char **argv)
5429 {
5430         return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
5431 }
5432
5433 /*
5434  * zpool detach [-f] <pool> <device>
5435  *
5436  *      -f      Force detach of <device>, even if DTLs argue against it
5437  *              (not supported yet)
5438  *
5439  * Detach a device from a mirror.  The operation will be refused if <device>
5440  * is the last device in the mirror, or if the DTLs indicate that this device
5441  * has the only valid copy of some data.
5442  */
5443 /* ARGSUSED */
5444 int
5445 zpool_do_detach(int argc, char **argv)
5446 {
5447         int c;
5448         char *poolname, *path;
5449         zpool_handle_t *zhp;
5450         int ret;
5451
5452         /* check options */
5453         while ((c = getopt(argc, argv, "f")) != -1) {
5454                 switch (c) {
5455                 case 'f':
5456                 case '?':
5457                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5458                             optopt);
5459                         usage(B_FALSE);
5460                 }
5461         }
5462
5463         argc -= optind;
5464         argv += optind;
5465
5466         /* get pool name and check number of arguments */
5467         if (argc < 1) {
5468                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
5469                 usage(B_FALSE);
5470         }
5471
5472         if (argc < 2) {
5473                 (void) fprintf(stderr,
5474                     gettext("missing <device> specification\n"));
5475                 usage(B_FALSE);
5476         }
5477
5478         poolname = argv[0];
5479         path = argv[1];
5480
5481         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5482                 return (1);
5483
5484         ret = zpool_vdev_detach(zhp, path);
5485
5486         zpool_close(zhp);
5487
5488         return (ret);
5489 }
5490
5491 /*
5492  * zpool split [-gLnP] [-o prop=val] ...
5493  *              [-o mntopt] ...
5494  *              [-R altroot] <pool> <newpool> [<device> ...]
5495  *
5496  *      -g      Display guid for individual vdev name.
5497  *      -L      Follow links when resolving vdev path name.
5498  *      -n      Do not split the pool, but display the resulting layout if
5499  *              it were to be split.
5500  *      -o      Set property=value, or set mount options.
5501  *      -P      Display full path for vdev name.
5502  *      -R      Mount the split-off pool under an alternate root.
5503  *      -l      Load encryption keys while importing.
5504  *
5505  * Splits the named pool and gives it the new pool name.  Devices to be split
5506  * off may be listed, provided that no more than one device is specified
5507  * per top-level vdev mirror.  The newly split pool is left in an exported
5508  * state unless -R is specified.
5509  *
5510  * Restrictions: the top-level of the pool pool must only be made up of
5511  * mirrors; all devices in the pool must be healthy; no device may be
5512  * undergoing a resilvering operation.
5513  */
5514 int
5515 zpool_do_split(int argc, char **argv)
5516 {
5517         char *srcpool, *newpool, *propval;
5518         char *mntopts = NULL;
5519         splitflags_t flags;
5520         int c, ret = 0;
5521         boolean_t loadkeys = B_FALSE;
5522         zpool_handle_t *zhp;
5523         nvlist_t *config, *props = NULL;
5524
5525         flags.dryrun = B_FALSE;
5526         flags.import = B_FALSE;
5527         flags.name_flags = 0;
5528
5529         /* check options */
5530         while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
5531                 switch (c) {
5532                 case 'g':
5533                         flags.name_flags |= VDEV_NAME_GUID;
5534                         break;
5535                 case 'L':
5536                         flags.name_flags |= VDEV_NAME_FOLLOW_LINKS;
5537                         break;
5538                 case 'R':
5539                         flags.import = B_TRUE;
5540                         if (add_prop_list(
5541                             zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
5542                             &props, B_TRUE) != 0) {
5543                                 nvlist_free(props);
5544                                 usage(B_FALSE);
5545                         }
5546                         break;
5547                 case 'l':
5548                         loadkeys = B_TRUE;
5549                         break;
5550                 case 'n':
5551                         flags.dryrun = B_TRUE;
5552                         break;
5553                 case 'o':
5554                         if ((propval = strchr(optarg, '=')) != NULL) {
5555                                 *propval = '\0';
5556                                 propval++;
5557                                 if (add_prop_list(optarg, propval,
5558                                     &props, B_TRUE) != 0) {
5559                                         nvlist_free(props);
5560                                         usage(B_FALSE);
5561                                 }
5562                         } else {
5563                                 mntopts = optarg;
5564                         }
5565                         break;
5566                 case 'P':
5567                         flags.name_flags |= VDEV_NAME_PATH;
5568                         break;
5569                 case ':':
5570                         (void) fprintf(stderr, gettext("missing argument for "
5571                             "'%c' option\n"), optopt);
5572                         usage(B_FALSE);
5573                         break;
5574                 case '?':
5575                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5576                             optopt);
5577                         usage(B_FALSE);
5578                         break;
5579                 }
5580         }
5581
5582         if (!flags.import && mntopts != NULL) {
5583                 (void) fprintf(stderr, gettext("setting mntopts is only "
5584                     "valid when importing the pool\n"));
5585                 usage(B_FALSE);
5586         }
5587
5588         if (!flags.import && loadkeys) {
5589                 (void) fprintf(stderr, gettext("loading keys is only "
5590                     "valid when importing the pool\n"));
5591                 usage(B_FALSE);
5592         }
5593
5594         argc -= optind;
5595         argv += optind;
5596
5597         if (argc < 1) {
5598                 (void) fprintf(stderr, gettext("Missing pool name\n"));
5599                 usage(B_FALSE);
5600         }
5601         if (argc < 2) {
5602                 (void) fprintf(stderr, gettext("Missing new pool name\n"));
5603                 usage(B_FALSE);
5604         }
5605
5606         srcpool = argv[0];
5607         newpool = argv[1];
5608
5609         argc -= 2;
5610         argv += 2;
5611
5612         if ((zhp = zpool_open(g_zfs, srcpool)) == NULL) {
5613                 nvlist_free(props);
5614                 return (1);
5615         }
5616
5617         config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
5618         if (config == NULL) {
5619                 ret = 1;
5620         } else {
5621                 if (flags.dryrun) {
5622                         (void) printf(gettext("would create '%s' with the "
5623                             "following layout:\n\n"), newpool);
5624                         print_vdev_tree(NULL, newpool, config, 0, B_FALSE,
5625                             flags.name_flags);
5626                 }
5627         }
5628
5629         zpool_close(zhp);
5630
5631         if (ret != 0 || flags.dryrun || !flags.import) {
5632                 nvlist_free(config);
5633                 nvlist_free(props);
5634                 return (ret);
5635         }
5636
5637         /*
5638          * The split was successful. Now we need to open the new
5639          * pool and import it.
5640          */
5641         if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL) {
5642                 nvlist_free(config);
5643                 nvlist_free(props);
5644                 return (1);
5645         }
5646
5647         if (loadkeys) {
5648                 ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
5649                 if (ret != 0)
5650                         ret = 1;
5651         }
5652
5653         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
5654             zpool_enable_datasets(zhp, mntopts, 0) != 0) {
5655                 ret = 1;
5656                 (void) fprintf(stderr, gettext("Split was successful, but "
5657                     "the datasets could not all be mounted\n"));
5658                 (void) fprintf(stderr, gettext("Try doing '%s' with a "
5659                     "different altroot\n"), "zpool import");
5660         }
5661         zpool_close(zhp);
5662         nvlist_free(config);
5663         nvlist_free(props);
5664
5665         return (ret);
5666 }
5667
5668
5669
5670 /*
5671  * zpool online <pool> <device> ...
5672  */
5673 int
5674 zpool_do_online(int argc, char **argv)
5675 {
5676         int c, i;
5677         char *poolname;
5678         zpool_handle_t *zhp;
5679         int ret = 0;
5680         vdev_state_t newstate;
5681         int flags = 0;
5682
5683         /* check options */
5684         while ((c = getopt(argc, argv, "et")) != -1) {
5685                 switch (c) {
5686                 case 'e':
5687                         flags |= ZFS_ONLINE_EXPAND;
5688                         break;
5689                 case 't':
5690                 case '?':
5691                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5692                             optopt);
5693                         usage(B_FALSE);
5694                 }
5695         }
5696
5697         argc -= optind;
5698         argv += optind;
5699
5700         /* get pool name and check number of arguments */
5701         if (argc < 1) {
5702                 (void) fprintf(stderr, gettext("missing pool name\n"));
5703                 usage(B_FALSE);
5704         }
5705         if (argc < 2) {
5706                 (void) fprintf(stderr, gettext("missing device name\n"));
5707                 usage(B_FALSE);
5708         }
5709
5710         poolname = argv[0];
5711
5712         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5713                 return (1);
5714
5715         for (i = 1; i < argc; i++) {
5716                 if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
5717                         if (newstate != VDEV_STATE_HEALTHY) {
5718                                 (void) printf(gettext("warning: device '%s' "
5719                                     "onlined, but remains in faulted state\n"),
5720                                     argv[i]);
5721                                 if (newstate == VDEV_STATE_FAULTED)
5722                                         (void) printf(gettext("use 'zpool "
5723                                             "clear' to restore a faulted "
5724                                             "device\n"));
5725                                 else
5726                                         (void) printf(gettext("use 'zpool "
5727                                             "replace' to replace devices "
5728                                             "that are no longer present\n"));
5729                         }
5730                 } else {
5731                         ret = 1;
5732                 }
5733         }
5734
5735         zpool_close(zhp);
5736
5737         return (ret);
5738 }
5739
5740 /*
5741  * zpool offline [-ft] <pool> <device> ...
5742  *
5743  *      -f      Force the device into a faulted state.
5744  *
5745  *      -t      Only take the device off-line temporarily.  The offline/faulted
5746  *              state will not be persistent across reboots.
5747  */
5748 /* ARGSUSED */
5749 int
5750 zpool_do_offline(int argc, char **argv)
5751 {
5752         int c, i;
5753         char *poolname;
5754         zpool_handle_t *zhp;
5755         int ret = 0;
5756         boolean_t istmp = B_FALSE;
5757         boolean_t fault = B_FALSE;
5758
5759         /* check options */
5760         while ((c = getopt(argc, argv, "ft")) != -1) {
5761                 switch (c) {
5762                 case 'f':
5763                         fault = B_TRUE;
5764                         break;
5765                 case 't':
5766                         istmp = B_TRUE;
5767                         break;
5768                 case '?':
5769                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5770                             optopt);
5771                         usage(B_FALSE);
5772                 }
5773         }
5774
5775         argc -= optind;
5776         argv += optind;
5777
5778         /* get pool name and check number of arguments */
5779         if (argc < 1) {
5780                 (void) fprintf(stderr, gettext("missing pool name\n"));
5781                 usage(B_FALSE);
5782         }
5783         if (argc < 2) {
5784                 (void) fprintf(stderr, gettext("missing device name\n"));
5785                 usage(B_FALSE);
5786         }
5787
5788         poolname = argv[0];
5789
5790         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5791                 return (1);
5792
5793         for (i = 1; i < argc; i++) {
5794                 if (fault) {
5795                         uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
5796                         vdev_aux_t aux;
5797                         if (istmp == B_FALSE) {
5798                                 /* Force the fault to persist across imports */
5799                                 aux = VDEV_AUX_EXTERNAL_PERSIST;
5800                         } else {
5801                                 aux = VDEV_AUX_EXTERNAL;
5802                         }
5803
5804                         if (guid == 0 || zpool_vdev_fault(zhp, guid, aux) != 0)
5805                                 ret = 1;
5806                 } else {
5807                         if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
5808                                 ret = 1;
5809                 }
5810         }
5811
5812         zpool_close(zhp);
5813
5814         return (ret);
5815 }
5816
5817 /*
5818  * zpool clear <pool> [device]
5819  *
5820  * Clear all errors associated with a pool or a particular device.
5821  */
5822 int
5823 zpool_do_clear(int argc, char **argv)
5824 {
5825         int c;
5826         int ret = 0;
5827         boolean_t dryrun = B_FALSE;
5828         boolean_t do_rewind = B_FALSE;
5829         boolean_t xtreme_rewind = B_FALSE;
5830         uint32_t rewind_policy = ZPOOL_NO_REWIND;
5831         nvlist_t *policy = NULL;
5832         zpool_handle_t *zhp;
5833         char *pool, *device;
5834
5835         /* check options */
5836         while ((c = getopt(argc, argv, "FnX")) != -1) {
5837                 switch (c) {
5838                 case 'F':
5839                         do_rewind = B_TRUE;
5840                         break;
5841                 case 'n':
5842                         dryrun = B_TRUE;
5843                         break;
5844                 case 'X':
5845                         xtreme_rewind = B_TRUE;
5846                         break;
5847                 case '?':
5848                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5849                             optopt);
5850                         usage(B_FALSE);
5851                 }
5852         }
5853
5854         argc -= optind;
5855         argv += optind;
5856
5857         if (argc < 1) {
5858                 (void) fprintf(stderr, gettext("missing pool name\n"));
5859                 usage(B_FALSE);
5860         }
5861
5862         if (argc > 2) {
5863                 (void) fprintf(stderr, gettext("too many arguments\n"));
5864                 usage(B_FALSE);
5865         }
5866
5867         if ((dryrun || xtreme_rewind) && !do_rewind) {
5868                 (void) fprintf(stderr,
5869                     gettext("-n or -X only meaningful with -F\n"));
5870                 usage(B_FALSE);
5871         }
5872         if (dryrun)
5873                 rewind_policy = ZPOOL_TRY_REWIND;
5874         else if (do_rewind)
5875                 rewind_policy = ZPOOL_DO_REWIND;
5876         if (xtreme_rewind)
5877                 rewind_policy |= ZPOOL_EXTREME_REWIND;
5878
5879         /* In future, further rewind policy choices can be passed along here */
5880         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
5881             nvlist_add_uint32(policy, ZPOOL_REWIND_REQUEST, rewind_policy) != 0)
5882                 return (1);
5883
5884         pool = argv[0];
5885         device = argc == 2 ? argv[1] : NULL;
5886
5887         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
5888                 nvlist_free(policy);
5889                 return (1);
5890         }
5891
5892         if (zpool_clear(zhp, device, policy) != 0)
5893                 ret = 1;
5894
5895         zpool_close(zhp);
5896
5897         nvlist_free(policy);
5898
5899         return (ret);
5900 }
5901
5902 /*
5903  * zpool reguid <pool>
5904  */
5905 int
5906 zpool_do_reguid(int argc, char **argv)
5907 {
5908         int c;
5909         char *poolname;
5910         zpool_handle_t *zhp;
5911         int ret = 0;
5912
5913         /* check options */
5914         while ((c = getopt(argc, argv, "")) != -1) {
5915                 switch (c) {
5916                 case '?':
5917                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5918                             optopt);
5919                         usage(B_FALSE);
5920                 }
5921         }
5922
5923         argc -= optind;
5924         argv += optind;
5925
5926         /* get pool name and check number of arguments */
5927         if (argc < 1) {
5928                 (void) fprintf(stderr, gettext("missing pool name\n"));
5929                 usage(B_FALSE);
5930         }
5931
5932         if (argc > 1) {
5933                 (void) fprintf(stderr, gettext("too many arguments\n"));
5934                 usage(B_FALSE);
5935         }
5936
5937         poolname = argv[0];
5938         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5939                 return (1);
5940
5941         ret = zpool_reguid(zhp);
5942
5943         zpool_close(zhp);
5944         return (ret);
5945 }
5946
5947
5948 /*
5949  * zpool reopen <pool>
5950  *
5951  * Reopen the pool so that the kernel can update the sizes of all vdevs.
5952  */
5953 int
5954 zpool_do_reopen(int argc, char **argv)
5955 {
5956         int c;
5957         int ret = 0;
5958         boolean_t scrub_restart = B_TRUE;
5959
5960         /* check options */
5961         while ((c = getopt(argc, argv, "n")) != -1) {
5962                 switch (c) {
5963                 case 'n':
5964                         scrub_restart = B_FALSE;
5965                         break;
5966                 case '?':
5967                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5968                             optopt);
5969                         usage(B_FALSE);
5970                 }
5971         }
5972
5973         argc -= optind;
5974         argv += optind;
5975
5976         /* if argc == 0 we will execute zpool_reopen_one on all pools */
5977         ret = for_each_pool(argc, argv, B_TRUE, NULL, zpool_reopen_one,
5978             &scrub_restart);
5979
5980         return (ret);
5981 }
5982
5983 typedef struct scrub_cbdata {
5984         int     cb_type;
5985         int     cb_argc;
5986         char    **cb_argv;
5987         pool_scrub_cmd_t cb_scrub_cmd;
5988 } scrub_cbdata_t;
5989
5990 int
5991 scrub_callback(zpool_handle_t *zhp, void *data)
5992 {
5993         scrub_cbdata_t *cb = data;
5994         int err;
5995
5996         /*
5997          * Ignore faulted pools.
5998          */
5999         if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
6000                 (void) fprintf(stderr, gettext("cannot scrub '%s': pool is "
6001                     "currently unavailable\n"), zpool_get_name(zhp));
6002                 return (1);
6003         }
6004
6005         err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
6006
6007         return (err != 0);
6008 }
6009
6010 /*
6011  * zpool scrub [-s | -p] <pool> ...
6012  *
6013  *      -s      Stop.  Stops any in-progress scrub.
6014  *      -p      Pause. Pause in-progress scrub.
6015  */
6016 int
6017 zpool_do_scrub(int argc, char **argv)
6018 {
6019         int c;
6020         scrub_cbdata_t cb;
6021
6022         cb.cb_type = POOL_SCAN_SCRUB;
6023         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
6024
6025         /* check options */
6026         while ((c = getopt(argc, argv, "sp")) != -1) {
6027                 switch (c) {
6028                 case 's':
6029                         cb.cb_type = POOL_SCAN_NONE;
6030                         break;
6031                 case 'p':
6032                         cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
6033                         break;
6034                 case '?':
6035                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6036                             optopt);
6037                         usage(B_FALSE);
6038                 }
6039         }
6040
6041         if (cb.cb_type == POOL_SCAN_NONE &&
6042             cb.cb_scrub_cmd == POOL_SCRUB_PAUSE) {
6043                 (void) fprintf(stderr, gettext("invalid option combination: "
6044                     "-s and -p are mutually exclusive\n"));
6045                 usage(B_FALSE);
6046         }
6047
6048         cb.cb_argc = argc;
6049         cb.cb_argv = argv;
6050         argc -= optind;
6051         argv += optind;
6052
6053         if (argc < 1) {
6054                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6055                 usage(B_FALSE);
6056         }
6057
6058         return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
6059 }
6060
6061 /*
6062  * Print out detailed scrub status.
6063  */
6064 static void
6065 print_scan_status(pool_scan_stat_t *ps)
6066 {
6067         time_t start, end, pause;
6068         uint64_t total_secs_left;
6069         uint64_t elapsed, secs_left, mins_left, hours_left, days_left;
6070         uint64_t pass_scanned, scanned, pass_issued, issued, total;
6071         uint64_t scan_rate, issue_rate;
6072         double fraction_done;
6073         char processed_buf[7], scanned_buf[7], issued_buf[7], total_buf[7];
6074         char srate_buf[7], irate_buf[7];
6075
6076         (void) printf(gettext("  scan: "));
6077
6078         /* If there's never been a scan, there's not much to say. */
6079         if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
6080             ps->pss_func >= POOL_SCAN_FUNCS) {
6081                 (void) printf(gettext("none requested\n"));
6082                 return;
6083         }
6084
6085         start = ps->pss_start_time;
6086         end = ps->pss_end_time;
6087         pause = ps->pss_pass_scrub_pause;
6088
6089         zfs_nicebytes(ps->pss_processed, processed_buf, sizeof (processed_buf));
6090
6091         assert(ps->pss_func == POOL_SCAN_SCRUB ||
6092             ps->pss_func == POOL_SCAN_RESILVER);
6093
6094         /* Scan is finished or canceled. */
6095         if (ps->pss_state == DSS_FINISHED) {
6096                 total_secs_left = end - start;
6097                 days_left = total_secs_left / 60 / 60 / 24;
6098                 hours_left = (total_secs_left / 60 / 60) % 24;
6099                 mins_left = (total_secs_left / 60) % 60;
6100                 secs_left = (total_secs_left % 60);
6101
6102                 if (ps->pss_func == POOL_SCAN_SCRUB) {
6103                         (void) printf(gettext("scrub repaired %s "
6104                             "in %llu days %02llu:%02llu:%02llu "
6105                             "with %llu errors on %s"), processed_buf,
6106                             (u_longlong_t)days_left, (u_longlong_t)hours_left,
6107                             (u_longlong_t)mins_left, (u_longlong_t)secs_left,
6108                             (u_longlong_t)ps->pss_errors, ctime(&end));
6109                 } else if (ps->pss_func == POOL_SCAN_RESILVER) {
6110                         (void) printf(gettext("resilvered %s "
6111                             "in %llu days %02llu:%02llu:%02llu "
6112                             "with %llu errors on %s"), processed_buf,
6113                             (u_longlong_t)days_left, (u_longlong_t)hours_left,
6114                             (u_longlong_t)mins_left, (u_longlong_t)secs_left,
6115                             (u_longlong_t)ps->pss_errors, ctime(&end));
6116                 }
6117                 return;
6118         } else if (ps->pss_state == DSS_CANCELED) {
6119                 if (ps->pss_func == POOL_SCAN_SCRUB) {
6120                         (void) printf(gettext("scrub canceled on %s"),
6121                             ctime(&end));
6122                 } else if (ps->pss_func == POOL_SCAN_RESILVER) {
6123                         (void) printf(gettext("resilver canceled on %s"),
6124                             ctime(&end));
6125                 }
6126                 return;
6127         }
6128
6129         assert(ps->pss_state == DSS_SCANNING);
6130
6131         /* Scan is in progress. Resilvers can't be paused. */
6132         if (ps->pss_func == POOL_SCAN_SCRUB) {
6133                 if (pause == 0) {
6134                         (void) printf(gettext("scrub in progress since %s"),
6135                             ctime(&start));
6136                 } else {
6137                         (void) printf(gettext("scrub paused since %s"),
6138                             ctime(&pause));
6139                         (void) printf(gettext("\tscrub started on %s"),
6140                             ctime(&start));
6141                 }
6142         } else if (ps->pss_func == POOL_SCAN_RESILVER) {
6143                 (void) printf(gettext("resilver in progress since %s"),
6144                     ctime(&start));
6145         }
6146
6147         scanned = ps->pss_examined;
6148         pass_scanned = ps->pss_pass_exam;
6149         issued = ps->pss_issued;
6150         pass_issued = ps->pss_pass_issued;
6151         total = ps->pss_to_examine;
6152
6153         /* we are only done with a block once we have issued the IO for it */
6154         fraction_done = (double)issued / total;
6155
6156         /* elapsed time for this pass, rounding up to 1 if it's 0 */
6157         elapsed = time(NULL) - ps->pss_pass_start;
6158         elapsed -= ps->pss_pass_scrub_spent_paused;
6159         elapsed = (elapsed != 0) ? elapsed : 1;
6160
6161         scan_rate = pass_scanned / elapsed;
6162         issue_rate = pass_issued / elapsed;
6163         total_secs_left = (issue_rate != 0) ?
6164             ((total - issued) / issue_rate) : UINT64_MAX;
6165
6166         days_left = total_secs_left / 60 / 60 / 24;
6167         hours_left = (total_secs_left / 60 / 60) % 24;
6168         mins_left = (total_secs_left / 60) % 60;
6169         secs_left = (total_secs_left % 60);
6170
6171         /* format all of the numbers we will be reporting */
6172         zfs_nicebytes(scanned, scanned_buf, sizeof (scanned_buf));
6173         zfs_nicebytes(issued, issued_buf, sizeof (issued_buf));
6174         zfs_nicebytes(total, total_buf, sizeof (total_buf));
6175         zfs_nicebytes(scan_rate, srate_buf, sizeof (srate_buf));
6176         zfs_nicebytes(issue_rate, irate_buf, sizeof (irate_buf));
6177
6178         /* do not print estimated time if we have a paused scrub */
6179         if (pause == 0) {
6180                 (void) printf(gettext("\t%s scanned at %s/s, "
6181                     "%s issued at %s/s, %s total\n"),
6182                     scanned_buf, srate_buf, issued_buf, irate_buf, total_buf);
6183         } else {
6184                 (void) printf(gettext("\t%s scanned, %s issued, %s total\n"),
6185                     scanned_buf, issued_buf, total_buf);
6186         }
6187
6188         if (ps->pss_func == POOL_SCAN_RESILVER) {
6189                 (void) printf(gettext("\t%s resilvered, %.2f%% done"),
6190                     processed_buf, 100 * fraction_done);
6191         } else if (ps->pss_func == POOL_SCAN_SCRUB) {
6192                 (void) printf(gettext("\t%s repaired, %.2f%% done"),
6193                     processed_buf, 100 * fraction_done);
6194         }
6195
6196         if (pause == 0) {
6197                 if (issue_rate >= 10 * 1024 * 1024) {
6198                         (void) printf(gettext(", %llu days "
6199                             "%02llu:%02llu:%02llu to go\n"),
6200                             (u_longlong_t)days_left, (u_longlong_t)hours_left,
6201                             (u_longlong_t)mins_left, (u_longlong_t)secs_left);
6202                 } else {
6203                         (void) printf(gettext(", no estimated "
6204                             "completion time\n"));
6205                 }
6206         } else {
6207                 (void) printf(gettext("\n"));
6208         }
6209 }
6210
6211 /*
6212  * Print out detailed removal status.
6213  */
6214 static void
6215 print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
6216 {
6217         char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
6218         time_t start, end;
6219         nvlist_t *config, *nvroot;
6220         nvlist_t **child;
6221         uint_t children;
6222         char *vdev_name;
6223
6224         if (prs == NULL || prs->prs_state == DSS_NONE)
6225                 return;
6226
6227         /*
6228          * Determine name of vdev.
6229          */
6230         config = zpool_get_config(zhp, NULL);
6231         nvroot = fnvlist_lookup_nvlist(config,
6232             ZPOOL_CONFIG_VDEV_TREE);
6233         verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
6234             &child, &children) == 0);
6235         assert(prs->prs_removing_vdev < children);
6236         vdev_name = zpool_vdev_name(g_zfs, zhp,
6237             child[prs->prs_removing_vdev], B_TRUE);
6238
6239         (void) printf(gettext("remove: "));
6240
6241         start = prs->prs_start_time;
6242         end = prs->prs_end_time;
6243         zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
6244
6245         /*
6246          * Removal is finished or canceled.
6247          */
6248         if (prs->prs_state == DSS_FINISHED) {
6249                 uint64_t minutes_taken = (end - start) / 60;
6250
6251                 (void) printf(gettext("Removal of vdev %llu copied %s "
6252                     "in %lluh%um, completed on %s"),
6253                     (longlong_t)prs->prs_removing_vdev,
6254                     copied_buf,
6255                     (u_longlong_t)(minutes_taken / 60),
6256                     (uint_t)(minutes_taken % 60),
6257                     ctime((time_t *)&end));
6258         } else if (prs->prs_state == DSS_CANCELED) {
6259                 (void) printf(gettext("Removal of %s canceled on %s"),
6260                     vdev_name, ctime(&end));
6261         } else {
6262                 uint64_t copied, total, elapsed, mins_left, hours_left;
6263                 double fraction_done;
6264                 uint_t rate;
6265
6266                 assert(prs->prs_state == DSS_SCANNING);
6267
6268                 /*
6269                  * Removal is in progress.
6270                  */
6271                 (void) printf(gettext(
6272                     "Evacuation of %s in progress since %s"),
6273                     vdev_name, ctime(&start));
6274
6275                 copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
6276                 total = prs->prs_to_copy;
6277                 fraction_done = (double)copied / total;
6278
6279                 /* elapsed time for this pass */
6280                 elapsed = time(NULL) - prs->prs_start_time;
6281                 elapsed = elapsed > 0 ? elapsed : 1;
6282                 rate = copied / elapsed;
6283                 rate = rate > 0 ? rate : 1;
6284                 mins_left = ((total - copied) / rate) / 60;
6285                 hours_left = mins_left / 60;
6286
6287                 zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
6288                 zfs_nicenum(total, total_buf, sizeof (total_buf));
6289                 zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
6290
6291                 /*
6292                  * do not print estimated time if hours_left is more than
6293                  * 30 days
6294                  */
6295                 (void) printf(gettext("    %s copied out of %s at %s/s, "
6296                     "%.2f%% done"),
6297                     examined_buf, total_buf, rate_buf, 100 * fraction_done);
6298                 if (hours_left < (30 * 24)) {
6299                         (void) printf(gettext(", %lluh%um to go\n"),
6300                             (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
6301                 } else {
6302                         (void) printf(gettext(
6303                             ", (copy is slow, no estimated time)\n"));
6304                 }
6305         }
6306
6307         if (prs->prs_mapping_memory > 0) {
6308                 char mem_buf[7];
6309                 zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
6310                 (void) printf(gettext("    %s memory used for "
6311                     "removed device mappings\n"),
6312                     mem_buf);
6313         }
6314 }
6315
6316 static void
6317 print_error_log(zpool_handle_t *zhp)
6318 {
6319         nvlist_t *nverrlist = NULL;
6320         nvpair_t *elem;
6321         char *pathname;
6322         size_t len = MAXPATHLEN * 2;
6323
6324         if (zpool_get_errlog(zhp, &nverrlist) != 0)
6325                 return;
6326
6327         (void) printf("errors: Permanent errors have been "
6328             "detected in the following files:\n\n");
6329
6330         pathname = safe_malloc(len);
6331         elem = NULL;
6332         while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
6333                 nvlist_t *nv;
6334                 uint64_t dsobj, obj;
6335
6336                 verify(nvpair_value_nvlist(elem, &nv) == 0);
6337                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
6338                     &dsobj) == 0);
6339                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
6340                     &obj) == 0);
6341                 zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
6342                 (void) printf("%7s %s\n", "", pathname);
6343         }
6344         free(pathname);
6345         nvlist_free(nverrlist);
6346 }
6347
6348 static void
6349 print_spares(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **spares,
6350     uint_t nspares)
6351 {
6352         uint_t i;
6353         char *name;
6354
6355         if (nspares == 0)
6356                 return;
6357
6358         (void) printf(gettext("\tspares\n"));
6359
6360         for (i = 0; i < nspares; i++) {
6361                 name = zpool_vdev_name(g_zfs, zhp, spares[i],
6362                     cb->cb_name_flags);
6363                 print_status_config(zhp, cb, name, spares[i], 2, B_TRUE);
6364                 free(name);
6365         }
6366 }
6367
6368 static void
6369 print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
6370     uint_t nl2cache)
6371 {
6372         uint_t i;
6373         char *name;
6374
6375         if (nl2cache == 0)
6376                 return;
6377
6378         (void) printf(gettext("\tcache\n"));
6379
6380         for (i = 0; i < nl2cache; i++) {
6381                 name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
6382                     cb->cb_name_flags);
6383                 print_status_config(zhp, cb, name, l2cache[i], 2, B_FALSE);
6384                 free(name);
6385         }
6386 }
6387
6388 static void
6389 print_dedup_stats(nvlist_t *config)
6390 {
6391         ddt_histogram_t *ddh;
6392         ddt_stat_t *dds;
6393         ddt_object_t *ddo;
6394         uint_t c;
6395         char dspace[6], mspace[6];
6396
6397         /*
6398          * If the pool was faulted then we may not have been able to
6399          * obtain the config. Otherwise, if we have anything in the dedup
6400          * table continue processing the stats.
6401          */
6402         if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
6403             (uint64_t **)&ddo, &c) != 0)
6404                 return;
6405
6406         (void) printf("\n");
6407         (void) printf(gettext(" dedup: "));
6408         if (ddo->ddo_count == 0) {
6409                 (void) printf(gettext("no DDT entries\n"));
6410                 return;
6411         }
6412
6413         zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
6414         zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
6415         (void) printf("DDT entries %llu, size %s on disk, %s in core\n",
6416             (u_longlong_t)ddo->ddo_count,
6417             dspace,
6418             mspace);
6419
6420         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
6421             (uint64_t **)&dds, &c) == 0);
6422         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
6423             (uint64_t **)&ddh, &c) == 0);
6424         zpool_dump_ddt(dds, ddh);
6425 }
6426
6427 /*
6428  * Display a summary of pool status.  Displays a summary such as:
6429  *
6430  *        pool: tank
6431  *      status: DEGRADED
6432  *      reason: One or more devices ...
6433  *         see: http://zfsonlinux.org/msg/ZFS-xxxx-01
6434  *      config:
6435  *              mirror          DEGRADED
6436  *                c1t0d0        OK
6437  *                c2t0d0        UNAVAIL
6438  *
6439  * When given the '-v' option, we print out the complete config.  If the '-e'
6440  * option is specified, then we print out error rate information as well.
6441  */
6442 int
6443 status_callback(zpool_handle_t *zhp, void *data)
6444 {
6445         status_cbdata_t *cbp = data;
6446         nvlist_t *config, *nvroot;
6447         char *msgid;
6448         zpool_status_t reason;
6449         zpool_errata_t errata;
6450         const char *health;
6451         uint_t c;
6452         vdev_stat_t *vs;
6453
6454         config = zpool_get_config(zhp, NULL);
6455         reason = zpool_get_status(zhp, &msgid, &errata);
6456
6457         cbp->cb_count++;
6458
6459         /*
6460          * If we were given 'zpool status -x', only report those pools with
6461          * problems.
6462          */
6463         if (cbp->cb_explain &&
6464             (reason == ZPOOL_STATUS_OK ||
6465             reason == ZPOOL_STATUS_VERSION_OLDER ||
6466             reason == ZPOOL_STATUS_FEAT_DISABLED)) {
6467                 if (!cbp->cb_allpools) {
6468                         (void) printf(gettext("pool '%s' is healthy\n"),
6469                             zpool_get_name(zhp));
6470                         if (cbp->cb_first)
6471                                 cbp->cb_first = B_FALSE;
6472                 }
6473                 return (0);
6474         }
6475
6476         if (cbp->cb_first)
6477                 cbp->cb_first = B_FALSE;
6478         else
6479                 (void) printf("\n");
6480
6481         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
6482         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
6483             (uint64_t **)&vs, &c) == 0);
6484         health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
6485
6486         (void) printf(gettext("  pool: %s\n"), zpool_get_name(zhp));
6487         (void) printf(gettext(" state: %s\n"), health);
6488
6489         switch (reason) {
6490         case ZPOOL_STATUS_MISSING_DEV_R:
6491                 (void) printf(gettext("status: One or more devices could not "
6492                     "be opened.  Sufficient replicas exist for\n\tthe pool to "
6493                     "continue functioning in a degraded state.\n"));
6494                 (void) printf(gettext("action: Attach the missing device and "
6495                     "online it using 'zpool online'.\n"));
6496                 break;
6497
6498         case ZPOOL_STATUS_MISSING_DEV_NR:
6499                 (void) printf(gettext("status: One or more devices could not "
6500                     "be opened.  There are insufficient\n\treplicas for the "
6501                     "pool to continue functioning.\n"));
6502                 (void) printf(gettext("action: Attach the missing device and "
6503                     "online it using 'zpool online'.\n"));
6504                 break;
6505
6506         case ZPOOL_STATUS_CORRUPT_LABEL_R:
6507                 (void) printf(gettext("status: One or more devices could not "
6508                     "be used because the label is missing or\n\tinvalid.  "
6509                     "Sufficient replicas exist for the pool to continue\n\t"
6510                     "functioning in a degraded state.\n"));
6511                 (void) printf(gettext("action: Replace the device using "
6512                     "'zpool replace'.\n"));
6513                 break;
6514
6515         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
6516                 (void) printf(gettext("status: One or more devices could not "
6517                     "be used because the label is missing \n\tor invalid.  "
6518                     "There are insufficient replicas for the pool to "
6519                     "continue\n\tfunctioning.\n"));
6520                 zpool_explain_recover(zpool_get_handle(zhp),
6521                     zpool_get_name(zhp), reason, config);
6522                 break;
6523
6524         case ZPOOL_STATUS_FAILING_DEV:
6525                 (void) printf(gettext("status: One or more devices has "
6526                     "experienced an unrecoverable error.  An\n\tattempt was "
6527                     "made to correct the error.  Applications are "
6528                     "unaffected.\n"));
6529                 (void) printf(gettext("action: Determine if the device needs "
6530                     "to be replaced, and clear the errors\n\tusing "
6531                     "'zpool clear' or replace the device with 'zpool "
6532                     "replace'.\n"));
6533                 break;
6534
6535         case ZPOOL_STATUS_OFFLINE_DEV:
6536                 (void) printf(gettext("status: One or more devices has "
6537                     "been taken offline by the administrator.\n\tSufficient "
6538                     "replicas exist for the pool to continue functioning in "
6539                     "a\n\tdegraded state.\n"));
6540                 (void) printf(gettext("action: Online the device using "
6541                     "'zpool online' or replace the device with\n\t'zpool "
6542                     "replace'.\n"));
6543                 break;
6544
6545         case ZPOOL_STATUS_REMOVED_DEV:
6546                 (void) printf(gettext("status: One or more devices has "
6547                     "been removed by the administrator.\n\tSufficient "
6548                     "replicas exist for the pool to continue functioning in "
6549                     "a\n\tdegraded state.\n"));
6550                 (void) printf(gettext("action: Online the device using "
6551                     "'zpool online' or replace the device with\n\t'zpool "
6552                     "replace'.\n"));
6553                 break;
6554
6555         case ZPOOL_STATUS_RESILVERING:
6556                 (void) printf(gettext("status: One or more devices is "
6557                     "currently being resilvered.  The pool will\n\tcontinue "
6558                     "to function, possibly in a degraded state.\n"));
6559                 (void) printf(gettext("action: Wait for the resilver to "
6560                     "complete.\n"));
6561                 break;
6562
6563         case ZPOOL_STATUS_CORRUPT_DATA:
6564                 (void) printf(gettext("status: One or more devices has "
6565                     "experienced an error resulting in data\n\tcorruption.  "
6566                     "Applications may be affected.\n"));
6567                 (void) printf(gettext("action: Restore the file in question "
6568                     "if possible.  Otherwise restore the\n\tentire pool from "
6569                     "backup.\n"));
6570                 break;
6571
6572         case ZPOOL_STATUS_CORRUPT_POOL:
6573                 (void) printf(gettext("status: The pool metadata is corrupted "
6574                     "and the pool cannot be opened.\n"));
6575                 zpool_explain_recover(zpool_get_handle(zhp),
6576                     zpool_get_name(zhp), reason, config);
6577                 break;
6578
6579         case ZPOOL_STATUS_VERSION_OLDER:
6580                 (void) printf(gettext("status: The pool is formatted using a "
6581                     "legacy on-disk format.  The pool can\n\tstill be used, "
6582                     "but some features are unavailable.\n"));
6583                 (void) printf(gettext("action: Upgrade the pool using 'zpool "
6584                     "upgrade'.  Once this is done, the\n\tpool will no longer "
6585                     "be accessible on software that does not support\n\t"
6586                     "feature flags.\n"));
6587                 break;
6588
6589         case ZPOOL_STATUS_VERSION_NEWER:
6590                 (void) printf(gettext("status: The pool has been upgraded to a "
6591                     "newer, incompatible on-disk version.\n\tThe pool cannot "
6592                     "be accessed on this system.\n"));
6593                 (void) printf(gettext("action: Access the pool from a system "
6594                     "running more recent software, or\n\trestore the pool from "
6595                     "backup.\n"));
6596                 break;
6597
6598         case ZPOOL_STATUS_FEAT_DISABLED:
6599                 (void) printf(gettext("status: Some supported features are not "
6600                     "enabled on the pool. The pool can\n\tstill be used, but "
6601                     "some features are unavailable.\n"));
6602                 (void) printf(gettext("action: Enable all features using "
6603                     "'zpool upgrade'. Once this is done,\n\tthe pool may no "
6604                     "longer be accessible by software that does not support\n\t"
6605                     "the features. See zpool-features(5) for details.\n"));
6606                 break;
6607
6608         case ZPOOL_STATUS_UNSUP_FEAT_READ:
6609                 (void) printf(gettext("status: The pool cannot be accessed on "
6610                     "this system because it uses the\n\tfollowing feature(s) "
6611                     "not supported on this system:\n"));
6612                 zpool_print_unsup_feat(config);
6613                 (void) printf("\n");
6614                 (void) printf(gettext("action: Access the pool from a system "
6615                     "that supports the required feature(s),\n\tor restore the "
6616                     "pool from backup.\n"));
6617                 break;
6618
6619         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
6620                 (void) printf(gettext("status: The pool can only be accessed "
6621                     "in read-only mode on this system. It\n\tcannot be "
6622                     "accessed in read-write mode because it uses the "
6623                     "following\n\tfeature(s) not supported on this system:\n"));
6624                 zpool_print_unsup_feat(config);
6625                 (void) printf("\n");
6626                 (void) printf(gettext("action: The pool cannot be accessed in "
6627                     "read-write mode. Import the pool with\n"
6628                     "\t\"-o readonly=on\", access the pool from a system that "
6629                     "supports the\n\trequired feature(s), or restore the "
6630                     "pool from backup.\n"));
6631                 break;
6632
6633         case ZPOOL_STATUS_FAULTED_DEV_R:
6634                 (void) printf(gettext("status: One or more devices are "
6635                     "faulted in response to persistent errors.\n\tSufficient "
6636                     "replicas exist for the pool to continue functioning "
6637                     "in a\n\tdegraded state.\n"));
6638                 (void) printf(gettext("action: Replace the faulted device, "
6639                     "or use 'zpool clear' to mark the device\n\trepaired.\n"));
6640                 break;
6641
6642         case ZPOOL_STATUS_FAULTED_DEV_NR:
6643                 (void) printf(gettext("status: One or more devices are "
6644                     "faulted in response to persistent errors.  There are "
6645                     "insufficient replicas for the pool to\n\tcontinue "
6646                     "functioning.\n"));
6647                 (void) printf(gettext("action: Destroy and re-create the pool "
6648                     "from a backup source.  Manually marking the device\n"
6649                     "\trepaired using 'zpool clear' may allow some data "
6650                     "to be recovered.\n"));
6651                 break;
6652
6653         case ZPOOL_STATUS_IO_FAILURE_MMP:
6654                 (void) printf(gettext("status: The pool is suspended because "
6655                     "multihost writes failed or were delayed;\n\tanother "
6656                     "system could import the pool undetected.\n"));
6657                 (void) printf(gettext("action: Make sure the pool's devices "
6658                     "are connected, then reboot your system and\n\timport the "
6659                     "pool.\n"));
6660                 break;
6661
6662         case ZPOOL_STATUS_IO_FAILURE_WAIT:
6663         case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
6664                 (void) printf(gettext("status: One or more devices are "
6665                     "faulted in response to IO failures.\n"));
6666                 (void) printf(gettext("action: Make sure the affected devices "
6667                     "are connected, then run 'zpool clear'.\n"));
6668                 break;
6669
6670         case ZPOOL_STATUS_BAD_LOG:
6671                 (void) printf(gettext("status: An intent log record "
6672                     "could not be read.\n"
6673                     "\tWaiting for administrator intervention to fix the "
6674                     "faulted pool.\n"));
6675                 (void) printf(gettext("action: Either restore the affected "
6676                     "device(s) and run 'zpool online',\n"
6677                     "\tor ignore the intent log records by running "
6678                     "'zpool clear'.\n"));
6679                 break;
6680
6681         case ZPOOL_STATUS_HOSTID_MISMATCH:
6682                 (void) printf(gettext("status: Mismatch between pool hostid "
6683                     "and system hostid on imported pool.\n\tThis pool was "
6684                     "previously imported into a system with a different "
6685                     "hostid,\n\tand then was verbatim imported into this "
6686                     "system.\n"));
6687                 (void) printf(gettext("action: Export this pool on all systems "
6688                     "on which it is imported.\n"
6689                     "\tThen import it to correct the mismatch.\n"));
6690                 break;
6691
6692         case ZPOOL_STATUS_ERRATA:
6693                 (void) printf(gettext("status: Errata #%d detected.\n"),
6694                     errata);
6695
6696                 switch (errata) {
6697                 case ZPOOL_ERRATA_NONE:
6698                         break;
6699
6700                 case ZPOOL_ERRATA_ZOL_2094_SCRUB:
6701                         (void) printf(gettext("action: To correct the issue "
6702                             "run 'zpool scrub'.\n"));
6703                         break;
6704
6705                 case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
6706                         (void) printf(gettext("\tExisting encrypted datasets "
6707                             "contain an on-disk incompatibility\n\twhich "
6708                             "needs to be corrected.\n"));
6709                         (void) printf(gettext("action: To correct the issue "
6710                             "backup existing encrypted datasets to new\n\t"
6711                             "encrypted datasets and destroy the old ones. "
6712                             "'zfs mount -o ro' can\n\tbe used to temporarily "
6713                             "mount existing encrypted datasets readonly.\n"));
6714                         break;
6715
6716                 default:
6717                         /*
6718                          * All errata which allow the pool to be imported
6719                          * must contain an action message.
6720                          */
6721                         assert(0);
6722                 }
6723                 break;
6724
6725         default:
6726                 /*
6727                  * The remaining errors can't actually be generated, yet.
6728                  */
6729                 assert(reason == ZPOOL_STATUS_OK);
6730         }
6731
6732         if (msgid != NULL)
6733                 (void) printf(gettext("   see: http://zfsonlinux.org/msg/%s\n"),
6734                     msgid);
6735
6736         if (config != NULL) {
6737                 uint64_t nerr;
6738                 nvlist_t **spares, **l2cache;
6739                 uint_t nspares, nl2cache;
6740                 pool_scan_stat_t *ps = NULL;
6741                 pool_removal_stat_t *prs = NULL;
6742
6743                 (void) nvlist_lookup_uint64_array(nvroot,
6744                     ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&ps, &c);
6745                 print_scan_status(ps);
6746
6747                 (void) nvlist_lookup_uint64_array(nvroot,
6748                     ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
6749                 print_removal_status(zhp, prs);
6750
6751                 cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
6752                     cbp->cb_name_flags | VDEV_NAME_TYPE_ID);
6753                 if (cbp->cb_namewidth < 10)
6754                         cbp->cb_namewidth = 10;
6755
6756                 (void) printf(gettext("config:\n\n"));
6757                 (void) printf(gettext("\t%-*s  %-8s %5s %5s %5s"),
6758                     cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
6759                     "CKSUM");
6760
6761                 if (cbp->vcdl != NULL)
6762                         print_cmd_columns(cbp->vcdl, 0);
6763
6764                 printf("\n");
6765                 print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
6766                     B_FALSE);
6767
6768                 if (num_logs(nvroot) > 0)
6769                         print_logs(zhp, cbp, nvroot);
6770                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
6771                     &l2cache, &nl2cache) == 0)
6772                         print_l2cache(zhp, cbp, l2cache, nl2cache);
6773
6774                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
6775                     &spares, &nspares) == 0)
6776                         print_spares(zhp, cbp, spares, nspares);
6777
6778                 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
6779                     &nerr) == 0) {
6780                         nvlist_t *nverrlist = NULL;
6781
6782                         /*
6783                          * If the approximate error count is small, get a
6784                          * precise count by fetching the entire log and
6785                          * uniquifying the results.
6786                          */
6787                         if (nerr > 0 && nerr < 100 && !cbp->cb_verbose &&
6788                             zpool_get_errlog(zhp, &nverrlist) == 0) {
6789                                 nvpair_t *elem;
6790
6791                                 elem = NULL;
6792                                 nerr = 0;
6793                                 while ((elem = nvlist_next_nvpair(nverrlist,
6794                                     elem)) != NULL) {
6795                                         nerr++;
6796                                 }
6797                         }
6798                         nvlist_free(nverrlist);
6799
6800                         (void) printf("\n");
6801
6802                         if (nerr == 0)
6803                                 (void) printf(gettext("errors: No known data "
6804                                     "errors\n"));
6805                         else if (!cbp->cb_verbose)
6806                                 (void) printf(gettext("errors: %llu data "
6807                                     "errors, use '-v' for a list\n"),
6808                                     (u_longlong_t)nerr);
6809                         else
6810                                 print_error_log(zhp);
6811                 }
6812
6813                 if (cbp->cb_dedup_stats)
6814                         print_dedup_stats(config);
6815         } else {
6816                 (void) printf(gettext("config: The configuration cannot be "
6817                     "determined.\n"));
6818         }
6819
6820         return (0);
6821 }
6822
6823 /*
6824  * zpool status [-c [script1,script2,...]] [-gLPvx] [-T d|u] [pool] ...
6825  *              [interval [count]]
6826  *
6827  *      -c CMD  For each vdev, run command CMD
6828  *      -g      Display guid for individual vdev name.
6829  *      -L      Follow links when resolving vdev path name.
6830  *      -P      Display full path for vdev name.
6831  *      -v      Display complete error logs
6832  *      -x      Display only pools with potential problems
6833  *      -D      Display dedup status (undocumented)
6834  *      -T      Display a timestamp in date(1) or Unix format
6835  *
6836  * Describes the health status of all pools or some subset.
6837  */
6838 int
6839 zpool_do_status(int argc, char **argv)
6840 {
6841         int c;
6842         int ret;
6843         float interval = 0;
6844         unsigned long count = 0;
6845         status_cbdata_t cb = { 0 };
6846         char *cmd = NULL;
6847
6848         /* check options */
6849         while ((c = getopt(argc, argv, "c:gLPvxDT:")) != -1) {
6850                 switch (c) {
6851                 case 'c':
6852                         if (cmd != NULL) {
6853                                 fprintf(stderr,
6854                                     gettext("Can't set -c flag twice\n"));
6855                                 exit(1);
6856                         }
6857
6858                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
6859                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
6860                                 fprintf(stderr, gettext(
6861                                     "Can't run -c, disabled by "
6862                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
6863                                 exit(1);
6864                         }
6865
6866                         if ((getuid() <= 0 || geteuid() <= 0) &&
6867                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
6868                                 fprintf(stderr, gettext(
6869                                     "Can't run -c with root privileges "
6870                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
6871                                 exit(1);
6872                         }
6873                         cmd = optarg;
6874                         break;
6875                 case 'g':
6876                         cb.cb_name_flags |= VDEV_NAME_GUID;
6877                         break;
6878                 case 'L':
6879                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
6880                         break;
6881                 case 'P':
6882                         cb.cb_name_flags |= VDEV_NAME_PATH;
6883                         break;
6884                 case 'v':
6885                         cb.cb_verbose = B_TRUE;
6886                         break;
6887                 case 'x':
6888                         cb.cb_explain = B_TRUE;
6889                         break;
6890                 case 'D':
6891                         cb.cb_dedup_stats = B_TRUE;
6892                         break;
6893                 case 'T':
6894                         get_timestamp_arg(*optarg);
6895                         break;
6896                 case '?':
6897                         if (optopt == 'c') {
6898                                 print_zpool_script_list("status");
6899                                 exit(0);
6900                         } else {
6901                                 fprintf(stderr,
6902                                     gettext("invalid option '%c'\n"), optopt);
6903                         }
6904                         usage(B_FALSE);
6905                 }
6906         }
6907
6908         argc -= optind;
6909         argv += optind;
6910
6911         get_interval_count(&argc, argv, &interval, &count);
6912
6913         if (argc == 0)
6914                 cb.cb_allpools = B_TRUE;
6915
6916         cb.cb_first = B_TRUE;
6917         cb.cb_print_status = B_TRUE;
6918
6919         for (;;) {
6920                 if (timestamp_fmt != NODATE)
6921                         print_timestamp(timestamp_fmt);
6922
6923                 if (cmd != NULL)
6924                         cb.vcdl = all_pools_for_each_vdev_run(argc, argv, cmd,
6925                             NULL, NULL, 0, 0);
6926
6927                 ret = for_each_pool(argc, argv, B_TRUE, NULL,
6928                     status_callback, &cb);
6929
6930                 if (cb.vcdl != NULL)
6931                         free_vdev_cmd_data_list(cb.vcdl);
6932
6933                 if (argc == 0 && cb.cb_count == 0)
6934                         (void) fprintf(stderr, gettext("no pools available\n"));
6935                 else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
6936                         (void) printf(gettext("all pools are healthy\n"));
6937
6938                 if (ret != 0)
6939                         return (ret);
6940
6941                 if (interval == 0)
6942                         break;
6943
6944                 if (count != 0 && --count == 0)
6945                         break;
6946
6947                 (void) fsleep(interval);
6948         }
6949
6950         return (0);
6951 }
6952
6953 typedef struct upgrade_cbdata {
6954         int     cb_first;
6955         int     cb_argc;
6956         uint64_t cb_version;
6957         char    **cb_argv;
6958 } upgrade_cbdata_t;
6959
6960 static int
6961 check_unsupp_fs(zfs_handle_t *zhp, void *unsupp_fs)
6962 {
6963         int zfs_version = (int)zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
6964         int *count = (int *)unsupp_fs;
6965
6966         if (zfs_version > ZPL_VERSION) {
6967                 (void) printf(gettext("%s (v%d) is not supported by this "
6968                     "implementation of ZFS.\n"),
6969                     zfs_get_name(zhp), zfs_version);
6970                 (*count)++;
6971         }
6972
6973         zfs_iter_filesystems(zhp, check_unsupp_fs, unsupp_fs);
6974
6975         zfs_close(zhp);
6976
6977         return (0);
6978 }
6979
6980 static int
6981 upgrade_version(zpool_handle_t *zhp, uint64_t version)
6982 {
6983         int ret;
6984         nvlist_t *config;
6985         uint64_t oldversion;
6986         int unsupp_fs = 0;
6987
6988         config = zpool_get_config(zhp, NULL);
6989         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
6990             &oldversion) == 0);
6991
6992         assert(SPA_VERSION_IS_SUPPORTED(oldversion));
6993         assert(oldversion < version);
6994
6995         ret = zfs_iter_root(zpool_get_handle(zhp), check_unsupp_fs, &unsupp_fs);
6996         if (ret != 0)
6997                 return (ret);
6998
6999         if (unsupp_fs) {
7000                 (void) fprintf(stderr, gettext("Upgrade not performed due "
7001                     "to %d unsupported filesystems (max v%d).\n"),
7002                     unsupp_fs, (int)ZPL_VERSION);
7003                 return (1);
7004         }
7005
7006         ret = zpool_upgrade(zhp, version);
7007         if (ret != 0)
7008                 return (ret);
7009
7010         if (version >= SPA_VERSION_FEATURES) {
7011                 (void) printf(gettext("Successfully upgraded "
7012                     "'%s' from version %llu to feature flags.\n"),
7013                     zpool_get_name(zhp), (u_longlong_t)oldversion);
7014         } else {
7015                 (void) printf(gettext("Successfully upgraded "
7016                     "'%s' from version %llu to version %llu.\n"),
7017                     zpool_get_name(zhp), (u_longlong_t)oldversion,
7018                     (u_longlong_t)version);
7019         }
7020
7021         return (0);
7022 }
7023
7024 static int
7025 upgrade_enable_all(zpool_handle_t *zhp, int *countp)
7026 {
7027         int i, ret, count;
7028         boolean_t firstff = B_TRUE;
7029         nvlist_t *enabled = zpool_get_features(zhp);
7030
7031         count = 0;
7032         for (i = 0; i < SPA_FEATURES; i++) {
7033                 const char *fname = spa_feature_table[i].fi_uname;
7034                 const char *fguid = spa_feature_table[i].fi_guid;
7035                 if (!nvlist_exists(enabled, fguid)) {
7036                         char *propname;
7037                         verify(-1 != asprintf(&propname, "feature@%s", fname));
7038                         ret = zpool_set_prop(zhp, propname,
7039                             ZFS_FEATURE_ENABLED);
7040                         if (ret != 0) {
7041                                 free(propname);
7042                                 return (ret);
7043                         }
7044                         count++;
7045
7046                         if (firstff) {
7047                                 (void) printf(gettext("Enabled the "
7048                                     "following features on '%s':\n"),
7049                                     zpool_get_name(zhp));
7050                                 firstff = B_FALSE;
7051                         }
7052                         (void) printf(gettext("  %s\n"), fname);
7053                         free(propname);
7054                 }
7055         }
7056
7057         if (countp != NULL)
7058                 *countp = count;
7059         return (0);
7060 }
7061
7062 static int
7063 upgrade_cb(zpool_handle_t *zhp, void *arg)
7064 {
7065         upgrade_cbdata_t *cbp = arg;
7066         nvlist_t *config;
7067         uint64_t version;
7068         boolean_t printnl = B_FALSE;
7069         int ret;
7070
7071         config = zpool_get_config(zhp, NULL);
7072         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7073             &version) == 0);
7074
7075         assert(SPA_VERSION_IS_SUPPORTED(version));
7076
7077         if (version < cbp->cb_version) {
7078                 cbp->cb_first = B_FALSE;
7079                 ret = upgrade_version(zhp, cbp->cb_version);
7080                 if (ret != 0)
7081                         return (ret);
7082                 printnl = B_TRUE;
7083
7084                 /*
7085                  * If they did "zpool upgrade -a", then we could
7086                  * be doing ioctls to different pools.  We need
7087                  * to log this history once to each pool, and bypass
7088                  * the normal history logging that happens in main().
7089                  */
7090                 (void) zpool_log_history(g_zfs, history_str);
7091                 log_history = B_FALSE;
7092         }
7093
7094         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
7095                 int count;
7096                 ret = upgrade_enable_all(zhp, &count);
7097                 if (ret != 0)
7098                         return (ret);
7099
7100                 if (count > 0) {
7101                         cbp->cb_first = B_FALSE;
7102                         printnl = B_TRUE;
7103                 }
7104         }
7105
7106         if (printnl) {
7107                 (void) printf(gettext("\n"));
7108         }
7109
7110         return (0);
7111 }
7112
7113 static int
7114 upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
7115 {
7116         upgrade_cbdata_t *cbp = arg;
7117         nvlist_t *config;
7118         uint64_t version;
7119
7120         config = zpool_get_config(zhp, NULL);
7121         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7122             &version) == 0);
7123
7124         assert(SPA_VERSION_IS_SUPPORTED(version));
7125
7126         if (version < SPA_VERSION_FEATURES) {
7127                 if (cbp->cb_first) {
7128                         (void) printf(gettext("The following pools are "
7129                             "formatted with legacy version numbers and can\n"
7130                             "be upgraded to use feature flags.  After "
7131                             "being upgraded, these pools\nwill no "
7132                             "longer be accessible by software that does not "
7133                             "support feature\nflags.\n\n"));
7134                         (void) printf(gettext("VER  POOL\n"));
7135                         (void) printf(gettext("---  ------------\n"));
7136                         cbp->cb_first = B_FALSE;
7137                 }
7138
7139                 (void) printf("%2llu   %s\n", (u_longlong_t)version,
7140                     zpool_get_name(zhp));
7141         }
7142
7143         return (0);
7144 }
7145
7146 static int
7147 upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
7148 {
7149         upgrade_cbdata_t *cbp = arg;
7150         nvlist_t *config;
7151         uint64_t version;
7152
7153         config = zpool_get_config(zhp, NULL);
7154         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7155             &version) == 0);
7156
7157         if (version >= SPA_VERSION_FEATURES) {
7158                 int i;
7159                 boolean_t poolfirst = B_TRUE;
7160                 nvlist_t *enabled = zpool_get_features(zhp);
7161
7162                 for (i = 0; i < SPA_FEATURES; i++) {
7163                         const char *fguid = spa_feature_table[i].fi_guid;
7164                         const char *fname = spa_feature_table[i].fi_uname;
7165                         if (!nvlist_exists(enabled, fguid)) {
7166                                 if (cbp->cb_first) {
7167                                         (void) printf(gettext("\nSome "
7168                                             "supported features are not "
7169                                             "enabled on the following pools. "
7170                                             "Once a\nfeature is enabled the "
7171                                             "pool may become incompatible with "
7172                                             "software\nthat does not support "
7173                                             "the feature. See "
7174                                             "zpool-features(5) for "
7175                                             "details.\n\n"));
7176                                         (void) printf(gettext("POOL  "
7177                                             "FEATURE\n"));
7178                                         (void) printf(gettext("------"
7179                                             "---------\n"));
7180                                         cbp->cb_first = B_FALSE;
7181                                 }
7182
7183                                 if (poolfirst) {
7184                                         (void) printf(gettext("%s\n"),
7185                                             zpool_get_name(zhp));
7186                                         poolfirst = B_FALSE;
7187                                 }
7188
7189                                 (void) printf(gettext("      %s\n"), fname);
7190                         }
7191                         /*
7192                          * If they did "zpool upgrade -a", then we could
7193                          * be doing ioctls to different pools.  We need
7194                          * to log this history once to each pool, and bypass
7195                          * the normal history logging that happens in main().
7196                          */
7197                         (void) zpool_log_history(g_zfs, history_str);
7198                         log_history = B_FALSE;
7199                 }
7200         }
7201
7202         return (0);
7203 }
7204
7205 /* ARGSUSED */
7206 static int
7207 upgrade_one(zpool_handle_t *zhp, void *data)
7208 {
7209         boolean_t printnl = B_FALSE;
7210         upgrade_cbdata_t *cbp = data;
7211         uint64_t cur_version;
7212         int ret;
7213
7214         if (strcmp("log", zpool_get_name(zhp)) == 0) {
7215                 (void) fprintf(stderr, gettext("'log' is now a reserved word\n"
7216                     "Pool 'log' must be renamed using export and import"
7217                     " to upgrade.\n"));
7218                 return (1);
7219         }
7220
7221         cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
7222         if (cur_version > cbp->cb_version) {
7223                 (void) printf(gettext("Pool '%s' is already formatted "
7224                     "using more current version '%llu'.\n\n"),
7225                     zpool_get_name(zhp), (u_longlong_t)cur_version);
7226                 return (0);
7227         }
7228
7229         if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
7230                 (void) printf(gettext("Pool '%s' is already formatted "
7231                     "using version %llu.\n\n"), zpool_get_name(zhp),
7232                     (u_longlong_t)cbp->cb_version);
7233                 return (0);
7234         }
7235
7236         if (cur_version != cbp->cb_version) {
7237                 printnl = B_TRUE;
7238                 ret = upgrade_version(zhp, cbp->cb_version);
7239                 if (ret != 0)
7240                         return (ret);
7241         }
7242
7243         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
7244                 int count = 0;
7245                 ret = upgrade_enable_all(zhp, &count);
7246                 if (ret != 0)
7247                         return (ret);
7248
7249                 if (count != 0) {
7250                         printnl = B_TRUE;
7251                 } else if (cur_version == SPA_VERSION) {
7252                         (void) printf(gettext("Pool '%s' already has all "
7253                             "supported features enabled.\n"),
7254                             zpool_get_name(zhp));
7255                 }
7256         }
7257
7258         if (printnl) {
7259                 (void) printf(gettext("\n"));
7260         }
7261
7262         return (0);
7263 }
7264
7265 /*
7266  * zpool upgrade
7267  * zpool upgrade -v
7268  * zpool upgrade [-V version] <-a | pool ...>
7269  *
7270  * With no arguments, display downrev'd ZFS pool available for upgrade.
7271  * Individual pools can be upgraded by specifying the pool, and '-a' will
7272  * upgrade all pools.
7273  */
7274 int
7275 zpool_do_upgrade(int argc, char **argv)
7276 {
7277         int c;
7278         upgrade_cbdata_t cb = { 0 };
7279         int ret = 0;
7280         boolean_t showversions = B_FALSE;
7281         boolean_t upgradeall = B_FALSE;
7282         char *end;
7283
7284
7285         /* check options */
7286         while ((c = getopt(argc, argv, ":avV:")) != -1) {
7287                 switch (c) {
7288                 case 'a':
7289                         upgradeall = B_TRUE;
7290                         break;
7291                 case 'v':
7292                         showversions = B_TRUE;
7293                         break;
7294                 case 'V':
7295                         cb.cb_version = strtoll(optarg, &end, 10);
7296                         if (*end != '\0' ||
7297                             !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
7298                                 (void) fprintf(stderr,
7299                                     gettext("invalid version '%s'\n"), optarg);
7300                                 usage(B_FALSE);
7301                         }
7302                         break;
7303                 case ':':
7304                         (void) fprintf(stderr, gettext("missing argument for "
7305                             "'%c' option\n"), optopt);
7306                         usage(B_FALSE);
7307                         break;
7308                 case '?':
7309                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7310                             optopt);
7311                         usage(B_FALSE);
7312                 }
7313         }
7314
7315         cb.cb_argc = argc;
7316         cb.cb_argv = argv;
7317         argc -= optind;
7318         argv += optind;
7319
7320         if (cb.cb_version == 0) {
7321                 cb.cb_version = SPA_VERSION;
7322         } else if (!upgradeall && argc == 0) {
7323                 (void) fprintf(stderr, gettext("-V option is "
7324                     "incompatible with other arguments\n"));
7325                 usage(B_FALSE);
7326         }
7327
7328         if (showversions) {
7329                 if (upgradeall || argc != 0) {
7330                         (void) fprintf(stderr, gettext("-v option is "
7331                             "incompatible with other arguments\n"));
7332                         usage(B_FALSE);
7333                 }
7334         } else if (upgradeall) {
7335                 if (argc != 0) {
7336                         (void) fprintf(stderr, gettext("-a option should not "
7337                             "be used along with a pool name\n"));
7338                         usage(B_FALSE);
7339                 }
7340         }
7341
7342         (void) printf(gettext("This system supports ZFS pool feature "
7343             "flags.\n\n"));
7344         if (showversions) {
7345                 int i;
7346
7347                 (void) printf(gettext("The following features are "
7348                     "supported:\n\n"));
7349                 (void) printf(gettext("FEAT DESCRIPTION\n"));
7350                 (void) printf("----------------------------------------------"
7351                     "---------------\n");
7352                 for (i = 0; i < SPA_FEATURES; i++) {
7353                         zfeature_info_t *fi = &spa_feature_table[i];
7354                         const char *ro =
7355                             (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
7356                             " (read-only compatible)" : "";
7357
7358                         (void) printf("%-37s%s\n", fi->fi_uname, ro);
7359                         (void) printf("     %s\n", fi->fi_desc);
7360                 }
7361                 (void) printf("\n");
7362
7363                 (void) printf(gettext("The following legacy versions are also "
7364                     "supported:\n\n"));
7365                 (void) printf(gettext("VER  DESCRIPTION\n"));
7366                 (void) printf("---  -----------------------------------------"
7367                     "---------------\n");
7368                 (void) printf(gettext(" 1   Initial ZFS version\n"));
7369                 (void) printf(gettext(" 2   Ditto blocks "
7370                     "(replicated metadata)\n"));
7371                 (void) printf(gettext(" 3   Hot spares and double parity "
7372                     "RAID-Z\n"));
7373                 (void) printf(gettext(" 4   zpool history\n"));
7374                 (void) printf(gettext(" 5   Compression using the gzip "
7375                     "algorithm\n"));
7376                 (void) printf(gettext(" 6   bootfs pool property\n"));
7377                 (void) printf(gettext(" 7   Separate intent log devices\n"));
7378                 (void) printf(gettext(" 8   Delegated administration\n"));
7379                 (void) printf(gettext(" 9   refquota and refreservation "
7380                     "properties\n"));
7381                 (void) printf(gettext(" 10  Cache devices\n"));
7382                 (void) printf(gettext(" 11  Improved scrub performance\n"));
7383                 (void) printf(gettext(" 12  Snapshot properties\n"));
7384                 (void) printf(gettext(" 13  snapused property\n"));
7385                 (void) printf(gettext(" 14  passthrough-x aclinherit\n"));
7386                 (void) printf(gettext(" 15  user/group space accounting\n"));
7387                 (void) printf(gettext(" 16  stmf property support\n"));
7388                 (void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
7389                 (void) printf(gettext(" 18  Snapshot user holds\n"));
7390                 (void) printf(gettext(" 19  Log device removal\n"));
7391                 (void) printf(gettext(" 20  Compression using zle "
7392                     "(zero-length encoding)\n"));
7393                 (void) printf(gettext(" 21  Deduplication\n"));
7394                 (void) printf(gettext(" 22  Received properties\n"));
7395                 (void) printf(gettext(" 23  Slim ZIL\n"));
7396                 (void) printf(gettext(" 24  System attributes\n"));
7397                 (void) printf(gettext(" 25  Improved scrub stats\n"));
7398                 (void) printf(gettext(" 26  Improved snapshot deletion "
7399                     "performance\n"));
7400                 (void) printf(gettext(" 27  Improved snapshot creation "
7401                     "performance\n"));
7402                 (void) printf(gettext(" 28  Multiple vdev replacements\n"));
7403                 (void) printf(gettext("\nFor more information on a particular "
7404                     "version, including supported releases,\n"));
7405                 (void) printf(gettext("see the ZFS Administration Guide.\n\n"));
7406         } else if (argc == 0 && upgradeall) {
7407                 cb.cb_first = B_TRUE;
7408                 ret = zpool_iter(g_zfs, upgrade_cb, &cb);
7409                 if (ret == 0 && cb.cb_first) {
7410                         if (cb.cb_version == SPA_VERSION) {
7411                                 (void) printf(gettext("All pools are already "
7412                                     "formatted using feature flags.\n\n"));
7413                                 (void) printf(gettext("Every feature flags "
7414                                     "pool already has all supported features "
7415                                     "enabled.\n"));
7416                         } else {
7417                                 (void) printf(gettext("All pools are already "
7418                                     "formatted with version %llu or higher.\n"),
7419                                     (u_longlong_t)cb.cb_version);
7420                         }
7421                 }
7422         } else if (argc == 0) {
7423                 cb.cb_first = B_TRUE;
7424                 ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
7425                 assert(ret == 0);
7426
7427                 if (cb.cb_first) {
7428                         (void) printf(gettext("All pools are formatted "
7429                             "using feature flags.\n\n"));
7430                 } else {
7431                         (void) printf(gettext("\nUse 'zpool upgrade -v' "
7432                             "for a list of available legacy versions.\n"));
7433                 }
7434
7435                 cb.cb_first = B_TRUE;
7436                 ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
7437                 assert(ret == 0);
7438
7439                 if (cb.cb_first) {
7440                         (void) printf(gettext("Every feature flags pool has "
7441                             "all supported features enabled.\n"));
7442                 } else {
7443                         (void) printf(gettext("\n"));
7444                 }
7445         } else {
7446                 ret = for_each_pool(argc, argv, B_FALSE, NULL,
7447                     upgrade_one, &cb);
7448         }
7449
7450         return (ret);
7451 }
7452
7453 typedef struct hist_cbdata {
7454         boolean_t first;
7455         boolean_t longfmt;
7456         boolean_t internal;
7457 } hist_cbdata_t;
7458
7459 /*
7460  * Print out the command history for a specific pool.
7461  */
7462 static int
7463 get_history_one(zpool_handle_t *zhp, void *data)
7464 {
7465         nvlist_t *nvhis;
7466         nvlist_t **records;
7467         uint_t numrecords;
7468         int ret, i;
7469         hist_cbdata_t *cb = (hist_cbdata_t *)data;
7470
7471         cb->first = B_FALSE;
7472
7473         (void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
7474
7475         if ((ret = zpool_get_history(zhp, &nvhis)) != 0)
7476                 return (ret);
7477
7478         verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
7479             &records, &numrecords) == 0);
7480         for (i = 0; i < numrecords; i++) {
7481                 nvlist_t *rec = records[i];
7482                 char tbuf[30] = "";
7483
7484                 if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
7485                         time_t tsec;
7486                         struct tm t;
7487
7488                         tsec = fnvlist_lookup_uint64(records[i],
7489                             ZPOOL_HIST_TIME);
7490                         (void) localtime_r(&tsec, &t);
7491                         (void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
7492                 }
7493
7494                 if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
7495                         (void) printf("%s %s", tbuf,
7496                             fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
7497                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
7498                         int ievent =
7499                             fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
7500                         if (!cb->internal)
7501                                 continue;
7502                         if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
7503                                 (void) printf("%s unrecognized record:\n",
7504                                     tbuf);
7505                                 dump_nvlist(rec, 4);
7506                                 continue;
7507                         }
7508                         (void) printf("%s [internal %s txg:%lld] %s", tbuf,
7509                             zfs_history_event_names[ievent],
7510                             (longlong_t)fnvlist_lookup_uint64(
7511                             rec, ZPOOL_HIST_TXG),
7512                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
7513                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
7514                         if (!cb->internal)
7515                                 continue;
7516                         (void) printf("%s [txg:%lld] %s", tbuf,
7517                             (longlong_t)fnvlist_lookup_uint64(
7518                             rec, ZPOOL_HIST_TXG),
7519                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
7520                         if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
7521                                 (void) printf(" %s (%llu)",
7522                                     fnvlist_lookup_string(rec,
7523                                     ZPOOL_HIST_DSNAME),
7524                                     (u_longlong_t)fnvlist_lookup_uint64(rec,
7525                                     ZPOOL_HIST_DSID));
7526                         }
7527                         (void) printf(" %s", fnvlist_lookup_string(rec,
7528                             ZPOOL_HIST_INT_STR));
7529                 } else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
7530                         if (!cb->internal)
7531                                 continue;
7532                         (void) printf("%s ioctl %s\n", tbuf,
7533                             fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
7534                         if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
7535                                 (void) printf("    input:\n");
7536                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
7537                                     ZPOOL_HIST_INPUT_NVL), 8);
7538                         }
7539                         if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
7540                                 (void) printf("    output:\n");
7541                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
7542                                     ZPOOL_HIST_OUTPUT_NVL), 8);
7543                         }
7544                         if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
7545                                 (void) printf("    errno: %lld\n",
7546                                     (longlong_t)fnvlist_lookup_int64(rec,
7547                                     ZPOOL_HIST_ERRNO));
7548                         }
7549                 } else {
7550                         if (!cb->internal)
7551                                 continue;
7552                         (void) printf("%s unrecognized record:\n", tbuf);
7553                         dump_nvlist(rec, 4);
7554                 }
7555
7556                 if (!cb->longfmt) {
7557                         (void) printf("\n");
7558                         continue;
7559                 }
7560                 (void) printf(" [");
7561                 if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
7562                         uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
7563                         struct passwd *pwd = getpwuid(who);
7564                         (void) printf("user %d ", (int)who);
7565                         if (pwd != NULL)
7566                                 (void) printf("(%s) ", pwd->pw_name);
7567                 }
7568                 if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
7569                         (void) printf("on %s",
7570                             fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
7571                 }
7572                 if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
7573                         (void) printf(":%s",
7574                             fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
7575                 }
7576
7577                 (void) printf("]");
7578                 (void) printf("\n");
7579         }
7580         (void) printf("\n");
7581         nvlist_free(nvhis);
7582
7583         return (ret);
7584 }
7585
7586 /*
7587  * zpool history <pool>
7588  *
7589  * Displays the history of commands that modified pools.
7590  */
7591 int
7592 zpool_do_history(int argc, char **argv)
7593 {
7594         hist_cbdata_t cbdata = { 0 };
7595         int ret;
7596         int c;
7597
7598         cbdata.first = B_TRUE;
7599         /* check options */
7600         while ((c = getopt(argc, argv, "li")) != -1) {
7601                 switch (c) {
7602                 case 'l':
7603                         cbdata.longfmt = B_TRUE;
7604                         break;
7605                 case 'i':
7606                         cbdata.internal = B_TRUE;
7607                         break;
7608                 case '?':
7609                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7610                             optopt);
7611                         usage(B_FALSE);
7612                 }
7613         }
7614         argc -= optind;
7615         argv += optind;
7616
7617         ret = for_each_pool(argc, argv, B_FALSE,  NULL, get_history_one,
7618             &cbdata);
7619
7620         if (argc == 0 && cbdata.first == B_TRUE) {
7621                 (void) fprintf(stderr, gettext("no pools available\n"));
7622                 return (0);
7623         }
7624
7625         return (ret);
7626 }
7627
7628 typedef struct ev_opts {
7629         int verbose;
7630         int scripted;
7631         int follow;
7632         int clear;
7633         char poolname[ZFS_MAX_DATASET_NAME_LEN];
7634 } ev_opts_t;
7635
7636 static void
7637 zpool_do_events_short(nvlist_t *nvl, ev_opts_t *opts)
7638 {
7639         char ctime_str[26], str[32], *ptr;
7640         int64_t *tv;
7641         uint_t n;
7642
7643         verify(nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0);
7644         memset(str, ' ', 32);
7645         (void) ctime_r((const time_t *)&tv[0], ctime_str);
7646         (void) strncpy(str, ctime_str+4,  6);           /* 'Jun 30' */
7647         (void) strncpy(str+7, ctime_str+20, 4);         /* '1993' */
7648         (void) strncpy(str+12, ctime_str+11, 8);        /* '21:49:08' */
7649         (void) sprintf(str+20, ".%09lld", (longlong_t)tv[1]); /* '.123456789' */
7650         if (opts->scripted)
7651                 (void) printf(gettext("%s\t"), str);
7652         else
7653                 (void) printf(gettext("%s "), str);
7654
7655         verify(nvlist_lookup_string(nvl, FM_CLASS, &ptr) == 0);
7656         (void) printf(gettext("%s\n"), ptr);
7657 }
7658
7659 static void
7660 zpool_do_events_nvprint(nvlist_t *nvl, int depth)
7661 {
7662         nvpair_t *nvp;
7663
7664         for (nvp = nvlist_next_nvpair(nvl, NULL);
7665             nvp != NULL; nvp = nvlist_next_nvpair(nvl, nvp)) {
7666
7667                 data_type_t type = nvpair_type(nvp);
7668                 const char *name = nvpair_name(nvp);
7669
7670                 boolean_t b;
7671                 uint8_t i8;
7672                 uint16_t i16;
7673                 uint32_t i32;
7674                 uint64_t i64;
7675                 char *str;
7676                 nvlist_t *cnv;
7677
7678                 printf(gettext("%*s%s = "), depth, "", name);
7679
7680                 switch (type) {
7681                 case DATA_TYPE_BOOLEAN:
7682                         printf(gettext("%s"), "1");
7683                         break;
7684
7685                 case DATA_TYPE_BOOLEAN_VALUE:
7686                         (void) nvpair_value_boolean_value(nvp, &b);
7687                         printf(gettext("%s"), b ? "1" : "0");
7688                         break;
7689
7690                 case DATA_TYPE_BYTE:
7691                         (void) nvpair_value_byte(nvp, &i8);
7692                         printf(gettext("0x%x"), i8);
7693                         break;
7694
7695                 case DATA_TYPE_INT8:
7696                         (void) nvpair_value_int8(nvp, (void *)&i8);
7697                         printf(gettext("0x%x"), i8);
7698                         break;
7699
7700                 case DATA_TYPE_UINT8:
7701                         (void) nvpair_value_uint8(nvp, &i8);
7702                         printf(gettext("0x%x"), i8);
7703                         break;
7704
7705                 case DATA_TYPE_INT16:
7706                         (void) nvpair_value_int16(nvp, (void *)&i16);
7707                         printf(gettext("0x%x"), i16);
7708                         break;
7709
7710                 case DATA_TYPE_UINT16:
7711                         (void) nvpair_value_uint16(nvp, &i16);
7712                         printf(gettext("0x%x"), i16);
7713                         break;
7714
7715                 case DATA_TYPE_INT32:
7716                         (void) nvpair_value_int32(nvp, (void *)&i32);
7717                         printf(gettext("0x%x"), i32);
7718                         break;
7719
7720                 case DATA_TYPE_UINT32:
7721                         (void) nvpair_value_uint32(nvp, &i32);
7722                         printf(gettext("0x%x"), i32);
7723                         break;
7724
7725                 case DATA_TYPE_INT64:
7726                         (void) nvpair_value_int64(nvp, (void *)&i64);
7727                         printf(gettext("0x%llx"), (u_longlong_t)i64);
7728                         break;
7729
7730                 case DATA_TYPE_UINT64:
7731                         (void) nvpair_value_uint64(nvp, &i64);
7732                         /*
7733                          * translate vdev state values to readable
7734                          * strings to aide zpool events consumers
7735                          */
7736                         if (strcmp(name,
7737                             FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE) == 0 ||
7738                             strcmp(name,
7739                             FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE) == 0) {
7740                                 printf(gettext("\"%s\" (0x%llx)"),
7741                                     zpool_state_to_name(i64, VDEV_AUX_NONE),
7742                                     (u_longlong_t)i64);
7743                         } else {
7744                                 printf(gettext("0x%llx"), (u_longlong_t)i64);
7745                         }
7746                         break;
7747
7748                 case DATA_TYPE_HRTIME:
7749                         (void) nvpair_value_hrtime(nvp, (void *)&i64);
7750                         printf(gettext("0x%llx"), (u_longlong_t)i64);
7751                         break;
7752
7753                 case DATA_TYPE_STRING:
7754                         (void) nvpair_value_string(nvp, &str);
7755                         printf(gettext("\"%s\""), str ? str : "<NULL>");
7756                         break;
7757
7758                 case DATA_TYPE_NVLIST:
7759                         printf(gettext("(embedded nvlist)\n"));
7760                         (void) nvpair_value_nvlist(nvp, &cnv);
7761                         zpool_do_events_nvprint(cnv, depth + 8);
7762                         printf(gettext("%*s(end %s)"), depth, "", name);
7763                         break;
7764
7765                 case DATA_TYPE_NVLIST_ARRAY: {
7766                         nvlist_t **val;
7767                         uint_t i, nelem;
7768
7769                         (void) nvpair_value_nvlist_array(nvp, &val, &nelem);
7770                         printf(gettext("(%d embedded nvlists)\n"), nelem);
7771                         for (i = 0; i < nelem; i++) {
7772                                 printf(gettext("%*s%s[%d] = %s\n"),
7773                                     depth, "", name, i, "(embedded nvlist)");
7774                                 zpool_do_events_nvprint(val[i], depth + 8);
7775                                 printf(gettext("%*s(end %s[%i])\n"),
7776                                     depth, "", name, i);
7777                         }
7778                         printf(gettext("%*s(end %s)\n"), depth, "", name);
7779                         }
7780                         break;
7781
7782                 case DATA_TYPE_INT8_ARRAY: {
7783                         int8_t *val;
7784                         uint_t i, nelem;
7785
7786                         (void) nvpair_value_int8_array(nvp, &val, &nelem);
7787                         for (i = 0; i < nelem; i++)
7788                                 printf(gettext("0x%x "), val[i]);
7789
7790                         break;
7791                         }
7792
7793                 case DATA_TYPE_UINT8_ARRAY: {
7794                         uint8_t *val;
7795                         uint_t i, nelem;
7796
7797                         (void) nvpair_value_uint8_array(nvp, &val, &nelem);
7798                         for (i = 0; i < nelem; i++)
7799                                 printf(gettext("0x%x "), val[i]);
7800
7801                         break;
7802                         }
7803
7804                 case DATA_TYPE_INT16_ARRAY: {
7805                         int16_t *val;
7806                         uint_t i, nelem;
7807
7808                         (void) nvpair_value_int16_array(nvp, &val, &nelem);
7809                         for (i = 0; i < nelem; i++)
7810                                 printf(gettext("0x%x "), val[i]);
7811
7812                         break;
7813                         }
7814
7815                 case DATA_TYPE_UINT16_ARRAY: {
7816                         uint16_t *val;
7817                         uint_t i, nelem;
7818
7819                         (void) nvpair_value_uint16_array(nvp, &val, &nelem);
7820                         for (i = 0; i < nelem; i++)
7821                                 printf(gettext("0x%x "), val[i]);
7822
7823                         break;
7824                         }
7825
7826                 case DATA_TYPE_INT32_ARRAY: {
7827                         int32_t *val;
7828                         uint_t i, nelem;
7829
7830                         (void) nvpair_value_int32_array(nvp, &val, &nelem);
7831                         for (i = 0; i < nelem; i++)
7832                                 printf(gettext("0x%x "), val[i]);
7833
7834                         break;
7835                         }
7836
7837                 case DATA_TYPE_UINT32_ARRAY: {
7838                         uint32_t *val;
7839                         uint_t i, nelem;
7840
7841                         (void) nvpair_value_uint32_array(nvp, &val, &nelem);
7842                         for (i = 0; i < nelem; i++)
7843                                 printf(gettext("0x%x "), val[i]);
7844
7845                         break;
7846                         }
7847
7848                 case DATA_TYPE_INT64_ARRAY: {
7849                         int64_t *val;
7850                         uint_t i, nelem;
7851
7852                         (void) nvpair_value_int64_array(nvp, &val, &nelem);
7853                         for (i = 0; i < nelem; i++)
7854                                 printf(gettext("0x%llx "),
7855                                     (u_longlong_t)val[i]);
7856
7857                         break;
7858                         }
7859
7860                 case DATA_TYPE_UINT64_ARRAY: {
7861                         uint64_t *val;
7862                         uint_t i, nelem;
7863
7864                         (void) nvpair_value_uint64_array(nvp, &val, &nelem);
7865                         for (i = 0; i < nelem; i++)
7866                                 printf(gettext("0x%llx "),
7867                                     (u_longlong_t)val[i]);
7868
7869                         break;
7870                         }
7871
7872                 case DATA_TYPE_STRING_ARRAY: {
7873                         char **str;
7874                         uint_t i, nelem;
7875
7876                         (void) nvpair_value_string_array(nvp, &str, &nelem);
7877                         for (i = 0; i < nelem; i++)
7878                                 printf(gettext("\"%s\" "),
7879                                     str[i] ? str[i] : "<NULL>");
7880
7881                         break;
7882                         }
7883
7884                 case DATA_TYPE_BOOLEAN_ARRAY:
7885                 case DATA_TYPE_BYTE_ARRAY:
7886                 case DATA_TYPE_DOUBLE:
7887                 case DATA_TYPE_UNKNOWN:
7888                         printf(gettext("<unknown>"));
7889                         break;
7890                 }
7891
7892                 printf(gettext("\n"));
7893         }
7894 }
7895
7896 static int
7897 zpool_do_events_next(ev_opts_t *opts)
7898 {
7899         nvlist_t *nvl;
7900         int zevent_fd, ret, dropped;
7901         char *pool;
7902
7903         zevent_fd = open(ZFS_DEV, O_RDWR);
7904         VERIFY(zevent_fd >= 0);
7905
7906         if (!opts->scripted)
7907                 (void) printf(gettext("%-30s %s\n"), "TIME", "CLASS");
7908
7909         while (1) {
7910                 ret = zpool_events_next(g_zfs, &nvl, &dropped,
7911                     (opts->follow ? ZEVENT_NONE : ZEVENT_NONBLOCK), zevent_fd);
7912                 if (ret || nvl == NULL)
7913                         break;
7914
7915                 if (dropped > 0)
7916                         (void) printf(gettext("dropped %d events\n"), dropped);
7917
7918                 if (strlen(opts->poolname) > 0 &&
7919                     nvlist_lookup_string(nvl, FM_FMRI_ZFS_POOL, &pool) == 0 &&
7920                     strcmp(opts->poolname, pool) != 0)
7921                         continue;
7922
7923                 zpool_do_events_short(nvl, opts);
7924
7925                 if (opts->verbose) {
7926                         zpool_do_events_nvprint(nvl, 8);
7927                         printf(gettext("\n"));
7928                 }
7929                 (void) fflush(stdout);
7930
7931                 nvlist_free(nvl);
7932         }
7933
7934         VERIFY(0 == close(zevent_fd));
7935
7936         return (ret);
7937 }
7938
7939 static int
7940 zpool_do_events_clear(ev_opts_t *opts)
7941 {
7942         int count, ret;
7943
7944         ret = zpool_events_clear(g_zfs, &count);
7945         if (!ret)
7946                 (void) printf(gettext("cleared %d events\n"), count);
7947
7948         return (ret);
7949 }
7950
7951 /*
7952  * zpool events [-vHf [pool] | -c]
7953  *
7954  * Displays events logs by ZFS.
7955  */
7956 int
7957 zpool_do_events(int argc, char **argv)
7958 {
7959         ev_opts_t opts = { 0 };
7960         int ret;
7961         int c;
7962
7963         /* check options */
7964         while ((c = getopt(argc, argv, "vHfc")) != -1) {
7965                 switch (c) {
7966                 case 'v':
7967                         opts.verbose = 1;
7968                         break;
7969                 case 'H':
7970                         opts.scripted = 1;
7971                         break;
7972                 case 'f':
7973                         opts.follow = 1;
7974                         break;
7975                 case 'c':
7976                         opts.clear = 1;
7977                         break;
7978                 case '?':
7979                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7980                             optopt);
7981                         usage(B_FALSE);
7982                 }
7983         }
7984         argc -= optind;
7985         argv += optind;
7986
7987         if (argc > 1) {
7988                 (void) fprintf(stderr, gettext("too many arguments\n"));
7989                 usage(B_FALSE);
7990         } else if (argc == 1) {
7991                 (void) strlcpy(opts.poolname, argv[0], sizeof (opts.poolname));
7992                 if (!zfs_name_valid(opts.poolname, ZFS_TYPE_POOL)) {
7993                         (void) fprintf(stderr,
7994                             gettext("invalid pool name '%s'\n"), opts.poolname);
7995                         usage(B_FALSE);
7996                 }
7997         }
7998
7999         if ((argc == 1 || opts.verbose || opts.scripted || opts.follow) &&
8000             opts.clear) {
8001                 (void) fprintf(stderr,
8002                     gettext("invalid options combined with -c\n"));
8003                 usage(B_FALSE);
8004         }
8005
8006         if (opts.clear)
8007                 ret = zpool_do_events_clear(&opts);
8008         else
8009                 ret = zpool_do_events_next(&opts);
8010
8011         return (ret);
8012 }
8013
8014 static int
8015 get_callback(zpool_handle_t *zhp, void *data)
8016 {
8017         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
8018         char value[MAXNAMELEN];
8019         zprop_source_t srctype;
8020         zprop_list_t *pl;
8021
8022         for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
8023
8024                 /*
8025                  * Skip the special fake placeholder. This will also skip
8026                  * over the name property when 'all' is specified.
8027                  */
8028                 if (pl->pl_prop == ZPOOL_PROP_NAME &&
8029                     pl == cbp->cb_proplist)
8030                         continue;
8031
8032                 if (pl->pl_prop == ZPROP_INVAL &&
8033                     (zpool_prop_feature(pl->pl_user_prop) ||
8034                     zpool_prop_unsupported(pl->pl_user_prop))) {
8035                         srctype = ZPROP_SRC_LOCAL;
8036
8037                         if (zpool_prop_get_feature(zhp, pl->pl_user_prop,
8038                             value, sizeof (value)) == 0) {
8039                                 zprop_print_one_property(zpool_get_name(zhp),
8040                                     cbp, pl->pl_user_prop, value, srctype,
8041                                     NULL, NULL);
8042                         }
8043                 } else {
8044                         if (zpool_get_prop(zhp, pl->pl_prop, value,
8045                             sizeof (value), &srctype, cbp->cb_literal) != 0)
8046                                 continue;
8047
8048                         zprop_print_one_property(zpool_get_name(zhp), cbp,
8049                             zpool_prop_to_name(pl->pl_prop), value, srctype,
8050                             NULL, NULL);
8051                 }
8052         }
8053         return (0);
8054 }
8055
8056 /*
8057  * zpool get [-Hp] [-o "all" | field[,...]] <"all" | property[,...]> <pool> ...
8058  *
8059  *      -H      Scripted mode.  Don't display headers, and separate properties
8060  *              by a single tab.
8061  *      -o      List of columns to display.  Defaults to
8062  *              "name,property,value,source".
8063  *      -p      Display values in parsable (exact) format.
8064  *
8065  * Get properties of pools in the system. Output space statistics
8066  * for each one as well as other attributes.
8067  */
8068 int
8069 zpool_do_get(int argc, char **argv)
8070 {
8071         zprop_get_cbdata_t cb = { 0 };
8072         zprop_list_t fake_name = { 0 };
8073         int ret;
8074         int c, i;
8075         char *value;
8076
8077         cb.cb_first = B_TRUE;
8078
8079         /*
8080          * Set up default columns and sources.
8081          */
8082         cb.cb_sources = ZPROP_SRC_ALL;
8083         cb.cb_columns[0] = GET_COL_NAME;
8084         cb.cb_columns[1] = GET_COL_PROPERTY;
8085         cb.cb_columns[2] = GET_COL_VALUE;
8086         cb.cb_columns[3] = GET_COL_SOURCE;
8087         cb.cb_type = ZFS_TYPE_POOL;
8088
8089         /* check options */
8090         while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
8091                 switch (c) {
8092                 case 'p':
8093                         cb.cb_literal = B_TRUE;
8094                         break;
8095                 case 'H':
8096                         cb.cb_scripted = B_TRUE;
8097                         break;
8098                 case 'o':
8099                         bzero(&cb.cb_columns, sizeof (cb.cb_columns));
8100                         i = 0;
8101                         while (*optarg != '\0') {
8102                                 static char *col_subopts[] =
8103                                 { "name", "property", "value", "source",
8104                                 "all", NULL };
8105
8106                                 if (i == ZFS_GET_NCOLS) {
8107                                         (void) fprintf(stderr, gettext("too "
8108                                         "many fields given to -o "
8109                                         "option\n"));
8110                                         usage(B_FALSE);
8111                                 }
8112
8113                                 switch (getsubopt(&optarg, col_subopts,
8114                                     &value)) {
8115                                 case 0:
8116                                         cb.cb_columns[i++] = GET_COL_NAME;
8117                                         break;
8118                                 case 1:
8119                                         cb.cb_columns[i++] = GET_COL_PROPERTY;
8120                                         break;
8121                                 case 2:
8122                                         cb.cb_columns[i++] = GET_COL_VALUE;
8123                                         break;
8124                                 case 3:
8125                                         cb.cb_columns[i++] = GET_COL_SOURCE;
8126                                         break;
8127                                 case 4:
8128                                         if (i > 0) {
8129                                                 (void) fprintf(stderr,
8130                                                     gettext("\"all\" conflicts "
8131                                                     "with specific fields "
8132                                                     "given to -o option\n"));
8133                                                 usage(B_FALSE);
8134                                         }
8135                                         cb.cb_columns[0] = GET_COL_NAME;
8136                                         cb.cb_columns[1] = GET_COL_PROPERTY;
8137                                         cb.cb_columns[2] = GET_COL_VALUE;
8138                                         cb.cb_columns[3] = GET_COL_SOURCE;
8139                                         i = ZFS_GET_NCOLS;
8140                                         break;
8141                                 default:
8142                                         (void) fprintf(stderr,
8143                                             gettext("invalid column name "
8144                                             "'%s'\n"), value);
8145                                         usage(B_FALSE);
8146                                 }
8147                         }
8148                         break;
8149                 case '?':
8150                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
8151                             optopt);
8152                         usage(B_FALSE);
8153                 }
8154         }
8155
8156         argc -= optind;
8157         argv += optind;
8158
8159         if (argc < 1) {
8160                 (void) fprintf(stderr, gettext("missing property "
8161                     "argument\n"));
8162                 usage(B_FALSE);
8163         }
8164
8165         if (zprop_get_list(g_zfs, argv[0], &cb.cb_proplist,
8166             ZFS_TYPE_POOL) != 0)
8167                 usage(B_FALSE);
8168
8169         argc--;
8170         argv++;
8171
8172         if (cb.cb_proplist != NULL) {
8173                 fake_name.pl_prop = ZPOOL_PROP_NAME;
8174                 fake_name.pl_width = strlen(gettext("NAME"));
8175                 fake_name.pl_next = cb.cb_proplist;
8176                 cb.cb_proplist = &fake_name;
8177         }
8178
8179         ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist,
8180             get_callback, &cb);
8181
8182         if (cb.cb_proplist == &fake_name)
8183                 zprop_free_list(fake_name.pl_next);
8184         else
8185                 zprop_free_list(cb.cb_proplist);
8186
8187         return (ret);
8188 }
8189
8190 typedef struct set_cbdata {
8191         char *cb_propname;
8192         char *cb_value;
8193         boolean_t cb_any_successful;
8194 } set_cbdata_t;
8195
8196 int
8197 set_callback(zpool_handle_t *zhp, void *data)
8198 {
8199         int error;
8200         set_cbdata_t *cb = (set_cbdata_t *)data;
8201
8202         error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
8203
8204         if (!error)
8205                 cb->cb_any_successful = B_TRUE;
8206
8207         return (error);
8208 }
8209
8210 int
8211 zpool_do_set(int argc, char **argv)
8212 {
8213         set_cbdata_t cb = { 0 };
8214         int error;
8215
8216         if (argc > 1 && argv[1][0] == '-') {
8217                 (void) fprintf(stderr, gettext("invalid option '%c'\n"),
8218                     argv[1][1]);
8219                 usage(B_FALSE);
8220         }
8221
8222         if (argc < 2) {
8223                 (void) fprintf(stderr, gettext("missing property=value "
8224                     "argument\n"));
8225                 usage(B_FALSE);
8226         }
8227
8228         if (argc < 3) {
8229                 (void) fprintf(stderr, gettext("missing pool name\n"));
8230                 usage(B_FALSE);
8231         }
8232
8233         if (argc > 3) {
8234                 (void) fprintf(stderr, gettext("too many pool names\n"));
8235                 usage(B_FALSE);
8236         }
8237
8238         cb.cb_propname = argv[1];
8239         cb.cb_value = strchr(cb.cb_propname, '=');
8240         if (cb.cb_value == NULL) {
8241                 (void) fprintf(stderr, gettext("missing value in "
8242                     "property=value argument\n"));
8243                 usage(B_FALSE);
8244         }
8245
8246         *(cb.cb_value) = '\0';
8247         cb.cb_value++;
8248
8249         error = for_each_pool(argc - 2, argv + 2, B_TRUE, NULL,
8250             set_callback, &cb);
8251
8252         return (error);
8253 }
8254
8255 static int
8256 find_command_idx(char *command, int *idx)
8257 {
8258         int i;
8259
8260         for (i = 0; i < NCOMMAND; i++) {
8261                 if (command_table[i].name == NULL)
8262                         continue;
8263
8264                 if (strcmp(command, command_table[i].name) == 0) {
8265                         *idx = i;
8266                         return (0);
8267                 }
8268         }
8269         return (1);
8270 }
8271
8272 int
8273 main(int argc, char **argv)
8274 {
8275         int ret = 0;
8276         int i = 0;
8277         char *cmdname;
8278
8279         (void) setlocale(LC_ALL, "");
8280         (void) textdomain(TEXT_DOMAIN);
8281         srand(time(NULL));
8282
8283         opterr = 0;
8284
8285         /*
8286          * Make sure the user has specified some command.
8287          */
8288         if (argc < 2) {
8289                 (void) fprintf(stderr, gettext("missing command\n"));
8290                 usage(B_FALSE);
8291         }
8292
8293         cmdname = argv[1];
8294
8295         /*
8296          * Special case '-?'
8297          */
8298         if ((strcmp(cmdname, "-?") == 0) || strcmp(cmdname, "--help") == 0)
8299                 usage(B_TRUE);
8300
8301         if ((g_zfs = libzfs_init()) == NULL) {
8302                 (void) fprintf(stderr, "%s", libzfs_error_init(errno));
8303                 return (1);
8304         }
8305
8306         libzfs_print_on_error(g_zfs, B_TRUE);
8307
8308         zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
8309
8310         /*
8311          * Run the appropriate command.
8312          */
8313         if (find_command_idx(cmdname, &i) == 0) {
8314                 current_command = &command_table[i];
8315                 ret = command_table[i].func(argc - 1, argv + 1);
8316         } else if (strchr(cmdname, '=')) {
8317                 verify(find_command_idx("set", &i) == 0);
8318                 current_command = &command_table[i];
8319                 ret = command_table[i].func(argc, argv);
8320         } else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
8321                 /*
8322                  * 'freeze' is a vile debugging abomination, so we treat
8323                  * it as such.
8324                  */
8325                 zfs_cmd_t zc = {"\0"};
8326
8327                 (void) strlcpy(zc.zc_name, argv[2], sizeof (zc.zc_name));
8328                 ret = zfs_ioctl(g_zfs, ZFS_IOC_POOL_FREEZE, &zc);
8329                 if (ret != 0) {
8330                         (void) fprintf(stderr,
8331                         gettext("failed to freeze pool: %d\n"), errno);
8332                         ret = 1;
8333                 }
8334
8335                 log_history = 0;
8336         } else {
8337                 (void) fprintf(stderr, gettext("unrecognized "
8338                     "command '%s'\n"), cmdname);
8339                 usage(B_FALSE);
8340                 ret = 1;
8341         }
8342
8343         if (ret == 0 && log_history)
8344                 (void) zpool_log_history(g_zfs, history_str);
8345
8346         libzfs_fini(g_zfs);
8347
8348         /*
8349          * The 'ZFS_ABORT' environment variable causes us to dump core on exit
8350          * for the purposes of running ::findleaks.
8351          */
8352         if (getenv("ZFS_ABORT") != NULL) {
8353                 (void) printf("dumping core by request\n");
8354                 abort();
8355         }
8356
8357         return (ret);
8358 }