granicus.if.org Git - zfs/blob - cmd/zpool/zpool_main.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
  26  * Copyright (c) 2012 by Frederik Wessels. All rights reserved.
  27  * Copyright (c) 2012 by Cyril Plisko. All rights reserved.
  28  * Copyright (c) 2013 by Prasad Joshi (sTec). All rights reserved.
  29  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
  30  * Copyright (c) 2017 Datto Inc.
  31  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  32  */
  33
  34 #include <assert.h>
  35 #include <ctype.h>
  36 #include <dirent.h>
  37 #include <errno.h>
  38 #include <fcntl.h>
  39 #include <libgen.h>
  40 #include <libintl.h>
  41 #include <libuutil.h>
  42 #include <locale.h>
  43 #include <stdio.h>
  44 #include <stdlib.h>
  45 #include <string.h>
  46 #include <strings.h>
  47 #include <unistd.h>
  48 #include <pwd.h>
  49 #include <zone.h>
  50 #include <sys/wait.h>
  51 #include <zfs_prop.h>
  52 #include <sys/fs/zfs.h>
  53 #include <sys/stat.h>
  54 #include <sys/systeminfo.h>
  55 #include <sys/fm/fs/zfs.h>
  56 #include <sys/fm/util.h>
  57 #include <sys/fm/protocol.h>
  58 #include <sys/zfs_ioctl.h>
  59 #include <sys/mount.h>
  60 #include <sys/sysmacros.h>
  61
  62 #include <math.h>
  63
  64 #include <libzfs.h>
  65
  66 #include "zpool_util.h"
  67 #include "zfs_comutil.h"
  68 #include "zfeature_common.h"
  69
  70 #include "statcommon.h"
  71
  72 static int zpool_do_create(int, char **);
  73 static int zpool_do_destroy(int, char **);
  74
  75 static int zpool_do_add(int, char **);
  76 static int zpool_do_remove(int, char **);
  77 static int zpool_do_labelclear(int, char **);
  78
  79 static int zpool_do_list(int, char **);
  80 static int zpool_do_iostat(int, char **);
  81 static int zpool_do_status(int, char **);
  82
  83 static int zpool_do_online(int, char **);
  84 static int zpool_do_offline(int, char **);
  85 static int zpool_do_clear(int, char **);
  86 static int zpool_do_reopen(int, char **);
  87
  88 static int zpool_do_reguid(int, char **);
  89
  90 static int zpool_do_attach(int, char **);
  91 static int zpool_do_detach(int, char **);
  92 static int zpool_do_replace(int, char **);
  93 static int zpool_do_split(int, char **);
  94
  95 static int zpool_do_scrub(int, char **);
  96
  97 static int zpool_do_import(int, char **);
  98 static int zpool_do_export(int, char **);
  99
 100 static int zpool_do_upgrade(int, char **);
 101
 102 static int zpool_do_history(int, char **);
 103 static int zpool_do_events(int, char **);
 104
 105 static int zpool_do_get(int, char **);
 106 static int zpool_do_set(int, char **);
 107
 108 static int zpool_do_sync(int, char **);
 109
 110 /*
 111  * These libumem hooks provide a reasonable set of defaults for the allocator's
 112  * debugging facilities.
 113  */
 114
 115 #ifdef DEBUG
 116 const char *
 117 _umem_debug_init(void)
 118 {
 119         return ("default,verbose"); /* $UMEM_DEBUG setting */
 120 }
 121
 122 const char *
 123 _umem_logging_init(void)
 124 {
 125         return ("fail,contents"); /* $UMEM_LOGGING setting */
 126 }
 127 #endif
 128
 129 typedef enum {
 130         HELP_ADD,
 131         HELP_ATTACH,
 132         HELP_CLEAR,
 133         HELP_CREATE,
 134         HELP_DESTROY,
 135         HELP_DETACH,
 136         HELP_EXPORT,
 137         HELP_HISTORY,
 138         HELP_IMPORT,
 139         HELP_IOSTAT,
 140         HELP_LABELCLEAR,
 141         HELP_LIST,
 142         HELP_OFFLINE,
 143         HELP_ONLINE,
 144         HELP_REPLACE,
 145         HELP_REMOVE,
 146         HELP_SCRUB,
 147         HELP_STATUS,
 148         HELP_UPGRADE,
 149         HELP_EVENTS,
 150         HELP_GET,
 151         HELP_SET,
 152         HELP_SPLIT,
 153         HELP_SYNC,
 154         HELP_REGUID,
 155         HELP_REOPEN
 156 } zpool_help_t;
 157
 158
 159 /*
 160  * Flags for stats to display with "zpool iostats"
 161  */
 162 enum iostat_type {
 163         IOS_DEFAULT = 0,
 164         IOS_LATENCY = 1,
 165         IOS_QUEUES = 2,
 166         IOS_L_HISTO = 3,
 167         IOS_RQ_HISTO = 4,
 168         IOS_COUNT,      /* always last element */
 169 };
 170
 171 /* iostat_type entries as bitmasks */
 172 #define IOS_DEFAULT_M   (1ULL << IOS_DEFAULT)
 173 #define IOS_LATENCY_M   (1ULL << IOS_LATENCY)
 174 #define IOS_QUEUES_M    (1ULL << IOS_QUEUES)
 175 #define IOS_L_HISTO_M   (1ULL << IOS_L_HISTO)
 176 #define IOS_RQ_HISTO_M  (1ULL << IOS_RQ_HISTO)
 177
 178 /* Mask of all the histo bits */
 179 #define IOS_ANYHISTO_M (IOS_L_HISTO_M | IOS_RQ_HISTO_M)
 180
 181 /*
 182  * Lookup table for iostat flags to nvlist names.  Basically a list
 183  * of all the nvlists a flag requires.  Also specifies the order in
 184  * which data gets printed in zpool iostat.
 185  */
 186 static const char *vsx_type_to_nvlist[IOS_COUNT][11] = {
 187         [IOS_L_HISTO] = {
 188             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 189             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 190             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 191             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 192             ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
 193             ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
 194             ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
 195             ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
 196             ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
 197             NULL},
 198         [IOS_LATENCY] = {
 199             ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
 200             ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
 201             ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
 202             ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
 203             NULL},
 204         [IOS_QUEUES] = {
 205             ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
 206             ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
 207             ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
 208             ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
 209             ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
 210             NULL},
 211         [IOS_RQ_HISTO] = {
 212             ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO,
 213             ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO,
 214             ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO,
 215             ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO,
 216             ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO,
 217             ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO,
 218             ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO,
 219             ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO,
 220             ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO,
 221             ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO,
 222             NULL},
 223 };
 224
 225
 226 /*
 227  * Given a cb->cb_flags with a histogram bit set, return the iostat_type.
 228  * Right now, only one histo bit is ever set at one time, so we can
 229  * just do a highbit64(a)
 230  */
 231 #define IOS_HISTO_IDX(a)        (highbit64(a & IOS_ANYHISTO_M) - 1)
 232
 233 typedef struct zpool_command {
 234         const char      *name;
 235         int             (*func)(int, char **);
 236         zpool_help_t    usage;
 237 } zpool_command_t;
 238
 239 /*
 240  * Master command table.  Each ZFS command has a name, associated function, and
 241  * usage message.  The usage messages need to be internationalized, so we have
 242  * to have a function to return the usage message based on a command index.
 243  *
 244  * These commands are organized according to how they are displayed in the usage
 245  * message.  An empty command (one with a NULL name) indicates an empty line in
 246  * the generic usage message.
 247  */
 248 static zpool_command_t command_table[] = {
 249         { "create",     zpool_do_create,        HELP_CREATE             },
 250         { "destroy",    zpool_do_destroy,       HELP_DESTROY            },
 251         { NULL },
 252         { "add",        zpool_do_add,           HELP_ADD                },
 253         { "remove",     zpool_do_remove,        HELP_REMOVE             },
 254         { NULL },
 255         { "labelclear", zpool_do_labelclear,    HELP_LABELCLEAR         },
 256         { NULL },
 257         { "list",       zpool_do_list,          HELP_LIST               },
 258         { "iostat",     zpool_do_iostat,        HELP_IOSTAT             },
 259         { "status",     zpool_do_status,        HELP_STATUS             },
 260         { NULL },
 261         { "online",     zpool_do_online,        HELP_ONLINE             },
 262         { "offline",    zpool_do_offline,       HELP_OFFLINE            },
 263         { "clear",      zpool_do_clear,         HELP_CLEAR              },
 264         { "reopen",     zpool_do_reopen,        HELP_REOPEN             },
 265         { NULL },
 266         { "attach",     zpool_do_attach,        HELP_ATTACH             },
 267         { "detach",     zpool_do_detach,        HELP_DETACH             },
 268         { "replace",    zpool_do_replace,       HELP_REPLACE            },
 269         { "split",      zpool_do_split,         HELP_SPLIT              },
 270         { NULL },
 271         { "scrub",      zpool_do_scrub,         HELP_SCRUB              },
 272         { NULL },
 273         { "import",     zpool_do_import,        HELP_IMPORT             },
 274         { "export",     zpool_do_export,        HELP_EXPORT             },
 275         { "upgrade",    zpool_do_upgrade,       HELP_UPGRADE            },
 276         { "reguid",     zpool_do_reguid,        HELP_REGUID             },
 277         { NULL },
 278         { "history",    zpool_do_history,       HELP_HISTORY            },
 279         { "events",     zpool_do_events,        HELP_EVENTS             },
 280         { NULL },
 281         { "get",        zpool_do_get,           HELP_GET                },
 282         { "set",        zpool_do_set,           HELP_SET                },
 283         { "sync",       zpool_do_sync,          HELP_SYNC               },
 284 };
 285
 286 #define NCOMMAND        (ARRAY_SIZE(command_table))
 287
 288 static zpool_command_t *current_command;
 289 static char history_str[HIS_MAX_RECORD_LEN];
 290 static boolean_t log_history = B_TRUE;
 291 static uint_t timestamp_fmt = NODATE;
 292
 293 static const char *
 294 get_usage(zpool_help_t idx)
 295 {
 296         switch (idx) {
 297         case HELP_ADD:
 298                 return (gettext("\tadd [-fgLnP] [-o property=value] "
 299                     "<pool> <vdev> ...\n"));
 300         case HELP_ATTACH:
 301                 return (gettext("\tattach [-f] [-o property=value] "
 302                     "<pool> <device> <new-device>\n"));
 303         case HELP_CLEAR:
 304                 return (gettext("\tclear [-nF] <pool> [device]\n"));
 305         case HELP_CREATE:
 306                 return (gettext("\tcreate [-fnd] [-o property=value] ... \n"
 307                     "\t    [-O file-system-property=value] ... \n"
 308                     "\t    [-m mountpoint] [-R root] <pool> <vdev> ...\n"));
 309         case HELP_DESTROY:
 310                 return (gettext("\tdestroy [-f] <pool>\n"));
 311         case HELP_DETACH:
 312                 return (gettext("\tdetach <pool> <device>\n"));
 313         case HELP_EXPORT:
 314                 return (gettext("\texport [-af] <pool> ...\n"));
 315         case HELP_HISTORY:
 316                 return (gettext("\thistory [-il] [<pool>] ...\n"));
 317         case HELP_IMPORT:
 318                 return (gettext("\timport [-d dir] [-D]\n"
 319                     "\timport [-d dir | -c cachefile] [-F [-n]] [-l] "
 320                     "<pool | id>\n"
 321                     "\timport [-o mntopts] [-o property=value] ... \n"
 322                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 323                     "[-R root] [-F [-n]] -a\n"
 324                     "\timport [-o mntopts] [-o property=value] ... \n"
 325                     "\t    [-d dir | -c cachefile] [-D] [-l] [-f] [-m] [-N] "
 326                     "[-R root] [-F [-n]]\n"
 327                     "\t    <pool | id> [newpool]\n"));
 328         case HELP_IOSTAT:
 329                 return (gettext("\tiostat [[[-c [script1,script2,...]"
 330                     "[-lq]]|[-rw]] [-T d | u] [-ghHLpPvy]\n"
 331                     "\t    [[pool ...]|[pool vdev ...]|[vdev ...]]"
 332                     " [interval [count]]\n"));
 333         case HELP_LABELCLEAR:
 334                 return (gettext("\tlabelclear [-f] <vdev>\n"));
 335         case HELP_LIST:
 336                 return (gettext("\tlist [-gHLpPv] [-o property[,...]] "
 337                     "[-T d|u] [pool] ... \n"
 338                     "\t    [interval [count]]\n"));
 339         case HELP_OFFLINE:
 340                 return (gettext("\toffline [-f] [-t] <pool> <device> ...\n"));
 341         case HELP_ONLINE:
 342                 return (gettext("\tonline <pool> <device> ...\n"));
 343         case HELP_REPLACE:
 344                 return (gettext("\treplace [-f] [-o property=value] "
 345                     "<pool> <device> [new-device]\n"));
 346         case HELP_REMOVE:
 347                 return (gettext("\tremove [-nps] <pool> <device> ...\n"));
 348         case HELP_REOPEN:
 349                 return (gettext("\treopen [-n] <pool>\n"));
 350         case HELP_SCRUB:
 351                 return (gettext("\tscrub [-s | -p] <pool> ...\n"));
 352         case HELP_STATUS:
 353                 return (gettext("\tstatus [-c [script1,script2,...]] [-gLPvxD]"
 354                     "[-T d|u] [pool] ... \n"
 355                     "\t    [interval [count]]\n"));
 356         case HELP_UPGRADE:
 357                 return (gettext("\tupgrade\n"
 358                     "\tupgrade -v\n"
 359                     "\tupgrade [-V version] <-a | pool ...>\n"));
 360         case HELP_EVENTS:
 361                 return (gettext("\tevents [-vHf [pool] | -c]\n"));
 362         case HELP_GET:
 363                 return (gettext("\tget [-Hp] [-o \"all\" | field[,...]] "
 364                     "<\"all\" | property[,...]> <pool> ...\n"));
 365         case HELP_SET:
 366                 return (gettext("\tset <property=value> <pool> \n"));
 367         case HELP_SPLIT:
 368                 return (gettext("\tsplit [-gLnPl] [-R altroot] [-o mntopts]\n"
 369                     "\t    [-o property=value] <pool> <newpool> "
 370                     "[<device> ...]\n"));
 371         case HELP_REGUID:
 372                 return (gettext("\treguid <pool>\n"));
 373         case HELP_SYNC:
 374                 return (gettext("\tsync [pool] ...\n"));
 375         }
 376
 377         abort();
 378         /* NOTREACHED */
 379 }
 380
 381
 382 /*
 383  * Callback routine that will print out a pool property value.
 384  */
 385 static int
 386 print_prop_cb(int prop, void *cb)
 387 {
 388         FILE *fp = cb;
 389
 390         (void) fprintf(fp, "\t%-15s  ", zpool_prop_to_name(prop));
 391
 392         if (zpool_prop_readonly(prop))
 393                 (void) fprintf(fp, "  NO   ");
 394         else
 395                 (void) fprintf(fp, " YES   ");
 396
 397         if (zpool_prop_values(prop) == NULL)
 398                 (void) fprintf(fp, "-\n");
 399         else
 400                 (void) fprintf(fp, "%s\n", zpool_prop_values(prop));
 401
 402         return (ZPROP_CONT);
 403 }
 404
 405 /*
 406  * Display usage message.  If we're inside a command, display only the usage for
 407  * that command.  Otherwise, iterate over the entire command table and display
 408  * a complete usage message.
 409  */
 410 void
 411 usage(boolean_t requested)
 412 {
 413         FILE *fp = requested ? stdout : stderr;
 414
 415         if (current_command == NULL) {
 416                 int i;
 417
 418                 (void) fprintf(fp, gettext("usage: zpool command args ...\n"));
 419                 (void) fprintf(fp,
 420                     gettext("where 'command' is one of the following:\n\n"));
 421
 422                 for (i = 0; i < NCOMMAND; i++) {
 423                         if (command_table[i].name == NULL)
 424                                 (void) fprintf(fp, "\n");
 425                         else
 426                                 (void) fprintf(fp, "%s",
 427                                     get_usage(command_table[i].usage));
 428                 }
 429         } else {
 430                 (void) fprintf(fp, gettext("usage:\n"));
 431                 (void) fprintf(fp, "%s", get_usage(current_command->usage));
 432         }
 433
 434         if (current_command != NULL &&
 435             ((strcmp(current_command->name, "set") == 0) ||
 436             (strcmp(current_command->name, "get") == 0) ||
 437             (strcmp(current_command->name, "list") == 0))) {
 438
 439                 (void) fprintf(fp,
 440                     gettext("\nthe following properties are supported:\n"));
 441
 442                 (void) fprintf(fp, "\n\t%-15s  %s   %s\n\n",
 443                     "PROPERTY", "EDIT", "VALUES");
 444
 445                 /* Iterate over all properties */
 446                 (void) zprop_iter(print_prop_cb, fp, B_FALSE, B_TRUE,
 447                     ZFS_TYPE_POOL);
 448
 449                 (void) fprintf(fp, "\t%-15s   ", "feature@...");
 450                 (void) fprintf(fp, "YES   disabled | enabled | active\n");
 451
 452                 (void) fprintf(fp, gettext("\nThe feature@ properties must be "
 453                     "appended with a feature name.\nSee zpool-features(5).\n"));
 454         }
 455
 456         /*
 457          * See comments at end of main().
 458          */
 459         if (getenv("ZFS_ABORT") != NULL) {
 460                 (void) printf("dumping core by request\n");
 461                 abort();
 462         }
 463
 464         exit(requested ? 0 : 2);
 465 }
 466
 467 void
 468 print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
 469     boolean_t print_logs, int name_flags)
 470 {
 471         nvlist_t **child;
 472         uint_t c, children;
 473         char *vname;
 474
 475         if (name != NULL)
 476                 (void) printf("\t%*s%s\n", indent, "", name);
 477
 478         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
 479             &child, &children) != 0)
 480                 return;
 481
 482         for (c = 0; c < children; c++) {
 483                 uint64_t is_log = B_FALSE;
 484
 485                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
 486                     &is_log);
 487                 if ((is_log && !print_logs) || (!is_log && print_logs))
 488                         continue;
 489
 490                 vname = zpool_vdev_name(g_zfs, zhp, child[c], name_flags);
 491                 print_vdev_tree(zhp, vname, child[c], indent + 2,
 492                     B_FALSE, name_flags);
 493                 free(vname);
 494         }
 495 }
 496
 497 static boolean_t
 498 prop_list_contains_feature(nvlist_t *proplist)
 499 {
 500         nvpair_t *nvp;
 501         for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
 502             nvp = nvlist_next_nvpair(proplist, nvp)) {
 503                 if (zpool_prop_feature(nvpair_name(nvp)))
 504                         return (B_TRUE);
 505         }
 506         return (B_FALSE);
 507 }
 508
 509 /*
 510  * Add a property pair (name, string-value) into a property nvlist.
 511  */
 512 static int
 513 add_prop_list(const char *propname, char *propval, nvlist_t **props,
 514     boolean_t poolprop)
 515 {
 516         zpool_prop_t prop = ZPOOL_PROP_INVAL;
 517         zfs_prop_t fprop;
 518         nvlist_t *proplist;
 519         const char *normnm;
 520         char *strval;
 521
 522         if (*props == NULL &&
 523             nvlist_alloc(props, NV_UNIQUE_NAME, 0) != 0) {
 524                 (void) fprintf(stderr,
 525                     gettext("internal error: out of memory\n"));
 526                 return (1);
 527         }
 528
 529         proplist = *props;
 530
 531         if (poolprop) {
 532                 const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
 533
 534                 if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
 535                     !zpool_prop_feature(propname)) {
 536                         (void) fprintf(stderr, gettext("property '%s' is "
 537                             "not a valid pool property\n"), propname);
 538                         return (2);
 539                 }
 540
 541                 /*
 542                  * feature@ properties and version should not be specified
 543                  * at the same time.
 544                  */
 545                 if ((prop == ZPOOL_PROP_INVAL && zpool_prop_feature(propname) &&
 546                     nvlist_exists(proplist, vname)) ||
 547                     (prop == ZPOOL_PROP_VERSION &&
 548                     prop_list_contains_feature(proplist))) {
 549                         (void) fprintf(stderr, gettext("'feature@' and "
 550                             "'version' properties cannot be specified "
 551                             "together\n"));
 552                         return (2);
 553                 }
 554
 555
 556                 if (zpool_prop_feature(propname))
 557                         normnm = propname;
 558                 else
 559                         normnm = zpool_prop_to_name(prop);
 560         } else {
 561                 if ((fprop = zfs_name_to_prop(propname)) != ZPROP_INVAL) {
 562                         normnm = zfs_prop_to_name(fprop);
 563                 } else {
 564                         normnm = propname;
 565                 }
 566         }
 567
 568         if (nvlist_lookup_string(proplist, normnm, &strval) == 0 &&
 569             prop != ZPOOL_PROP_CACHEFILE) {
 570                 (void) fprintf(stderr, gettext("property '%s' "
 571                     "specified multiple times\n"), propname);
 572                 return (2);
 573         }
 574
 575         if (nvlist_add_string(proplist, normnm, propval) != 0) {
 576                 (void) fprintf(stderr, gettext("internal "
 577                     "error: out of memory\n"));
 578                 return (1);
 579         }
 580
 581         return (0);
 582 }
 583
 584 /*
 585  * Set a default property pair (name, string-value) in a property nvlist
 586  */
 587 static int
 588 add_prop_list_default(const char *propname, char *propval, nvlist_t **props,
 589     boolean_t poolprop)
 590 {
 591         char *pval;
 592
 593         if (nvlist_lookup_string(*props, propname, &pval) == 0)
 594                 return (0);
 595
 596         return (add_prop_list(propname, propval, props, B_TRUE));
 597 }
 598
 599 /*
 600  * zpool add [-fgLnP] [-o property=value] <pool> <vdev> ...
 601  *
 602  *      -f      Force addition of devices, even if they appear in use
 603  *      -g      Display guid for individual vdev name.
 604  *      -L      Follow links when resolving vdev path name.
 605  *      -n      Do not add the devices, but display the resulting layout if
 606  *              they were to be added.
 607  *      -o      Set property=value.
 608  *      -P      Display full path for vdev name.
 609  *
 610  * Adds the given vdevs to 'pool'.  As with create, the bulk of this work is
 611  * handled by get_vdev_spec(), which constructs the nvlist needed to pass to
 612  * libzfs.
 613  */
 614 int
 615 zpool_do_add(int argc, char **argv)
 616 {
 617         boolean_t force = B_FALSE;
 618         boolean_t dryrun = B_FALSE;
 619         int name_flags = 0;
 620         int c;
 621         nvlist_t *nvroot;
 622         char *poolname;
 623         int ret;
 624         zpool_handle_t *zhp;
 625         nvlist_t *config;
 626         nvlist_t *props = NULL;
 627         char *propval;
 628
 629         /* check options */
 630         while ((c = getopt(argc, argv, "fgLno:P")) != -1) {
 631                 switch (c) {
 632                 case 'f':
 633                         force = B_TRUE;
 634                         break;
 635                 case 'g':
 636                         name_flags |= VDEV_NAME_GUID;
 637                         break;
 638                 case 'L':
 639                         name_flags |= VDEV_NAME_FOLLOW_LINKS;
 640                         break;
 641                 case 'n':
 642                         dryrun = B_TRUE;
 643                         break;
 644                 case 'o':
 645                         if ((propval = strchr(optarg, '=')) == NULL) {
 646                                 (void) fprintf(stderr, gettext("missing "
 647                                     "'=' for -o option\n"));
 648                                 usage(B_FALSE);
 649                         }
 650                         *propval = '\0';
 651                         propval++;
 652
 653                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
 654                             (add_prop_list(optarg, propval, &props, B_TRUE)))
 655                                 usage(B_FALSE);
 656                         break;
 657                 case 'P':
 658                         name_flags |= VDEV_NAME_PATH;
 659                         break;
 660                 case '?':
 661                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
 662                             optopt);
 663                         usage(B_FALSE);
 664                 }
 665         }
 666
 667         argc -= optind;
 668         argv += optind;
 669
 670         /* get pool name and check number of arguments */
 671         if (argc < 1) {
 672                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 673                 usage(B_FALSE);
 674         }
 675         if (argc < 2) {
 676                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
 677                 usage(B_FALSE);
 678         }
 679
 680         poolname = argv[0];
 681
 682         argc--;
 683         argv++;
 684
 685         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
 686                 return (1);
 687
 688         if ((config = zpool_get_config(zhp, NULL)) == NULL) {
 689                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
 690                     poolname);
 691                 zpool_close(zhp);
 692                 return (1);
 693         }
 694
 695         /* unless manually specified use "ashift" pool property (if set) */
 696         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
 697                 int intval;
 698                 zprop_source_t src;
 699                 char strval[ZPOOL_MAXPROPLEN];
 700
 701                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
 702                 if (src != ZPROP_SRC_DEFAULT) {
 703                         (void) sprintf(strval, "%" PRId32, intval);
 704                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
 705                             &props, B_TRUE) == 0);
 706                 }
 707         }
 708
 709         /* pass off to get_vdev_spec for processing */
 710         nvroot = make_root_vdev(zhp, props, force, !force, B_FALSE, dryrun,
 711             argc, argv);
 712         if (nvroot == NULL) {
 713                 zpool_close(zhp);
 714                 return (1);
 715         }
 716
 717         if (dryrun) {
 718                 nvlist_t *poolnvroot;
 719                 nvlist_t **l2child;
 720                 uint_t l2children, c;
 721                 char *vname;
 722                 boolean_t hadcache = B_FALSE;
 723
 724                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
 725                     &poolnvroot) == 0);
 726
 727                 (void) printf(gettext("would update '%s' to the following "
 728                     "configuration:\n"), zpool_get_name(zhp));
 729
 730                 /* print original main pool and new tree */
 731                 print_vdev_tree(zhp, poolname, poolnvroot, 0, B_FALSE,
 732                     name_flags);
 733                 print_vdev_tree(zhp, NULL, nvroot, 0, B_FALSE, name_flags);
 734
 735                 /* Do the same for the logs */
 736                 if (num_logs(poolnvroot) > 0) {
 737                         print_vdev_tree(zhp, "logs", poolnvroot, 0, B_TRUE,
 738                             name_flags);
 739                         print_vdev_tree(zhp, NULL, nvroot, 0, B_TRUE,
 740                             name_flags);
 741                 } else if (num_logs(nvroot) > 0) {
 742                         print_vdev_tree(zhp, "logs", nvroot, 0, B_TRUE,
 743                             name_flags);
 744                 }
 745
 746                 /* Do the same for the caches */
 747                 if (nvlist_lookup_nvlist_array(poolnvroot, ZPOOL_CONFIG_L2CACHE,
 748                     &l2child, &l2children) == 0 && l2children) {
 749                         hadcache = B_TRUE;
 750                         (void) printf(gettext("\tcache\n"));
 751                         for (c = 0; c < l2children; c++) {
 752                                 vname = zpool_vdev_name(g_zfs, NULL,
 753                                     l2child[c], name_flags);
 754                                 (void) printf("\t  %s\n", vname);
 755                                 free(vname);
 756                         }
 757                 }
 758                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
 759                     &l2child, &l2children) == 0 && l2children) {
 760                         if (!hadcache)
 761                                 (void) printf(gettext("\tcache\n"));
 762                         for (c = 0; c < l2children; c++) {
 763                                 vname = zpool_vdev_name(g_zfs, NULL,
 764                                     l2child[c], name_flags);
 765                                 (void) printf("\t  %s\n", vname);
 766                                 free(vname);
 767                         }
 768                 }
 769
 770                 ret = 0;
 771         } else {
 772                 ret = (zpool_add(zhp, nvroot) != 0);
 773         }
 774
 775         nvlist_free(props);
 776         nvlist_free(nvroot);
 777         zpool_close(zhp);
 778
 779         return (ret);
 780 }
 781
 782 /*
 783  * zpool remove  <pool> <vdev> ...
 784  *
 785  * Removes the given vdev from the pool.
 786  */
 787 int
 788 zpool_do_remove(int argc, char **argv)
 789 {
 790         char *poolname;
 791         int i, ret = 0;
 792         zpool_handle_t *zhp = NULL;
 793         boolean_t stop = B_FALSE;
 794         char c;
 795         boolean_t noop = B_FALSE;
 796         boolean_t parsable = B_FALSE;
 797
 798         /* check options */
 799         while ((c = getopt(argc, argv, "nps")) != -1) {
 800                 switch (c) {
 801                 case 'n':
 802                         noop = B_TRUE;
 803                         break;
 804                 case 'p':
 805                         parsable = B_TRUE;
 806                         break;
 807                 case 's':
 808                         stop = B_TRUE;
 809                         break;
 810                 case '?':
 811                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
 812                             optopt);
 813                         usage(B_FALSE);
 814                 }
 815         }
 816
 817         argc -= optind;
 818         argv += optind;
 819
 820         /* get pool name and check number of arguments */
 821         if (argc < 1) {
 822                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
 823                 usage(B_FALSE);
 824         }
 825
 826         poolname = argv[0];
 827
 828         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
 829                 return (1);
 830
 831         if (stop && noop) {
 832                 (void) fprintf(stderr, gettext("stop request ignored\n"));
 833                 return (0);
 834         }
 835
 836         if (stop) {
 837                 if (argc > 1) {
 838                         (void) fprintf(stderr, gettext("too many arguments\n"));
 839                         usage(B_FALSE);
 840                 }
 841                 if (zpool_vdev_remove_cancel(zhp) != 0)
 842                         ret = 1;
 843         } else {
 844                 if (argc < 2) {
 845                         (void) fprintf(stderr, gettext("missing device\n"));
 846                         usage(B_FALSE);
 847                 }
 848
 849                 for (i = 1; i < argc; i++) {
 850                         if (noop) {
 851                                 uint64_t size;
 852
 853                                 if (zpool_vdev_indirect_size(zhp, argv[i],
 854                                     &size) != 0) {
 855                                         ret = 1;
 856                                         break;
 857                                 }
 858                                 if (parsable) {
 859                                         (void) printf("%s %llu\n",
 860                                             argv[i], (unsigned long long)size);
 861                                 } else {
 862                                         char valstr[32];
 863                                         zfs_nicenum(size, valstr,
 864                                             sizeof (valstr));
 865                                         (void) printf("Memory that will be "
 866                                             "used after removing %s: %s\n",
 867                                             argv[i], valstr);
 868                                 }
 869                         } else {
 870                                 if (zpool_vdev_remove(zhp, argv[i]) != 0)
 871                                         ret = 1;
 872                         }
 873                 }
 874         }
 875         zpool_close(zhp);
 876
 877         return (ret);
 878 }
 879
 880 /*
 881  * zpool labelclear [-f] <vdev>
 882  *
 883  *      -f      Force clearing the label for the vdevs which are members of
 884  *              the exported or foreign pools.
 885  *
 886  * Verifies that the vdev is not active and zeros out the label information
 887  * on the device.
 888  */
 889 int
 890 zpool_do_labelclear(int argc, char **argv)
 891 {
 892         char vdev[MAXPATHLEN];
 893         char *name = NULL;
 894         struct stat st;
 895         int c, fd = -1, ret = 0;
 896         nvlist_t *config;
 897         pool_state_t state;
 898         boolean_t inuse = B_FALSE;
 899         boolean_t force = B_FALSE;
 900
 901         /* check options */
 902         while ((c = getopt(argc, argv, "f")) != -1) {
 903                 switch (c) {
 904                 case 'f':
 905                         force = B_TRUE;
 906                         break;
 907                 default:
 908                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
 909                             optopt);
 910                         usage(B_FALSE);
 911                 }
 912         }
 913
 914         argc -= optind;
 915         argv += optind;
 916
 917         /* get vdev name */
 918         if (argc < 1) {
 919                 (void) fprintf(stderr, gettext("missing vdev name\n"));
 920                 usage(B_FALSE);
 921         }
 922         if (argc > 1) {
 923                 (void) fprintf(stderr, gettext("too many arguments\n"));
 924                 usage(B_FALSE);
 925         }
 926
 927         /*
 928          * Check if we were given absolute path and use it as is.
 929          * Otherwise if the provided vdev name doesn't point to a file,
 930          * try prepending expected disk paths and partition numbers.
 931          */
 932         (void) strlcpy(vdev, argv[0], sizeof (vdev));
 933         if (vdev[0] != '/' && stat(vdev, &st) != 0) {
 934                 int error;
 935
 936                 error = zfs_resolve_shortname(argv[0], vdev, MAXPATHLEN);
 937                 if (error == 0 && zfs_dev_is_whole_disk(vdev)) {
 938                         if (zfs_append_partition(vdev, MAXPATHLEN) == -1)
 939                                 error = ENOENT;
 940                 }
 941
 942                 if (error || (stat(vdev, &st) != 0)) {
 943                         (void) fprintf(stderr, gettext(
 944                             "failed to find device %s, try specifying absolute "
 945                             "path instead\n"), argv[0]);
 946                         return (1);
 947                 }
 948         }
 949
 950         if ((fd = open(vdev, O_RDWR)) < 0) {
 951                 (void) fprintf(stderr, gettext("failed to open %s: %s\n"),
 952                     vdev, strerror(errno));
 953                 return (1);
 954         }
 955
 956         if (ioctl(fd, BLKFLSBUF) != 0)
 957                 (void) fprintf(stderr, gettext("failed to invalidate "
 958                     "cache for %s: %s\n"), vdev, strerror(errno));
 959
 960         if (zpool_read_label(fd, &config, NULL) != 0 || config == NULL) {
 961                 (void) fprintf(stderr,
 962                     gettext("failed to check state for %s\n"), vdev);
 963                 ret = 1;
 964                 goto errout;
 965         }
 966         nvlist_free(config);
 967
 968         ret = zpool_in_use(g_zfs, fd, &state, &name, &inuse);
 969         if (ret != 0) {
 970                 (void) fprintf(stderr,
 971                     gettext("failed to check state for %s\n"), vdev);
 972                 ret = 1;
 973                 goto errout;
 974         }
 975
 976         if (!inuse)
 977                 goto wipe_label;
 978
 979         switch (state) {
 980         default:
 981         case POOL_STATE_ACTIVE:
 982         case POOL_STATE_SPARE:
 983         case POOL_STATE_L2CACHE:
 984                 (void) fprintf(stderr, gettext(
 985                     "%s is a member (%s) of pool \"%s\"\n"),
 986                     vdev, zpool_pool_state_to_name(state), name);
 987                 ret = 1;
 988                 goto errout;
 989
 990         case POOL_STATE_EXPORTED:
 991                 if (force)
 992                         break;
 993                 (void) fprintf(stderr, gettext(
 994                     "use '-f' to override the following error:\n"
 995                     "%s is a member of exported pool \"%s\"\n"),
 996                     vdev, name);
 997                 ret = 1;
 998                 goto errout;
 999
1000         case POOL_STATE_POTENTIALLY_ACTIVE:
1001                 if (force)
1002                         break;
1003                 (void) fprintf(stderr, gettext(
1004                     "use '-f' to override the following error:\n"
1005                     "%s is a member of potentially active pool \"%s\"\n"),
1006                     vdev, name);
1007                 ret = 1;
1008                 goto errout;
1009
1010         case POOL_STATE_DESTROYED:
1011                 /* inuse should never be set for a destroyed pool */
1012                 assert(0);
1013                 break;
1014         }
1015
1016 wipe_label:
1017         ret = zpool_clear_label(fd);
1018         if (ret != 0) {
1019                 (void) fprintf(stderr,
1020                     gettext("failed to clear label for %s\n"), vdev);
1021         }
1022
1023 errout:
1024         free(name);
1025         (void) close(fd);
1026
1027         return (ret);
1028 }
1029
1030 /*
1031  * zpool create [-fnd] [-o property=value] ...
1032  *              [-O file-system-property=value] ...
1033  *              [-R root] [-m mountpoint] <pool> <dev> ...
1034  *
1035  *      -f      Force creation, even if devices appear in use
1036  *      -n      Do not create the pool, but display the resulting layout if it
1037  *              were to be created.
1038  *      -R      Create a pool under an alternate root
1039  *      -m      Set default mountpoint for the root dataset.  By default it's
1040  *              '/<pool>'
1041  *      -o      Set property=value.
1042  *      -o      Set feature@feature=enabled|disabled.
1043  *      -d      Don't automatically enable all supported pool features
1044  *              (individual features can be enabled with -o).
1045  *      -O      Set fsproperty=value in the pool's root file system
1046  *
1047  * Creates the named pool according to the given vdev specification.  The
1048  * bulk of the vdev processing is done in get_vdev_spec() in zpool_vdev.c.  Once
1049  * we get the nvlist back from get_vdev_spec(), we either print out the contents
1050  * (if '-n' was specified), or pass it to libzfs to do the creation.
1051  */
1052 int
1053 zpool_do_create(int argc, char **argv)
1054 {
1055         boolean_t force = B_FALSE;
1056         boolean_t dryrun = B_FALSE;
1057         boolean_t enable_all_pool_feat = B_TRUE;
1058         int c;
1059         nvlist_t *nvroot = NULL;
1060         char *poolname;
1061         char *tname = NULL;
1062         int ret = 1;
1063         char *altroot = NULL;
1064         char *mountpoint = NULL;
1065         nvlist_t *fsprops = NULL;
1066         nvlist_t *props = NULL;
1067         char *propval;
1068
1069         /* check options */
1070         while ((c = getopt(argc, argv, ":fndR:m:o:O:t:")) != -1) {
1071                 switch (c) {
1072                 case 'f':
1073                         force = B_TRUE;
1074                         break;
1075                 case 'n':
1076                         dryrun = B_TRUE;
1077                         break;
1078                 case 'd':
1079                         enable_all_pool_feat = B_FALSE;
1080                         break;
1081                 case 'R':
1082                         altroot = optarg;
1083                         if (add_prop_list(zpool_prop_to_name(
1084                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
1085                                 goto errout;
1086                         if (add_prop_list_default(zpool_prop_to_name(
1087                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1088                                 goto errout;
1089                         break;
1090                 case 'm':
1091                         /* Equivalent to -O mountpoint=optarg */
1092                         mountpoint = optarg;
1093                         break;
1094                 case 'o':
1095                         if ((propval = strchr(optarg, '=')) == NULL) {
1096                                 (void) fprintf(stderr, gettext("missing "
1097                                     "'=' for -o option\n"));
1098                                 goto errout;
1099                         }
1100                         *propval = '\0';
1101                         propval++;
1102
1103                         if (add_prop_list(optarg, propval, &props, B_TRUE))
1104                                 goto errout;
1105
1106                         /*
1107                          * If the user is creating a pool that doesn't support
1108                          * feature flags, don't enable any features.
1109                          */
1110                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_VERSION) {
1111                                 char *end;
1112                                 u_longlong_t ver;
1113
1114                                 ver = strtoull(propval, &end, 10);
1115                                 if (*end == '\0' &&
1116                                     ver < SPA_VERSION_FEATURES) {
1117                                         enable_all_pool_feat = B_FALSE;
1118                                 }
1119                         }
1120                         if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
1121                                 altroot = propval;
1122                         break;
1123                 case 'O':
1124                         if ((propval = strchr(optarg, '=')) == NULL) {
1125                                 (void) fprintf(stderr, gettext("missing "
1126                                     "'=' for -O option\n"));
1127                                 goto errout;
1128                         }
1129                         *propval = '\0';
1130                         propval++;
1131
1132                         /*
1133                          * Mountpoints are checked and then added later.
1134                          * Uniquely among properties, they can be specified
1135                          * more than once, to avoid conflict with -m.
1136                          */
1137                         if (0 == strcmp(optarg,
1138                             zfs_prop_to_name(ZFS_PROP_MOUNTPOINT))) {
1139                                 mountpoint = propval;
1140                         } else if (add_prop_list(optarg, propval, &fsprops,
1141                             B_FALSE)) {
1142                                 goto errout;
1143                         }
1144                         break;
1145                 case 't':
1146                         /*
1147                          * Sanity check temporary pool name.
1148                          */
1149                         if (strchr(optarg, '/') != NULL) {
1150                                 (void) fprintf(stderr, gettext("cannot create "
1151                                     "'%s': invalid character '/' in temporary "
1152                                     "name\n"), optarg);
1153                                 (void) fprintf(stderr, gettext("use 'zfs "
1154                                     "create' to create a dataset\n"));
1155                                 goto errout;
1156                         }
1157
1158                         if (add_prop_list(zpool_prop_to_name(
1159                             ZPOOL_PROP_TNAME), optarg, &props, B_TRUE))
1160                                 goto errout;
1161                         if (add_prop_list_default(zpool_prop_to_name(
1162                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
1163                                 goto errout;
1164                         tname = optarg;
1165                         break;
1166                 case ':':
1167                         (void) fprintf(stderr, gettext("missing argument for "
1168                             "'%c' option\n"), optopt);
1169                         goto badusage;
1170                 case '?':
1171                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1172                             optopt);
1173                         goto badusage;
1174                 }
1175         }
1176
1177         argc -= optind;
1178         argv += optind;
1179
1180         /* get pool name and check number of arguments */
1181         if (argc < 1) {
1182                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
1183                 goto badusage;
1184         }
1185         if (argc < 2) {
1186                 (void) fprintf(stderr, gettext("missing vdev specification\n"));
1187                 goto badusage;
1188         }
1189
1190         poolname = argv[0];
1191
1192         /*
1193          * As a special case, check for use of '/' in the name, and direct the
1194          * user to use 'zfs create' instead.
1195          */
1196         if (strchr(poolname, '/') != NULL) {
1197                 (void) fprintf(stderr, gettext("cannot create '%s': invalid "
1198                     "character '/' in pool name\n"), poolname);
1199                 (void) fprintf(stderr, gettext("use 'zfs create' to "
1200                     "create a dataset\n"));
1201                 goto errout;
1202         }
1203
1204         /* pass off to get_vdev_spec for bulk processing */
1205         nvroot = make_root_vdev(NULL, props, force, !force, B_FALSE, dryrun,
1206             argc - 1, argv + 1);
1207         if (nvroot == NULL)
1208                 goto errout;
1209
1210         /* make_root_vdev() allows 0 toplevel children if there are spares */
1211         if (!zfs_allocatable_devs(nvroot)) {
1212                 (void) fprintf(stderr, gettext("invalid vdev "
1213                     "specification: at least one toplevel vdev must be "
1214                     "specified\n"));
1215                 goto errout;
1216         }
1217
1218         if (altroot != NULL && altroot[0] != '/') {
1219                 (void) fprintf(stderr, gettext("invalid alternate root '%s': "
1220                     "must be an absolute path\n"), altroot);
1221                 goto errout;
1222         }
1223
1224         /*
1225          * Check the validity of the mountpoint and direct the user to use the
1226          * '-m' mountpoint option if it looks like its in use.
1227          */
1228         if (mountpoint == NULL ||
1229             (strcmp(mountpoint, ZFS_MOUNTPOINT_LEGACY) != 0 &&
1230             strcmp(mountpoint, ZFS_MOUNTPOINT_NONE) != 0)) {
1231                 char buf[MAXPATHLEN];
1232                 DIR *dirp;
1233
1234                 if (mountpoint && mountpoint[0] != '/') {
1235                         (void) fprintf(stderr, gettext("invalid mountpoint "
1236                             "'%s': must be an absolute path, 'legacy', or "
1237                             "'none'\n"), mountpoint);
1238                         goto errout;
1239                 }
1240
1241                 if (mountpoint == NULL) {
1242                         if (altroot != NULL)
1243                                 (void) snprintf(buf, sizeof (buf), "%s/%s",
1244                                     altroot, poolname);
1245                         else
1246                                 (void) snprintf(buf, sizeof (buf), "/%s",
1247                                     poolname);
1248                 } else {
1249                         if (altroot != NULL)
1250                                 (void) snprintf(buf, sizeof (buf), "%s%s",
1251                                     altroot, mountpoint);
1252                         else
1253                                 (void) snprintf(buf, sizeof (buf), "%s",
1254                                     mountpoint);
1255                 }
1256
1257                 if ((dirp = opendir(buf)) == NULL && errno != ENOENT) {
1258                         (void) fprintf(stderr, gettext("mountpoint '%s' : "
1259                             "%s\n"), buf, strerror(errno));
1260                         (void) fprintf(stderr, gettext("use '-m' "
1261                             "option to provide a different default\n"));
1262                         goto errout;
1263                 } else if (dirp) {
1264                         int count = 0;
1265
1266                         while (count < 3 && readdir(dirp) != NULL)
1267                                 count++;
1268                         (void) closedir(dirp);
1269
1270                         if (count > 2) {
1271                                 (void) fprintf(stderr, gettext("mountpoint "
1272                                     "'%s' exists and is not empty\n"), buf);
1273                                 (void) fprintf(stderr, gettext("use '-m' "
1274                                     "option to provide a "
1275                                     "different default\n"));
1276                                 goto errout;
1277                         }
1278                 }
1279         }
1280
1281         /*
1282          * Now that the mountpoint's validity has been checked, ensure that
1283          * the property is set appropriately prior to creating the pool.
1284          */
1285         if (mountpoint != NULL) {
1286                 ret = add_prop_list(zfs_prop_to_name(ZFS_PROP_MOUNTPOINT),
1287                     mountpoint, &fsprops, B_FALSE);
1288                 if (ret != 0)
1289                         goto errout;
1290         }
1291
1292         ret = 1;
1293         if (dryrun) {
1294                 /*
1295                  * For a dry run invocation, print out a basic message and run
1296                  * through all the vdevs in the list and print out in an
1297                  * appropriate hierarchy.
1298                  */
1299                 (void) printf(gettext("would create '%s' with the "
1300                     "following layout:\n\n"), poolname);
1301
1302                 print_vdev_tree(NULL, poolname, nvroot, 0, B_FALSE, 0);
1303                 if (num_logs(nvroot) > 0)
1304                         print_vdev_tree(NULL, "logs", nvroot, 0, B_TRUE, 0);
1305
1306                 ret = 0;
1307         } else {
1308                 /*
1309                  * Hand off to libzfs.
1310                  */
1311                 spa_feature_t i;
1312                 for (i = 0; i < SPA_FEATURES; i++) {
1313                         char propname[MAXPATHLEN];
1314                         char *propval;
1315                         zfeature_info_t *feat = &spa_feature_table[i];
1316
1317                         (void) snprintf(propname, sizeof (propname),
1318                             "feature@%s", feat->fi_uname);
1319
1320                         /*
1321                          * Only features contained in props will be enabled:
1322                          * remove from the nvlist every ZFS_FEATURE_DISABLED
1323                          * value and add every missing ZFS_FEATURE_ENABLED if
1324                          * enable_all_pool_feat is set.
1325                          */
1326                         if (!nvlist_lookup_string(props, propname, &propval)) {
1327                                 if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
1328                                         (void) nvlist_remove_all(props,
1329                                             propname);
1330                         } else if (enable_all_pool_feat) {
1331                                 ret = add_prop_list(propname,
1332                                     ZFS_FEATURE_ENABLED, &props, B_TRUE);
1333                                 if (ret != 0)
1334                                         goto errout;
1335                         }
1336                 }
1337
1338                 ret = 1;
1339                 if (zpool_create(g_zfs, poolname,
1340                     nvroot, props, fsprops) == 0) {
1341                         zfs_handle_t *pool = zfs_open(g_zfs,
1342                             tname ? tname : poolname, ZFS_TYPE_FILESYSTEM);
1343                         if (pool != NULL) {
1344                                 if (zfs_mount(pool, NULL, 0) == 0)
1345                                         ret = zfs_shareall(pool);
1346                                 zfs_close(pool);
1347                         }
1348                 } else if (libzfs_errno(g_zfs) == EZFS_INVALIDNAME) {
1349                         (void) fprintf(stderr, gettext("pool name may have "
1350                             "been omitted\n"));
1351                 }
1352         }
1353
1354 errout:
1355         nvlist_free(nvroot);
1356         nvlist_free(fsprops);
1357         nvlist_free(props);
1358         return (ret);
1359 badusage:
1360         nvlist_free(fsprops);
1361         nvlist_free(props);
1362         usage(B_FALSE);
1363         return (2);
1364 }
1365
1366 /*
1367  * zpool destroy <pool>
1368  *
1369  *      -f      Forcefully unmount any datasets
1370  *
1371  * Destroy the given pool.  Automatically unmounts any datasets in the pool.
1372  */
1373 int
1374 zpool_do_destroy(int argc, char **argv)
1375 {
1376         boolean_t force = B_FALSE;
1377         int c;
1378         char *pool;
1379         zpool_handle_t *zhp;
1380         int ret;
1381
1382         /* check options */
1383         while ((c = getopt(argc, argv, "f")) != -1) {
1384                 switch (c) {
1385                 case 'f':
1386                         force = B_TRUE;
1387                         break;
1388                 case '?':
1389                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1390                             optopt);
1391                         usage(B_FALSE);
1392                 }
1393         }
1394
1395         argc -= optind;
1396         argv += optind;
1397
1398         /* check arguments */
1399         if (argc < 1) {
1400                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1401                 usage(B_FALSE);
1402         }
1403         if (argc > 1) {
1404                 (void) fprintf(stderr, gettext("too many arguments\n"));
1405                 usage(B_FALSE);
1406         }
1407
1408         pool = argv[0];
1409
1410         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
1411                 /*
1412                  * As a special case, check for use of '/' in the name, and
1413                  * direct the user to use 'zfs destroy' instead.
1414                  */
1415                 if (strchr(pool, '/') != NULL)
1416                         (void) fprintf(stderr, gettext("use 'zfs destroy' to "
1417                             "destroy a dataset\n"));
1418                 return (1);
1419         }
1420
1421         if (zpool_disable_datasets(zhp, force) != 0) {
1422                 (void) fprintf(stderr, gettext("could not destroy '%s': "
1423                     "could not unmount datasets\n"), zpool_get_name(zhp));
1424                 zpool_close(zhp);
1425                 return (1);
1426         }
1427
1428         /* The history must be logged as part of the export */
1429         log_history = B_FALSE;
1430
1431         ret = (zpool_destroy(zhp, history_str) != 0);
1432
1433         zpool_close(zhp);
1434
1435         return (ret);
1436 }
1437
1438 typedef struct export_cbdata {
1439         boolean_t force;
1440         boolean_t hardforce;
1441 } export_cbdata_t;
1442
1443 /*
1444  * Export one pool
1445  */
1446 int
1447 zpool_export_one(zpool_handle_t *zhp, void *data)
1448 {
1449         export_cbdata_t *cb = data;
1450
1451         if (zpool_disable_datasets(zhp, cb->force) != 0)
1452                 return (1);
1453
1454         /* The history must be logged as part of the export */
1455         log_history = B_FALSE;
1456
1457         if (cb->hardforce) {
1458                 if (zpool_export_force(zhp, history_str) != 0)
1459                         return (1);
1460         } else if (zpool_export(zhp, cb->force, history_str) != 0) {
1461                 return (1);
1462         }
1463
1464         return (0);
1465 }
1466
1467 /*
1468  * zpool export [-f] <pool> ...
1469  *
1470  *      -a      Export all pools
1471  *      -f      Forcefully unmount datasets
1472  *
1473  * Export the given pools.  By default, the command will attempt to cleanly
1474  * unmount any active datasets within the pool.  If the '-f' flag is specified,
1475  * then the datasets will be forcefully unmounted.
1476  */
1477 int
1478 zpool_do_export(int argc, char **argv)
1479 {
1480         export_cbdata_t cb;
1481         boolean_t do_all = B_FALSE;
1482         boolean_t force = B_FALSE;
1483         boolean_t hardforce = B_FALSE;
1484         int c, ret;
1485
1486         /* check options */
1487         while ((c = getopt(argc, argv, "afF")) != -1) {
1488                 switch (c) {
1489                 case 'a':
1490                         do_all = B_TRUE;
1491                         break;
1492                 case 'f':
1493                         force = B_TRUE;
1494                         break;
1495                 case 'F':
1496                         hardforce = B_TRUE;
1497                         break;
1498                 case '?':
1499                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
1500                             optopt);
1501                         usage(B_FALSE);
1502                 }
1503         }
1504
1505         cb.force = force;
1506         cb.hardforce = hardforce;
1507         argc -= optind;
1508         argv += optind;
1509
1510         if (do_all) {
1511                 if (argc != 0) {
1512                         (void) fprintf(stderr, gettext("too many arguments\n"));
1513                         usage(B_FALSE);
1514                 }
1515
1516                 return (for_each_pool(argc, argv, B_TRUE, NULL,
1517                     zpool_export_one, &cb));
1518         }
1519
1520         /* check arguments */
1521         if (argc < 1) {
1522                 (void) fprintf(stderr, gettext("missing pool argument\n"));
1523                 usage(B_FALSE);
1524         }
1525
1526         ret = for_each_pool(argc, argv, B_TRUE, NULL, zpool_export_one, &cb);
1527
1528         return (ret);
1529 }
1530
1531 /*
1532  * Given a vdev configuration, determine the maximum width needed for the device
1533  * name column.
1534  */
1535 static int
1536 max_width(zpool_handle_t *zhp, nvlist_t *nv, int depth, int max,
1537     int name_flags)
1538 {
1539         char *name;
1540         nvlist_t **child;
1541         uint_t c, children;
1542         int ret;
1543
1544         name = zpool_vdev_name(g_zfs, zhp, nv, name_flags);
1545         if (strlen(name) + depth > max)
1546                 max = strlen(name) + depth;
1547
1548         free(name);
1549
1550         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1551             &child, &children) == 0) {
1552                 for (c = 0; c < children; c++)
1553                         if ((ret = max_width(zhp, child[c], depth + 2,
1554                             max, name_flags)) > max)
1555                                 max = ret;
1556         }
1557
1558         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1559             &child, &children) == 0) {
1560                 for (c = 0; c < children; c++)
1561                         if ((ret = max_width(zhp, child[c], depth + 2,
1562                             max, name_flags)) > max)
1563                                 max = ret;
1564         }
1565
1566         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1567             &child, &children) == 0) {
1568                 for (c = 0; c < children; c++)
1569                         if ((ret = max_width(zhp, child[c], depth + 2,
1570                             max, name_flags)) > max)
1571                                 max = ret;
1572         }
1573
1574         return (max);
1575 }
1576
1577 typedef struct spare_cbdata {
1578         uint64_t        cb_guid;
1579         zpool_handle_t  *cb_zhp;
1580 } spare_cbdata_t;
1581
1582 static boolean_t
1583 find_vdev(nvlist_t *nv, uint64_t search)
1584 {
1585         uint64_t guid;
1586         nvlist_t **child;
1587         uint_t c, children;
1588
1589         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
1590             search == guid)
1591                 return (B_TRUE);
1592
1593         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1594             &child, &children) == 0) {
1595                 for (c = 0; c < children; c++)
1596                         if (find_vdev(child[c], search))
1597                                 return (B_TRUE);
1598         }
1599
1600         return (B_FALSE);
1601 }
1602
1603 static int
1604 find_spare(zpool_handle_t *zhp, void *data)
1605 {
1606         spare_cbdata_t *cbp = data;
1607         nvlist_t *config, *nvroot;
1608
1609         config = zpool_get_config(zhp, NULL);
1610         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
1611             &nvroot) == 0);
1612
1613         if (find_vdev(nvroot, cbp->cb_guid)) {
1614                 cbp->cb_zhp = zhp;
1615                 return (1);
1616         }
1617
1618         zpool_close(zhp);
1619         return (0);
1620 }
1621
1622 typedef struct status_cbdata {
1623         int             cb_count;
1624         int             cb_name_flags;
1625         int             cb_namewidth;
1626         boolean_t       cb_allpools;
1627         boolean_t       cb_verbose;
1628         boolean_t       cb_explain;
1629         boolean_t       cb_first;
1630         boolean_t       cb_dedup_stats;
1631         boolean_t       cb_print_status;
1632         vdev_cmd_data_list_t    *vcdl;
1633 } status_cbdata_t;
1634
1635 /* Return 1 if string is NULL, empty, or whitespace; return 0 otherwise. */
1636 static int
1637 is_blank_str(char *str)
1638 {
1639         while (str != NULL && *str != '\0') {
1640                 if (!isblank(*str))
1641                         return (0);
1642                 str++;
1643         }
1644         return (1);
1645 }
1646
1647 /* Print command output lines for specific vdev in a specific pool */
1648 static void
1649 zpool_print_cmd(vdev_cmd_data_list_t *vcdl, const char *pool, char *path)
1650 {
1651         vdev_cmd_data_t *data;
1652         int i, j;
1653         char *val;
1654
1655         for (i = 0; i < vcdl->count; i++) {
1656                 if ((strcmp(vcdl->data[i].path, path) != 0) ||
1657                     (strcmp(vcdl->data[i].pool, pool) != 0)) {
1658                         /* Not the vdev we're looking for */
1659                         continue;
1660                 }
1661
1662                 data = &vcdl->data[i];
1663                 /* Print out all the output values for this vdev */
1664                 for (j = 0; j < vcdl->uniq_cols_cnt; j++) {
1665                         val = NULL;
1666                         /* Does this vdev have values for this column? */
1667                         for (int k = 0; k < data->cols_cnt; k++) {
1668                                 if (strcmp(data->cols[k],
1669                                     vcdl->uniq_cols[j]) == 0) {
1670                                         /* yes it does, record the value */
1671                                         val = data->lines[k];
1672                                         break;
1673                                 }
1674                         }
1675                         /*
1676                          * Mark empty values with dashes to make output
1677                          * awk-able.
1678                          */
1679                         if (is_blank_str(val))
1680                                 val = "-";
1681
1682                         printf("%*s", vcdl->uniq_cols_width[j], val);
1683                         if (j < vcdl->uniq_cols_cnt - 1)
1684                                 printf("  ");
1685                 }
1686
1687                 /* Print out any values that aren't in a column at the end */
1688                 for (j = data->cols_cnt; j < data->lines_cnt; j++) {
1689                         /* Did we have any columns?  If so print a spacer. */
1690                         if (vcdl->uniq_cols_cnt > 0)
1691                                 printf("  ");
1692
1693                         val = data->lines[j];
1694                         printf("%s", val ? val : "");
1695                 }
1696                 break;
1697         }
1698 }
1699
1700 /*
1701  * Print out configuration state as requested by status_callback.
1702  */
1703 static void
1704 print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
1705     nvlist_t *nv, int depth, boolean_t isspare)
1706 {
1707         nvlist_t **child;
1708         uint_t c, children;
1709         pool_scan_stat_t *ps = NULL;
1710         vdev_stat_t *vs;
1711         char rbuf[6], wbuf[6], cbuf[6];
1712         char *vname;
1713         uint64_t notpresent;
1714         spare_cbdata_t spare_cb;
1715         const char *state;
1716         char *type;
1717         char *path = NULL;
1718
1719         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1720             &child, &children) != 0)
1721                 children = 0;
1722
1723         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
1724             (uint64_t **)&vs, &c) == 0);
1725
1726         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1727
1728         if (strcmp(type, VDEV_TYPE_INDIRECT) == 0)
1729                 return;
1730
1731         state = zpool_state_to_name(vs->vs_state, vs->vs_aux);
1732         if (isspare) {
1733                 /*
1734                  * For hot spares, we use the terms 'INUSE' and 'AVAILABLE' for
1735                  * online drives.
1736                  */
1737                 if (vs->vs_aux == VDEV_AUX_SPARED)
1738                         state = "INUSE";
1739                 else if (vs->vs_state == VDEV_STATE_HEALTHY)
1740                         state = "AVAIL";
1741         }
1742
1743         (void) printf("\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
1744             name, state);
1745
1746         if (!isspare) {
1747                 zfs_nicenum(vs->vs_read_errors, rbuf, sizeof (rbuf));
1748                 zfs_nicenum(vs->vs_write_errors, wbuf, sizeof (wbuf));
1749                 zfs_nicenum(vs->vs_checksum_errors, cbuf, sizeof (cbuf));
1750                 (void) printf(" %5s %5s %5s", rbuf, wbuf, cbuf);
1751         }
1752
1753         if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_NOT_PRESENT,
1754             &notpresent) == 0) {
1755                 verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0);
1756                 (void) printf("  was %s", path);
1757         } else if (vs->vs_aux != 0) {
1758                 (void) printf("  ");
1759
1760                 switch (vs->vs_aux) {
1761                 case VDEV_AUX_OPEN_FAILED:
1762                         (void) printf(gettext("cannot open"));
1763                         break;
1764
1765                 case VDEV_AUX_BAD_GUID_SUM:
1766                         (void) printf(gettext("missing device"));
1767                         break;
1768
1769                 case VDEV_AUX_NO_REPLICAS:
1770                         (void) printf(gettext("insufficient replicas"));
1771                         break;
1772
1773                 case VDEV_AUX_VERSION_NEWER:
1774                         (void) printf(gettext("newer version"));
1775                         break;
1776
1777                 case VDEV_AUX_UNSUP_FEAT:
1778                         (void) printf(gettext("unsupported feature(s)"));
1779                         break;
1780
1781                 case VDEV_AUX_SPARED:
1782                         verify(nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID,
1783                             &spare_cb.cb_guid) == 0);
1784                         if (zpool_iter(g_zfs, find_spare, &spare_cb) == 1) {
1785                                 if (strcmp(zpool_get_name(spare_cb.cb_zhp),
1786                                     zpool_get_name(zhp)) == 0)
1787                                         (void) printf(gettext("currently in "
1788                                             "use"));
1789                                 else
1790                                         (void) printf(gettext("in use by "
1791                                             "pool '%s'"),
1792                                             zpool_get_name(spare_cb.cb_zhp));
1793                                 zpool_close(spare_cb.cb_zhp);
1794                         } else {
1795                                 (void) printf(gettext("currently in use"));
1796                         }
1797                         break;
1798
1799                 case VDEV_AUX_ERR_EXCEEDED:
1800                         (void) printf(gettext("too many errors"));
1801                         break;
1802
1803                 case VDEV_AUX_IO_FAILURE:
1804                         (void) printf(gettext("experienced I/O failures"));
1805                         break;
1806
1807                 case VDEV_AUX_BAD_LOG:
1808                         (void) printf(gettext("bad intent log"));
1809                         break;
1810
1811                 case VDEV_AUX_EXTERNAL:
1812                         (void) printf(gettext("external device fault"));
1813                         break;
1814
1815                 case VDEV_AUX_SPLIT_POOL:
1816                         (void) printf(gettext("split into new pool"));
1817                         break;
1818
1819                 case VDEV_AUX_ACTIVE:
1820                         (void) printf(gettext("currently in use"));
1821                         break;
1822
1823                 case VDEV_AUX_CHILDREN_OFFLINE:
1824                         (void) printf(gettext("all children offline"));
1825                         break;
1826
1827                 default:
1828                         (void) printf(gettext("corrupted data"));
1829                         break;
1830                 }
1831         }
1832
1833         (void) nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_SCAN_STATS,
1834             (uint64_t **)&ps, &c);
1835
1836         if (ps != NULL && ps->pss_state == DSS_SCANNING &&
1837             vs->vs_scan_processed != 0 && children == 0) {
1838                 (void) printf(gettext("  (%s)"),
1839                     (ps->pss_func == POOL_SCAN_RESILVER) ?
1840                     "resilvering" : "repairing");
1841         }
1842
1843         if (cb->vcdl != NULL) {
1844                 if (nvlist_lookup_string(nv, ZPOOL_CONFIG_PATH, &path) == 0) {
1845                         printf("  ");
1846                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
1847                 }
1848         }
1849
1850         (void) printf("\n");
1851
1852         for (c = 0; c < children; c++) {
1853                 uint64_t islog = B_FALSE, ishole = B_FALSE;
1854
1855                 /* Don't print logs or holes here */
1856                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1857                     &islog);
1858                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE,
1859                     &ishole);
1860                 if (islog || ishole)
1861                         continue;
1862                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
1863                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
1864                 print_status_config(zhp, cb, vname, child[c], depth + 2,
1865                     isspare);
1866                 free(vname);
1867         }
1868 }
1869
1870 /*
1871  * Print the configuration of an exported pool.  Iterate over all vdevs in the
1872  * pool, printing out the name and status for each one.
1873  */
1874 static void
1875 print_import_config(status_cbdata_t *cb, const char *name, nvlist_t *nv,
1876     int depth)
1877 {
1878         nvlist_t **child;
1879         uint_t c, children;
1880         vdev_stat_t *vs;
1881         char *type, *vname;
1882
1883         verify(nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) == 0);
1884         if (strcmp(type, VDEV_TYPE_MISSING) == 0 ||
1885             strcmp(type, VDEV_TYPE_HOLE) == 0)
1886                 return;
1887
1888         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
1889             (uint64_t **)&vs, &c) == 0);
1890
1891         (void) printf("\t%*s%-*s", depth, "", cb->cb_namewidth - depth, name);
1892         (void) printf("  %s", zpool_state_to_name(vs->vs_state, vs->vs_aux));
1893
1894         if (vs->vs_aux != 0) {
1895                 (void) printf("  ");
1896
1897                 switch (vs->vs_aux) {
1898                 case VDEV_AUX_OPEN_FAILED:
1899                         (void) printf(gettext("cannot open"));
1900                         break;
1901
1902                 case VDEV_AUX_BAD_GUID_SUM:
1903                         (void) printf(gettext("missing device"));
1904                         break;
1905
1906                 case VDEV_AUX_NO_REPLICAS:
1907                         (void) printf(gettext("insufficient replicas"));
1908                         break;
1909
1910                 case VDEV_AUX_VERSION_NEWER:
1911                         (void) printf(gettext("newer version"));
1912                         break;
1913
1914                 case VDEV_AUX_UNSUP_FEAT:
1915                         (void) printf(gettext("unsupported feature(s)"));
1916                         break;
1917
1918                 case VDEV_AUX_ERR_EXCEEDED:
1919                         (void) printf(gettext("too many errors"));
1920                         break;
1921
1922                 case VDEV_AUX_ACTIVE:
1923                         (void) printf(gettext("currently in use"));
1924                         break;
1925
1926                 case VDEV_AUX_CHILDREN_OFFLINE:
1927                         (void) printf(gettext("all children offline"));
1928                         break;
1929
1930                 default:
1931                         (void) printf(gettext("corrupted data"));
1932                         break;
1933                 }
1934         }
1935         (void) printf("\n");
1936
1937         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
1938             &child, &children) != 0)
1939                 return;
1940
1941         for (c = 0; c < children; c++) {
1942                 uint64_t is_log = B_FALSE;
1943
1944                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
1945                     &is_log);
1946                 if (is_log)
1947                         continue;
1948
1949                 vname = zpool_vdev_name(g_zfs, NULL, child[c],
1950                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
1951                 print_import_config(cb, vname, child[c], depth + 2);
1952                 free(vname);
1953         }
1954
1955         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
1956             &child, &children) == 0) {
1957                 (void) printf(gettext("\tcache\n"));
1958                 for (c = 0; c < children; c++) {
1959                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
1960                             cb->cb_name_flags);
1961                         (void) printf("\t  %s\n", vname);
1962                         free(vname);
1963                 }
1964         }
1965
1966         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES,
1967             &child, &children) == 0) {
1968                 (void) printf(gettext("\tspares\n"));
1969                 for (c = 0; c < children; c++) {
1970                         vname = zpool_vdev_name(g_zfs, NULL, child[c],
1971                             cb->cb_name_flags);
1972                         (void) printf("\t  %s\n", vname);
1973                         free(vname);
1974                 }
1975         }
1976 }
1977
1978 /*
1979  * Print log vdevs.
1980  * Logs are recorded as top level vdevs in the main pool child array
1981  * but with "is_log" set to 1. We use either print_status_config() or
1982  * print_import_config() to print the top level logs then any log
1983  * children (eg mirrored slogs) are printed recursively - which
1984  * works because only the top level vdev is marked "is_log"
1985  */
1986 static void
1987 print_logs(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv)
1988 {
1989         uint_t c, children;
1990         nvlist_t **child;
1991
1992         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, &child,
1993             &children) != 0)
1994                 return;
1995
1996         (void) printf(gettext("\tlogs\n"));
1997
1998         for (c = 0; c < children; c++) {
1999                 uint64_t is_log = B_FALSE;
2000                 char *name;
2001
2002                 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
2003                     &is_log);
2004                 if (!is_log)
2005                         continue;
2006                 name = zpool_vdev_name(g_zfs, zhp, child[c],
2007                     cb->cb_name_flags | VDEV_NAME_TYPE_ID);
2008                 if (cb->cb_print_status)
2009                         print_status_config(zhp, cb, name, child[c], 2,
2010                             B_FALSE);
2011                 else
2012                         print_import_config(cb, name, child[c], 2);
2013                 free(name);
2014         }
2015 }
2016
2017 /*
2018  * Display the status for the given pool.
2019  */
2020 static void
2021 show_import(nvlist_t *config)
2022 {
2023         uint64_t pool_state;
2024         vdev_stat_t *vs;
2025         char *name;
2026         uint64_t guid;
2027         uint64_t hostid = 0;
2028         char *msgid;
2029         char *hostname = "unknown";
2030         nvlist_t *nvroot, *nvinfo;
2031         zpool_status_t reason;
2032         zpool_errata_t errata;
2033         const char *health;
2034         uint_t vsc;
2035         char *comment;
2036         status_cbdata_t cb = { 0 };
2037
2038         verify(nvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME,
2039             &name) == 0);
2040         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID,
2041             &guid) == 0);
2042         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2043             &pool_state) == 0);
2044         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
2045             &nvroot) == 0);
2046
2047         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
2048             (uint64_t **)&vs, &vsc) == 0);
2049         health = zpool_state_to_name(vs->vs_state, vs->vs_aux);
2050
2051         reason = zpool_import_status(config, &msgid, &errata);
2052
2053         (void) printf(gettext("   pool: %s\n"), name);
2054         (void) printf(gettext("     id: %llu\n"), (u_longlong_t)guid);
2055         (void) printf(gettext("  state: %s"), health);
2056         if (pool_state == POOL_STATE_DESTROYED)
2057                 (void) printf(gettext(" (DESTROYED)"));
2058         (void) printf("\n");
2059
2060         switch (reason) {
2061         case ZPOOL_STATUS_MISSING_DEV_R:
2062         case ZPOOL_STATUS_MISSING_DEV_NR:
2063         case ZPOOL_STATUS_BAD_GUID_SUM:
2064                 (void) printf(gettext(" status: One or more devices are "
2065                     "missing from the system.\n"));
2066                 break;
2067
2068         case ZPOOL_STATUS_CORRUPT_LABEL_R:
2069         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
2070                 (void) printf(gettext(" status: One or more devices contains "
2071                     "corrupted data.\n"));
2072                 break;
2073
2074         case ZPOOL_STATUS_CORRUPT_DATA:
2075                 (void) printf(
2076                     gettext(" status: The pool data is corrupted.\n"));
2077                 break;
2078
2079         case ZPOOL_STATUS_OFFLINE_DEV:
2080                 (void) printf(gettext(" status: One or more devices "
2081                     "are offlined.\n"));
2082                 break;
2083
2084         case ZPOOL_STATUS_CORRUPT_POOL:
2085                 (void) printf(gettext(" status: The pool metadata is "
2086                     "corrupted.\n"));
2087                 break;
2088
2089         case ZPOOL_STATUS_VERSION_OLDER:
2090                 (void) printf(gettext(" status: The pool is formatted using a "
2091                     "legacy on-disk version.\n"));
2092                 break;
2093
2094         case ZPOOL_STATUS_VERSION_NEWER:
2095                 (void) printf(gettext(" status: The pool is formatted using an "
2096                     "incompatible version.\n"));
2097                 break;
2098
2099         case ZPOOL_STATUS_FEAT_DISABLED:
2100                 (void) printf(gettext(" status: Some supported features are "
2101                     "not enabled on the pool.\n"));
2102                 break;
2103
2104         case ZPOOL_STATUS_UNSUP_FEAT_READ:
2105                 (void) printf(gettext("status: The pool uses the following "
2106                     "feature(s) not supported on this system:\n"));
2107                 zpool_print_unsup_feat(config);
2108                 break;
2109
2110         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2111                 (void) printf(gettext("status: The pool can only be accessed "
2112                     "in read-only mode on this system. It\n\tcannot be "
2113                     "accessed in read-write mode because it uses the "
2114                     "following\n\tfeature(s) not supported on this system:\n"));
2115                 zpool_print_unsup_feat(config);
2116                 break;
2117
2118         case ZPOOL_STATUS_HOSTID_ACTIVE:
2119                 (void) printf(gettext(" status: The pool is currently "
2120                     "imported by another system.\n"));
2121                 break;
2122
2123         case ZPOOL_STATUS_HOSTID_REQUIRED:
2124                 (void) printf(gettext(" status: The pool has the "
2125                     "multihost property on.  It cannot\n\tbe safely imported "
2126                     "when the system hostid is not set.\n"));
2127                 break;
2128
2129         case ZPOOL_STATUS_HOSTID_MISMATCH:
2130                 (void) printf(gettext(" status: The pool was last accessed by "
2131                     "another system.\n"));
2132                 break;
2133
2134         case ZPOOL_STATUS_FAULTED_DEV_R:
2135         case ZPOOL_STATUS_FAULTED_DEV_NR:
2136                 (void) printf(gettext(" status: One or more devices are "
2137                     "faulted.\n"));
2138                 break;
2139
2140         case ZPOOL_STATUS_BAD_LOG:
2141                 (void) printf(gettext(" status: An intent log record cannot be "
2142                     "read.\n"));
2143                 break;
2144
2145         case ZPOOL_STATUS_RESILVERING:
2146                 (void) printf(gettext(" status: One or more devices were being "
2147                     "resilvered.\n"));
2148                 break;
2149
2150         case ZPOOL_STATUS_ERRATA:
2151                 (void) printf(gettext(" status: Errata #%d detected.\n"),
2152                     errata);
2153                 break;
2154
2155         default:
2156                 /*
2157                  * No other status can be seen when importing pools.
2158                  */
2159                 assert(reason == ZPOOL_STATUS_OK);
2160         }
2161
2162         /*
2163          * Print out an action according to the overall state of the pool.
2164          */
2165         if (vs->vs_state == VDEV_STATE_HEALTHY) {
2166                 if (reason == ZPOOL_STATUS_VERSION_OLDER ||
2167                     reason == ZPOOL_STATUS_FEAT_DISABLED) {
2168                         (void) printf(gettext(" action: The pool can be "
2169                             "imported using its name or numeric identifier, "
2170                             "though\n\tsome features will not be available "
2171                             "without an explicit 'zpool upgrade'.\n"));
2172                 } else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
2173                         (void) printf(gettext(" action: The pool can be "
2174                             "imported using its name or numeric "
2175                             "identifier and\n\tthe '-f' flag.\n"));
2176                 } else if (reason == ZPOOL_STATUS_ERRATA) {
2177                         switch (errata) {
2178                         case ZPOOL_ERRATA_NONE:
2179                                 break;
2180
2181                         case ZPOOL_ERRATA_ZOL_2094_SCRUB:
2182                                 (void) printf(gettext(" action: The pool can "
2183                                     "be imported using its name or numeric "
2184                                     "identifier,\n\thowever there is a compat"
2185                                     "ibility issue which should be corrected"
2186                                     "\n\tby running 'zpool scrub'\n"));
2187                                 break;
2188
2189                         case ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY:
2190                                 (void) printf(gettext(" action: The pool can"
2191                                     "not be imported with this version of ZFS "
2192                                     "due to\n\tan active asynchronous destroy. "
2193                                     "Revert to an earlier version\n\tand "
2194                                     "allow the destroy to complete before "
2195                                     "updating.\n"));
2196                                 break;
2197
2198                         case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
2199                                 (void) printf(gettext(" action: Existing "
2200                                     "encrypted datasets contain an on-disk "
2201                                     "incompatibility, which\n\tneeds to be "
2202                                     "corrected. Backup these datasets to new "
2203                                     "encrypted datasets\n\tand destroy the "
2204                                     "old ones.\n"));
2205                                 break;
2206
2207                         default:
2208                                 /*
2209                                  * All errata must contain an action message.
2210                                  */
2211                                 assert(0);
2212                         }
2213                 } else {
2214                         (void) printf(gettext(" action: The pool can be "
2215                             "imported using its name or numeric "
2216                             "identifier.\n"));
2217                 }
2218         } else if (vs->vs_state == VDEV_STATE_DEGRADED) {
2219                 (void) printf(gettext(" action: The pool can be imported "
2220                     "despite missing or damaged devices.  The\n\tfault "
2221                     "tolerance of the pool may be compromised if imported.\n"));
2222         } else {
2223                 switch (reason) {
2224                 case ZPOOL_STATUS_VERSION_NEWER:
2225                         (void) printf(gettext(" action: The pool cannot be "
2226                             "imported.  Access the pool on a system running "
2227                             "newer\n\tsoftware, or recreate the pool from "
2228                             "backup.\n"));
2229                         break;
2230                 case ZPOOL_STATUS_UNSUP_FEAT_READ:
2231                         (void) printf(gettext("action: The pool cannot be "
2232                             "imported. Access the pool on a system that "
2233                             "supports\n\tthe required feature(s), or recreate "
2234                             "the pool from backup.\n"));
2235                         break;
2236                 case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
2237                         (void) printf(gettext("action: The pool cannot be "
2238                             "imported in read-write mode. Import the pool "
2239                             "with\n"
2240                             "\t\"-o readonly=on\", access the pool on a system "
2241                             "that supports the\n\trequired feature(s), or "
2242                             "recreate the pool from backup.\n"));
2243                         break;
2244                 case ZPOOL_STATUS_MISSING_DEV_R:
2245                 case ZPOOL_STATUS_MISSING_DEV_NR:
2246                 case ZPOOL_STATUS_BAD_GUID_SUM:
2247                         (void) printf(gettext(" action: The pool cannot be "
2248                             "imported. Attach the missing\n\tdevices and try "
2249                             "again.\n"));
2250                         break;
2251                 case ZPOOL_STATUS_HOSTID_ACTIVE:
2252                         VERIFY0(nvlist_lookup_nvlist(config,
2253                             ZPOOL_CONFIG_LOAD_INFO, &nvinfo));
2254
2255                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2256                                 hostname = fnvlist_lookup_string(nvinfo,
2257                                     ZPOOL_CONFIG_MMP_HOSTNAME);
2258
2259                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2260                                 hostid = fnvlist_lookup_uint64(nvinfo,
2261                                     ZPOOL_CONFIG_MMP_HOSTID);
2262
2263                         (void) printf(gettext(" action: The pool must be "
2264                             "exported from %s (hostid=%lx)\n\tbefore it "
2265                             "can be safely imported.\n"), hostname,
2266                             (unsigned long) hostid);
2267                         break;
2268                 case ZPOOL_STATUS_HOSTID_REQUIRED:
2269                         (void) printf(gettext(" action: Set a unique system "
2270                             "hostid with the zgenhostid(8) command.\n"));
2271                         break;
2272                 default:
2273                         (void) printf(gettext(" action: The pool cannot be "
2274                             "imported due to damaged devices or data.\n"));
2275                 }
2276         }
2277
2278         /* Print the comment attached to the pool. */
2279         if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0)
2280                 (void) printf(gettext("comment: %s\n"), comment);
2281
2282         /*
2283          * If the state is "closed" or "can't open", and the aux state
2284          * is "corrupt data":
2285          */
2286         if (((vs->vs_state == VDEV_STATE_CLOSED) ||
2287             (vs->vs_state == VDEV_STATE_CANT_OPEN)) &&
2288             (vs->vs_aux == VDEV_AUX_CORRUPT_DATA)) {
2289                 if (pool_state == POOL_STATE_DESTROYED)
2290                         (void) printf(gettext("\tThe pool was destroyed, "
2291                             "but can be imported using the '-Df' flags.\n"));
2292                 else if (pool_state != POOL_STATE_EXPORTED)
2293                         (void) printf(gettext("\tThe pool may be active on "
2294                             "another system, but can be imported using\n\t"
2295                             "the '-f' flag.\n"));
2296         }
2297
2298         if (msgid != NULL)
2299                 (void) printf(gettext("   see: http://zfsonlinux.org/msg/%s\n"),
2300                     msgid);
2301
2302         (void) printf(gettext(" config:\n\n"));
2303
2304         cb.cb_namewidth = max_width(NULL, nvroot, 0, strlen(name),
2305             VDEV_NAME_TYPE_ID);
2306         if (cb.cb_namewidth < 10)
2307                 cb.cb_namewidth = 10;
2308
2309         print_import_config(&cb, name, nvroot, 0);
2310         if (num_logs(nvroot) > 0)
2311                 print_logs(NULL, &cb, nvroot);
2312
2313         if (reason == ZPOOL_STATUS_BAD_GUID_SUM) {
2314                 (void) printf(gettext("\n\tAdditional devices are known to "
2315                     "be part of this pool, though their\n\texact "
2316                     "configuration cannot be determined.\n"));
2317         }
2318 }
2319
2320 static boolean_t
2321 zfs_force_import_required(nvlist_t *config)
2322 {
2323         uint64_t state;
2324         uint64_t hostid = 0;
2325         nvlist_t *nvinfo;
2326
2327         state = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE);
2328         (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_HOSTID, &hostid);
2329
2330         if (state != POOL_STATE_EXPORTED && hostid != get_system_hostid())
2331                 return (B_TRUE);
2332
2333         nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2334         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE)) {
2335                 mmp_state_t mmp_state = fnvlist_lookup_uint64(nvinfo,
2336                     ZPOOL_CONFIG_MMP_STATE);
2337
2338                 if (mmp_state != MMP_STATE_INACTIVE)
2339                         return (B_TRUE);
2340         }
2341
2342         return (B_FALSE);
2343 }
2344
2345 /*
2346  * Perform the import for the given configuration.  This passes the heavy
2347  * lifting off to zpool_import_props(), and then mounts the datasets contained
2348  * within the pool.
2349  */
2350 static int
2351 do_import(nvlist_t *config, const char *newname, const char *mntopts,
2352     nvlist_t *props, int flags)
2353 {
2354         int ret = 0;
2355         zpool_handle_t *zhp;
2356         char *name;
2357         uint64_t version;
2358
2359         name = fnvlist_lookup_string(config, ZPOOL_CONFIG_POOL_NAME);
2360         version = fnvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION);
2361
2362         if (!SPA_VERSION_IS_SUPPORTED(version)) {
2363                 (void) fprintf(stderr, gettext("cannot import '%s': pool "
2364                     "is formatted using an unsupported ZFS version\n"), name);
2365                 return (1);
2366         } else if (zfs_force_import_required(config) &&
2367             !(flags & ZFS_IMPORT_ANY_HOST)) {
2368                 mmp_state_t mmp_state = MMP_STATE_INACTIVE;
2369                 nvlist_t *nvinfo;
2370
2371                 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO);
2372                 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_STATE))
2373                         mmp_state = fnvlist_lookup_uint64(nvinfo,
2374                             ZPOOL_CONFIG_MMP_STATE);
2375
2376                 if (mmp_state == MMP_STATE_ACTIVE) {
2377                         char *hostname = "<unknown>";
2378                         uint64_t hostid = 0;
2379
2380                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTNAME))
2381                                 hostname = fnvlist_lookup_string(nvinfo,
2382                                     ZPOOL_CONFIG_MMP_HOSTNAME);
2383
2384                         if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_HOSTID))
2385                                 hostid = fnvlist_lookup_uint64(nvinfo,
2386                                     ZPOOL_CONFIG_MMP_HOSTID);
2387
2388                         (void) fprintf(stderr, gettext("cannot import '%s': "
2389                             "pool is imported on %s (hostid: "
2390                             "0x%lx)\nExport the pool on the other system, "
2391                             "then run 'zpool import'.\n"),
2392                             name, hostname, (unsigned long) hostid);
2393                 } else if (mmp_state == MMP_STATE_NO_HOSTID) {
2394                         (void) fprintf(stderr, gettext("Cannot import '%s': "
2395                             "pool has the multihost property on and the\n"
2396                             "system's hostid is not set. Set a unique hostid "
2397                             "with the zgenhostid(8) command.\n"), name);
2398                 } else {
2399                         char *hostname = "<unknown>";
2400                         uint64_t timestamp = 0;
2401                         uint64_t hostid = 0;
2402
2403                         if (nvlist_exists(config, ZPOOL_CONFIG_HOSTNAME))
2404                                 hostname = fnvlist_lookup_string(config,
2405                                     ZPOOL_CONFIG_HOSTNAME);
2406
2407                         if (nvlist_exists(config, ZPOOL_CONFIG_TIMESTAMP))
2408                                 timestamp = fnvlist_lookup_uint64(config,
2409                                     ZPOOL_CONFIG_TIMESTAMP);
2410
2411                         if (nvlist_exists(config, ZPOOL_CONFIG_HOSTID))
2412                                 hostid = fnvlist_lookup_uint64(config,
2413                                     ZPOOL_CONFIG_HOSTID);
2414
2415                         (void) fprintf(stderr, gettext("cannot import '%s': "
2416                             "pool was previously in use from another system.\n"
2417                             "Last accessed by %s (hostid=%lx) at %s"
2418                             "The pool can be imported, use 'zpool import -f' "
2419                             "to import the pool.\n"), name, hostname,
2420                             (unsigned long)hostid, ctime((time_t *)&timestamp));
2421                 }
2422
2423                 return (1);
2424         }
2425
2426         if (zpool_import_props(g_zfs, config, newname, props, flags) != 0)
2427                 return (1);
2428
2429         if (newname != NULL)
2430                 name = (char *)newname;
2431
2432         if ((zhp = zpool_open_canfail(g_zfs, name)) == NULL)
2433                 return (1);
2434
2435         /*
2436          * Loading keys is best effort. We don't want to return immediately
2437          * if it fails but we do want to give the error to the caller.
2438          */
2439         if (flags & ZFS_IMPORT_LOAD_KEYS) {
2440                 ret = zfs_crypto_attempt_load_keys(g_zfs, name);
2441                 if (ret != 0)
2442                         ret = 1;
2443         }
2444
2445         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
2446             !(flags & ZFS_IMPORT_ONLY) &&
2447             zpool_enable_datasets(zhp, mntopts, 0) != 0) {
2448                 zpool_close(zhp);
2449                 return (1);
2450         }
2451
2452         zpool_close(zhp);
2453         return (ret);
2454 }
2455
2456 /*
2457  * zpool import [-d dir] [-D]
2458  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
2459  *              [-d dir | -c cachefile] [-f] -a
2460  *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-l]
2461  *              [-d dir | -c cachefile] [-f] [-n] [-F] <pool | id> [newpool]
2462  *
2463  *       -c     Read pool information from a cachefile instead of searching
2464  *              devices.
2465  *
2466  *       -d     Scan in a specific directory, other than /dev/.  More than
2467  *              one directory can be specified using multiple '-d' options.
2468  *
2469  *       -D     Scan for previously destroyed pools or import all or only
2470  *              specified destroyed pools.
2471  *
2472  *       -R     Temporarily import the pool, with all mountpoints relative to
2473  *              the given root.  The pool will remain exported when the machine
2474  *              is rebooted.
2475  *
2476  *       -V     Import even in the presence of faulted vdevs.  This is an
2477  *              intentionally undocumented option for testing purposes, and
2478  *              treats the pool configuration as complete, leaving any bad
2479  *              vdevs in the FAULTED state. In other words, it does verbatim
2480  *              import.
2481  *
2482  *       -f     Force import, even if it appears that the pool is active.
2483  *
2484  *       -F     Attempt rewind if necessary.
2485  *
2486  *       -n     See if rewind would work, but don't actually rewind.
2487  *
2488  *       -N     Import the pool but don't mount datasets.
2489  *
2490  *       -T     Specify a starting txg to use for import. This option is
2491  *              intentionally undocumented option for testing purposes.
2492  *
2493  *       -a     Import all pools found.
2494  *
2495  *       -l     Load encryption keys while importing.
2496  *
2497  *       -o     Set property=value and/or temporary mount options (without '=').
2498  *
2499  *       -s     Scan using the default search path, the libblkid cache will
2500  *              not be consulted.
2501  *
2502  * The import command scans for pools to import, and import pools based on pool
2503  * name and GUID.  The pool can also be renamed as part of the import process.
2504  */
2505 int
2506 zpool_do_import(int argc, char **argv)
2507 {
2508         char **searchdirs = NULL;
2509         char *env, *envdup = NULL;
2510         int nsearch = 0;
2511         int c;
2512         int err = 0;
2513         nvlist_t *pools = NULL;
2514         boolean_t do_all = B_FALSE;
2515         boolean_t do_destroyed = B_FALSE;
2516         char *mntopts = NULL;
2517         nvpair_t *elem;
2518         nvlist_t *config;
2519         uint64_t searchguid = 0;
2520         char *searchname = NULL;
2521         char *propval;
2522         nvlist_t *found_config;
2523         nvlist_t *policy = NULL;
2524         nvlist_t *props = NULL;
2525         boolean_t first;
2526         int flags = ZFS_IMPORT_NORMAL;
2527         uint32_t rewind_policy = ZPOOL_NO_REWIND;
2528         boolean_t dryrun = B_FALSE;
2529         boolean_t do_rewind = B_FALSE;
2530         boolean_t xtreme_rewind = B_FALSE;
2531         boolean_t do_scan = B_FALSE;
2532         uint64_t pool_state, txg = -1ULL;
2533         char *cachefile = NULL;
2534         importargs_t idata = { 0 };
2535         char *endptr;
2536
2537         /* check options */
2538         while ((c = getopt(argc, argv, ":aCc:d:DEfFlmnNo:R:stT:VX")) != -1) {
2539                 switch (c) {
2540                 case 'a':
2541                         do_all = B_TRUE;
2542                         break;
2543                 case 'c':
2544                         cachefile = optarg;
2545                         break;
2546                 case 'd':
2547                         if (searchdirs == NULL) {
2548                                 searchdirs = safe_malloc(sizeof (char *));
2549                         } else {
2550                                 char **tmp = safe_malloc((nsearch + 1) *
2551                                     sizeof (char *));
2552                                 bcopy(searchdirs, tmp, nsearch *
2553                                     sizeof (char *));
2554                                 free(searchdirs);
2555                                 searchdirs = tmp;
2556                         }
2557                         searchdirs[nsearch++] = optarg;
2558                         break;
2559                 case 'D':
2560                         do_destroyed = B_TRUE;
2561                         break;
2562                 case 'f':
2563                         flags |= ZFS_IMPORT_ANY_HOST;
2564                         break;
2565                 case 'F':
2566                         do_rewind = B_TRUE;
2567                         break;
2568                 case 'l':
2569                         flags |= ZFS_IMPORT_LOAD_KEYS;
2570                         break;
2571                 case 'm':
2572                         flags |= ZFS_IMPORT_MISSING_LOG;
2573                         break;
2574                 case 'n':
2575                         dryrun = B_TRUE;
2576                         break;
2577                 case 'N':
2578                         flags |= ZFS_IMPORT_ONLY;
2579                         break;
2580                 case 'o':
2581                         if ((propval = strchr(optarg, '=')) != NULL) {
2582                                 *propval = '\0';
2583                                 propval++;
2584                                 if (add_prop_list(optarg, propval,
2585                                     &props, B_TRUE))
2586                                         goto error;
2587                         } else {
2588                                 mntopts = optarg;
2589                         }
2590                         break;
2591                 case 'R':
2592                         if (add_prop_list(zpool_prop_to_name(
2593                             ZPOOL_PROP_ALTROOT), optarg, &props, B_TRUE))
2594                                 goto error;
2595                         if (add_prop_list_default(zpool_prop_to_name(
2596                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
2597                                 goto error;
2598                         break;
2599                 case 's':
2600                         do_scan = B_TRUE;
2601                         break;
2602                 case 't':
2603                         flags |= ZFS_IMPORT_TEMP_NAME;
2604                         if (add_prop_list_default(zpool_prop_to_name(
2605                             ZPOOL_PROP_CACHEFILE), "none", &props, B_TRUE))
2606                                 goto error;
2607                         break;
2608
2609                 case 'T':
2610                         errno = 0;
2611                         txg = strtoull(optarg, &endptr, 0);
2612                         if (errno != 0 || *endptr != '\0') {
2613                                 (void) fprintf(stderr,
2614                                     gettext("invalid txg value\n"));
2615                                 usage(B_FALSE);
2616                         }
2617                         rewind_policy = ZPOOL_DO_REWIND | ZPOOL_EXTREME_REWIND;
2618                         break;
2619                 case 'V':
2620                         flags |= ZFS_IMPORT_VERBATIM;
2621                         break;
2622                 case 'X':
2623                         xtreme_rewind = B_TRUE;
2624                         break;
2625                 case ':':
2626                         (void) fprintf(stderr, gettext("missing argument for "
2627                             "'%c' option\n"), optopt);
2628                         usage(B_FALSE);
2629                         break;
2630                 case '?':
2631                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
2632                             optopt);
2633                         usage(B_FALSE);
2634                 }
2635         }
2636
2637         argc -= optind;
2638         argv += optind;
2639
2640         if (cachefile && nsearch != 0) {
2641                 (void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
2642                 usage(B_FALSE);
2643         }
2644
2645         if ((flags & ZFS_IMPORT_LOAD_KEYS) && (flags & ZFS_IMPORT_ONLY)) {
2646                 (void) fprintf(stderr, gettext("-l is incompatible with -N\n"));
2647                 usage(B_FALSE);
2648         }
2649
2650         if ((flags & ZFS_IMPORT_LOAD_KEYS) && !do_all && argc == 0) {
2651                 (void) fprintf(stderr, gettext("-l is only meaningful during "
2652                     "an import\n"));
2653                 usage(B_FALSE);
2654         }
2655
2656         if ((dryrun || xtreme_rewind) && !do_rewind) {
2657                 (void) fprintf(stderr,
2658                     gettext("-n or -X only meaningful with -F\n"));
2659                 usage(B_FALSE);
2660         }
2661         if (dryrun)
2662                 rewind_policy = ZPOOL_TRY_REWIND;
2663         else if (do_rewind)
2664                 rewind_policy = ZPOOL_DO_REWIND;
2665         if (xtreme_rewind)
2666                 rewind_policy |= ZPOOL_EXTREME_REWIND;
2667
2668         /* In the future, we can capture further policy and include it here */
2669         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
2670             nvlist_add_uint64(policy, ZPOOL_LOAD_REQUEST_TXG, txg) != 0 ||
2671             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
2672             rewind_policy) != 0)
2673                 goto error;
2674
2675         /* check argument count */
2676         if (do_all) {
2677                 if (argc != 0) {
2678                         (void) fprintf(stderr, gettext("too many arguments\n"));
2679                         usage(B_FALSE);
2680                 }
2681         } else {
2682                 if (argc > 2) {
2683                         (void) fprintf(stderr, gettext("too many arguments\n"));
2684                         usage(B_FALSE);
2685                 }
2686         }
2687
2688         /*
2689          * Check for the effective uid.  We do this explicitly here because
2690          * otherwise any attempt to discover pools will silently fail.
2691          */
2692         if (argc == 0 && geteuid() != 0) {
2693                 (void) fprintf(stderr, gettext("cannot "
2694                     "discover pools: permission denied\n"));
2695                 if (searchdirs != NULL)
2696                         free(searchdirs);
2697
2698                 nvlist_free(props);
2699                 nvlist_free(policy);
2700                 return (1);
2701         }
2702
2703         /*
2704          * Depending on the arguments given, we do one of the following:
2705          *
2706          *      <none>  Iterate through all pools and display information about
2707          *              each one.
2708          *
2709          *      -a      Iterate through all pools and try to import each one.
2710          *
2711          *      <id>    Find the pool that corresponds to the given GUID/pool
2712          *              name and import that one.
2713          *
2714          *      -D      Above options applies only to destroyed pools.
2715          */
2716         if (argc != 0) {
2717                 char *endptr;
2718
2719                 errno = 0;
2720                 searchguid = strtoull(argv[0], &endptr, 10);
2721                 if (errno != 0 || *endptr != '\0') {
2722                         searchname = argv[0];
2723                         searchguid = 0;
2724                 }
2725                 found_config = NULL;
2726
2727                 /*
2728                  * User specified a name or guid.  Ensure it's unique.
2729                  */
2730                 idata.unique = B_TRUE;
2731         }
2732
2733         /*
2734          * Check the environment for the preferred search path.
2735          */
2736         if ((searchdirs == NULL) && (env = getenv("ZPOOL_IMPORT_PATH"))) {
2737                 char *dir;
2738
2739                 envdup = strdup(env);
2740
2741                 dir = strtok(envdup, ":");
2742                 while (dir != NULL) {
2743                         if (searchdirs == NULL) {
2744                                 searchdirs = safe_malloc(sizeof (char *));
2745                         } else {
2746                                 char **tmp = safe_malloc((nsearch + 1) *
2747                                     sizeof (char *));
2748                                 bcopy(searchdirs, tmp, nsearch *
2749                                     sizeof (char *));
2750                                 free(searchdirs);
2751                                 searchdirs = tmp;
2752                         }
2753                         searchdirs[nsearch++] = dir;
2754                         dir = strtok(NULL, ":");
2755                 }
2756         }
2757
2758         idata.path = searchdirs;
2759         idata.paths = nsearch;
2760         idata.poolname = searchname;
2761         idata.guid = searchguid;
2762         idata.cachefile = cachefile;
2763         idata.scan = do_scan;
2764         idata.policy = policy;
2765
2766         pools = zpool_search_import(g_zfs, &idata);
2767
2768         if (pools != NULL && idata.exists &&
2769             (argc == 1 || strcmp(argv[0], argv[1]) == 0)) {
2770                 (void) fprintf(stderr, gettext("cannot import '%s': "
2771                     "a pool with that name already exists\n"),
2772                     argv[0]);
2773                 (void) fprintf(stderr, gettext("use the form '%s "
2774                     "<pool | id> <newpool>' to give it a new name\n"),
2775                     "zpool import");
2776                 err = 1;
2777         } else if (pools == NULL && idata.exists) {
2778                 (void) fprintf(stderr, gettext("cannot import '%s': "
2779                     "a pool with that name is already created/imported,\n"),
2780                     argv[0]);
2781                 (void) fprintf(stderr, gettext("and no additional pools "
2782                     "with that name were found\n"));
2783                 err = 1;
2784         } else if (pools == NULL) {
2785                 if (argc != 0) {
2786                         (void) fprintf(stderr, gettext("cannot import '%s': "
2787                             "no such pool available\n"), argv[0]);
2788                 }
2789                 err = 1;
2790         }
2791
2792         if (err == 1) {
2793                 if (searchdirs != NULL)
2794                         free(searchdirs);
2795                 if (envdup != NULL)
2796                         free(envdup);
2797                 nvlist_free(policy);
2798                 nvlist_free(pools);
2799                 nvlist_free(props);
2800                 return (1);
2801         }
2802
2803         /*
2804          * At this point we have a list of import candidate configs. Even if
2805          * we were searching by pool name or guid, we still need to
2806          * post-process the list to deal with pool state and possible
2807          * duplicate names.
2808          */
2809         err = 0;
2810         elem = NULL;
2811         first = B_TRUE;
2812         while ((elem = nvlist_next_nvpair(pools, elem)) != NULL) {
2813
2814                 verify(nvpair_value_nvlist(elem, &config) == 0);
2815
2816                 verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE,
2817                     &pool_state) == 0);
2818                 if (!do_destroyed && pool_state == POOL_STATE_DESTROYED)
2819                         continue;
2820                 if (do_destroyed && pool_state != POOL_STATE_DESTROYED)
2821                         continue;
2822
2823                 verify(nvlist_add_nvlist(config, ZPOOL_LOAD_POLICY,
2824                     policy) == 0);
2825
2826                 if (argc == 0) {
2827                         if (first)
2828                                 first = B_FALSE;
2829                         else if (!do_all)
2830                                 (void) printf("\n");
2831
2832                         if (do_all) {
2833                                 err |= do_import(config, NULL, mntopts,
2834                                     props, flags);
2835                         } else {
2836                                 show_import(config);
2837                         }
2838                 } else if (searchname != NULL) {
2839                         char *name;
2840
2841                         /*
2842                          * We are searching for a pool based on name.
2843                          */
2844                         verify(nvlist_lookup_string(config,
2845                             ZPOOL_CONFIG_POOL_NAME, &name) == 0);
2846
2847                         if (strcmp(name, searchname) == 0) {
2848                                 if (found_config != NULL) {
2849                                         (void) fprintf(stderr, gettext(
2850                                             "cannot import '%s': more than "
2851                                             "one matching pool\n"), searchname);
2852                                         (void) fprintf(stderr, gettext(
2853                                             "import by numeric ID instead\n"));
2854                                         err = B_TRUE;
2855                                 }
2856                                 found_config = config;
2857                         }
2858                 } else {
2859                         uint64_t guid;
2860
2861                         /*
2862                          * Search for a pool by guid.
2863                          */
2864                         verify(nvlist_lookup_uint64(config,
2865                             ZPOOL_CONFIG_POOL_GUID, &guid) == 0);
2866
2867                         if (guid == searchguid)
2868                                 found_config = config;
2869                 }
2870         }
2871
2872         /*
2873          * If we were searching for a specific pool, verify that we found a
2874          * pool, and then do the import.
2875          */
2876         if (argc != 0 && err == 0) {
2877                 if (found_config == NULL) {
2878                         (void) fprintf(stderr, gettext("cannot import '%s': "
2879                             "no such pool available\n"), argv[0]);
2880                         err = B_TRUE;
2881                 } else {
2882                         err |= do_import(found_config, argc == 1 ? NULL :
2883                             argv[1], mntopts, props, flags);
2884                 }
2885         }
2886
2887         /*
2888          * If we were just looking for pools, report an error if none were
2889          * found.
2890          */
2891         if (argc == 0 && first)
2892                 (void) fprintf(stderr,
2893                     gettext("no pools available to import\n"));
2894
2895 error:
2896         nvlist_free(props);
2897         nvlist_free(pools);
2898         nvlist_free(policy);
2899         if (searchdirs != NULL)
2900                 free(searchdirs);
2901         if (envdup != NULL)
2902                 free(envdup);
2903
2904         return (err ? 1 : 0);
2905 }
2906
2907 /*
2908  * zpool sync [-f] [pool] ...
2909  *
2910  * -f (undocumented) force uberblock (and config including zpool cache file)
2911  *    update.
2912  *
2913  * Sync the specified pool(s).
2914  * Without arguments "zpool sync" will sync all pools.
2915  * This command initiates TXG sync(s) and will return after the TXG(s) commit.
2916  *
2917  */
2918 static int
2919 zpool_do_sync(int argc, char **argv)
2920 {
2921         int ret;
2922         boolean_t force = B_FALSE;
2923
2924         /* check options */
2925         while ((ret  = getopt(argc, argv, "f")) != -1) {
2926                 switch (ret) {
2927                 case 'f':
2928                         force = B_TRUE;
2929                         break;
2930                 case '?':
2931                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
2932                             optopt);
2933                         usage(B_FALSE);
2934                 }
2935         }
2936
2937         argc -= optind;
2938         argv += optind;
2939
2940         /* if argc == 0 we will execute zpool_sync_one on all pools */
2941         ret = for_each_pool(argc, argv, B_FALSE, NULL, zpool_sync_one, &force);
2942
2943         return (ret);
2944 }
2945
2946 typedef struct iostat_cbdata {
2947         uint64_t cb_flags;
2948         int cb_name_flags;
2949         int cb_namewidth;
2950         int cb_iteration;
2951         char **cb_vdev_names; /* Only show these vdevs */
2952         unsigned int cb_vdev_names_count;
2953         boolean_t cb_verbose;
2954         boolean_t cb_literal;
2955         boolean_t cb_scripted;
2956         zpool_list_t *cb_list;
2957         vdev_cmd_data_list_t *vcdl;
2958 } iostat_cbdata_t;
2959
2960 /*  iostat labels */
2961 typedef struct name_and_columns {
2962         const char *name;       /* Column name */
2963         unsigned int columns;   /* Center name to this number of columns */
2964 } name_and_columns_t;
2965
2966 #define IOSTAT_MAX_LABELS       11      /* Max number of labels on one line */
2967
2968 static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] =
2969 {
2970         [IOS_DEFAULT] = {{"capacity", 2}, {"operations", 2}, {"bandwidth", 2},
2971             {NULL}},
2972         [IOS_LATENCY] = {{"total_wait", 2}, {"disk_wait", 2}, {"syncq_wait", 2},
2973             {"asyncq_wait", 2}, {"scrub"}},
2974         [IOS_QUEUES] = {{"syncq_read", 2}, {"syncq_write", 2},
2975             {"asyncq_read", 2}, {"asyncq_write", 2}, {"scrubq_read", 2},
2976             {NULL}},
2977         [IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2},
2978             {"sync_queue", 2}, {"async_queue", 2}, {NULL}},
2979         [IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2},
2980             {"async_read", 2}, {"async_write", 2}, {"scrub", 2}, {NULL}},
2981
2982 };
2983
2984 /* Shorthand - if "columns" field not set, default to 1 column */
2985 static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] =
2986 {
2987         [IOS_DEFAULT] = {{"alloc"}, {"free"}, {"read"}, {"write"}, {"read"},
2988             {"write"}, {NULL}},
2989         [IOS_LATENCY] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
2990             {"write"}, {"read"}, {"write"}, {"wait"}, {NULL}},
2991         [IOS_QUEUES] = {{"pend"}, {"activ"}, {"pend"}, {"activ"}, {"pend"},
2992             {"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"}, {NULL}},
2993         [IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"},
2994             {"write"}, {"read"}, {"write"}, {"scrub"}, {NULL}},
2995         [IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"},
2996             {"ind"}, {"agg"}, {"ind"}, {"agg"}, {NULL}},
2997 };
2998
2999 static const char *histo_to_title[] = {
3000         [IOS_L_HISTO] = "latency",
3001         [IOS_RQ_HISTO] = "req_size",
3002 };
3003
3004 /*
3005  * Return the number of labels in a null-terminated name_and_columns_t
3006  * array.
3007  *
3008  */
3009 static unsigned int
3010 label_array_len(const name_and_columns_t *labels)
3011 {
3012         int i = 0;
3013
3014         while (labels[i].name)
3015                 i++;
3016
3017         return (i);
3018 }
3019
3020 /*
3021  * Return the number of strings in a null-terminated string array.
3022  * For example:
3023  *
3024  *     const char foo[] = {"bar", "baz", NULL}
3025  *
3026  * returns 2
3027  */
3028 static uint64_t
3029 str_array_len(const char *array[])
3030 {
3031         uint64_t i = 0;
3032         while (array[i])
3033                 i++;
3034
3035         return (i);
3036 }
3037
3038
3039 /*
3040  * Return a default column width for default/latency/queue columns. This does
3041  * not include histograms, which have their columns autosized.
3042  */
3043 static unsigned int
3044 default_column_width(iostat_cbdata_t *cb, enum iostat_type type)
3045 {
3046         unsigned long column_width = 5; /* Normal niceprint */
3047         static unsigned long widths[] = {
3048                 /*
3049                  * Choose some sane default column sizes for printing the
3050                  * raw numbers.
3051                  */
3052                 [IOS_DEFAULT] = 15, /* 1PB capacity */
3053                 [IOS_LATENCY] = 10, /* 1B ns = 10sec */
3054                 [IOS_QUEUES] = 6,   /* 1M queue entries */
3055         };
3056
3057         if (cb->cb_literal)
3058                 column_width = widths[type];
3059
3060         return (column_width);
3061 }
3062
3063 /*
3064  * Print the column labels, i.e:
3065  *
3066  *   capacity     operations     bandwidth
3067  * alloc   free   read  write   read  write  ...
3068  *
3069  * If force_column_width is set, use it for the column width.  If not set, use
3070  * the default column width.
3071  */
3072 void
3073 print_iostat_labels(iostat_cbdata_t *cb, unsigned int force_column_width,
3074     const name_and_columns_t labels[][IOSTAT_MAX_LABELS])
3075 {
3076         int i, idx, s;
3077         unsigned int text_start, rw_column_width, spaces_to_end;
3078         uint64_t flags = cb->cb_flags;
3079         uint64_t f;
3080         unsigned int column_width = force_column_width;
3081
3082         /* For each bit set in flags */
3083         for (f = flags; f; f &= ~(1ULL << idx)) {
3084                 idx = lowbit64(f) - 1;
3085                 if (!force_column_width)
3086                         column_width = default_column_width(cb, idx);
3087                 /* Print our top labels centered over "read  write" label. */
3088                 for (i = 0; i < label_array_len(labels[idx]); i++) {
3089                         const char *name = labels[idx][i].name;
3090                         /*
3091                          * We treat labels[][].columns == 0 as shorthand
3092                          * for one column.  It makes writing out the label
3093                          * tables more concise.
3094                          */
3095                         unsigned int columns = MAX(1, labels[idx][i].columns);
3096                         unsigned int slen = strlen(name);
3097
3098                         rw_column_width = (column_width * columns) +
3099                             (2 * (columns - 1));
3100
3101                         text_start = (int)((rw_column_width)/columns -
3102                             slen/columns);
3103
3104                         printf("  ");   /* Two spaces between columns */
3105
3106                         /* Space from beginning of column to label */
3107                         for (s = 0; s < text_start; s++)
3108                                 printf(" ");
3109
3110                         printf("%s", name);
3111
3112                         /* Print space after label to end of column */
3113                         spaces_to_end = rw_column_width - text_start - slen;
3114                         for (s = 0; s < spaces_to_end; s++)
3115                                 printf(" ");
3116
3117                 }
3118         }
3119 }
3120
3121
3122 /*
3123  * print_cmd_columns - Print custom column titles from -c
3124  *
3125  * If the user specified the "zpool status|iostat -c" then print their custom
3126  * column titles in the header.  For example, print_cmd_columns() would print
3127  * the "  col1  col2" part of this:
3128  *
3129  * $ zpool iostat -vc 'echo col1=val1; echo col2=val2'
3130  * ...
3131  *            capacity     operations     bandwidth
3132  * pool        alloc   free   read  write   read  write  col1  col2
3133  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
3134  * mypool       269K  1008M      0      0    107    946
3135  *   mirror     269K  1008M      0      0    107    946
3136  *     sdb         -      -      0      0    102    473  val1  val2
3137  *     sdc         -      -      0      0      5    473  val1  val2
3138  * ----------  -----  -----  -----  -----  -----  -----  ----  ----
3139  */
3140 void
3141 print_cmd_columns(vdev_cmd_data_list_t *vcdl, int use_dashes)
3142 {
3143         int i, j;
3144         vdev_cmd_data_t *data = &vcdl->data[0];
3145
3146         if (vcdl->count == 0 || data == NULL)
3147                 return;
3148
3149         /*
3150          * Each vdev cmd should have the same column names unless the user did
3151          * something weird with their cmd.  Just take the column names from the
3152          * first vdev and assume it works for all of them.
3153          */
3154         for (i = 0; i < vcdl->uniq_cols_cnt; i++) {
3155                 printf("  ");
3156                 if (use_dashes) {
3157                         for (j = 0; j < vcdl->uniq_cols_width[i]; j++)
3158                                 printf("-");
3159                 } else {
3160                         printf("%*s", vcdl->uniq_cols_width[i],
3161                             vcdl->uniq_cols[i]);
3162                 }
3163         }
3164 }
3165
3166
3167 /*
3168  * Utility function to print out a line of dashes like:
3169  *
3170  *      --------------------------------  -----  -----  -----  -----  -----
3171  *
3172  * ...or a dashed named-row line like:
3173  *
3174  *      logs                                  -      -      -      -      -
3175  *
3176  * @cb:                         iostat data
3177  *
3178  * @force_column_width          If non-zero, use the value as the column width.
3179  *                              Otherwise use the default column widths.
3180  *
3181  * @name:                       Print a dashed named-row line starting
3182  *                              with @name.  Otherwise, print a regular
3183  *                              dashed line.
3184  */
3185 static void
3186 print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width,
3187     const char *name)
3188 {
3189         int i;
3190         unsigned int namewidth;
3191         uint64_t flags = cb->cb_flags;
3192         uint64_t f;
3193         int idx;
3194         const name_and_columns_t *labels;
3195         const char *title;
3196
3197
3198         if (cb->cb_flags & IOS_ANYHISTO_M) {
3199                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3200         } else if (cb->cb_vdev_names_count) {
3201                 title = "vdev";
3202         } else  {
3203                 title = "pool";
3204         }
3205
3206         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3207             name ? strlen(name) : 0);
3208
3209
3210         if (name) {
3211                 printf("%-*s", namewidth, name);
3212         } else {
3213                 for (i = 0; i < namewidth; i++)
3214                         (void) printf("-");
3215         }
3216
3217         /* For each bit in flags */
3218         for (f = flags; f; f &= ~(1ULL << idx)) {
3219                 unsigned int column_width;
3220                 idx = lowbit64(f) - 1;
3221                 if (force_column_width)
3222                         column_width = force_column_width;
3223                 else
3224                         column_width = default_column_width(cb, idx);
3225
3226                 labels = iostat_bottom_labels[idx];
3227                 for (i = 0; i < label_array_len(labels); i++) {
3228                         if (name)
3229                                 printf("  %*s-", column_width - 1, " ");
3230                         else
3231                                 printf("  %.*s", column_width,
3232                                     "--------------------");
3233                 }
3234         }
3235 }
3236
3237
3238 static void
3239 print_iostat_separator_impl(iostat_cbdata_t *cb,
3240     unsigned int force_column_width)
3241 {
3242         print_iostat_dashes(cb, force_column_width, NULL);
3243 }
3244
3245 static void
3246 print_iostat_separator(iostat_cbdata_t *cb)
3247 {
3248         print_iostat_separator_impl(cb, 0);
3249 }
3250
3251 static void
3252 print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width,
3253     const char *histo_vdev_name)
3254 {
3255         unsigned int namewidth;
3256         const char *title;
3257
3258         if (cb->cb_flags & IOS_ANYHISTO_M) {
3259                 title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)];
3260         } else if (cb->cb_vdev_names_count) {
3261                 title = "vdev";
3262         } else  {
3263                 title = "pool";
3264         }
3265
3266         namewidth = MAX(MAX(strlen(title), cb->cb_namewidth),
3267             histo_vdev_name ? strlen(histo_vdev_name) : 0);
3268
3269         if (histo_vdev_name)
3270                 printf("%-*s", namewidth, histo_vdev_name);
3271         else
3272                 printf("%*s", namewidth, "");
3273
3274
3275         print_iostat_labels(cb, force_column_width, iostat_top_labels);
3276         printf("\n");
3277
3278         printf("%-*s", namewidth, title);
3279
3280         print_iostat_labels(cb, force_column_width, iostat_bottom_labels);
3281         if (cb->vcdl != NULL)
3282                 print_cmd_columns(cb->vcdl, 0);
3283
3284         printf("\n");
3285
3286         print_iostat_separator_impl(cb, force_column_width);
3287
3288         if (cb->vcdl != NULL)
3289                 print_cmd_columns(cb->vcdl, 1);
3290
3291         printf("\n");
3292 }
3293
3294 static void
3295 print_iostat_header(iostat_cbdata_t *cb)
3296 {
3297         print_iostat_header_impl(cb, 0, NULL);
3298 }
3299
3300
3301 /*
3302  * Display a single statistic.
3303  */
3304 static void
3305 print_one_stat(uint64_t value, enum zfs_nicenum_format format,
3306     unsigned int column_size, boolean_t scripted)
3307 {
3308         char buf[64];
3309
3310         zfs_nicenum_format(value, buf, sizeof (buf), format);
3311
3312         if (scripted)
3313                 printf("\t%s", buf);
3314         else
3315                 printf("  %*s", column_size, buf);
3316 }
3317
3318 /*
3319  * Calculate the default vdev stats
3320  *
3321  * Subtract oldvs from newvs, apply a scaling factor, and save the resulting
3322  * stats into calcvs.
3323  */
3324 static void
3325 calc_default_iostats(vdev_stat_t *oldvs, vdev_stat_t *newvs,
3326     vdev_stat_t *calcvs)
3327 {
3328         int i;
3329
3330         memcpy(calcvs, newvs, sizeof (*calcvs));
3331         for (i = 0; i < ARRAY_SIZE(calcvs->vs_ops); i++)
3332                 calcvs->vs_ops[i] = (newvs->vs_ops[i] - oldvs->vs_ops[i]);
3333
3334         for (i = 0; i < ARRAY_SIZE(calcvs->vs_bytes); i++)
3335                 calcvs->vs_bytes[i] = (newvs->vs_bytes[i] - oldvs->vs_bytes[i]);
3336 }
3337
3338 /*
3339  * Internal representation of the extended iostats data.
3340  *
3341  * The extended iostat stats are exported in nvlists as either uint64_t arrays
3342  * or single uint64_t's.  We make both look like arrays to make them easier
3343  * to process.  In order to make single uint64_t's look like arrays, we set
3344  * __data to the stat data, and then set *data = &__data with count = 1.  Then,
3345  * we can just use *data and count.
3346  */
3347 struct stat_array {
3348         uint64_t *data;
3349         uint_t count;   /* Number of entries in data[] */
3350         uint64_t __data; /* Only used when data is a single uint64_t */
3351 };
3352
3353 static uint64_t
3354 stat_histo_max(struct stat_array *nva, unsigned int len)
3355 {
3356         uint64_t max = 0;
3357         int i;
3358         for (i = 0; i < len; i++)
3359                 max = MAX(max, array64_max(nva[i].data, nva[i].count));
3360
3361         return (max);
3362 }
3363
3364 /*
3365  * Helper function to lookup a uint64_t array or uint64_t value and store its
3366  * data as a stat_array.  If the nvpair is a single uint64_t value, then we make
3367  * it look like a one element array to make it easier to process.
3368  */
3369 static int
3370 nvpair64_to_stat_array(nvlist_t *nvl, const char *name,
3371     struct stat_array *nva)
3372 {
3373         nvpair_t *tmp;
3374         int ret;
3375
3376         verify(nvlist_lookup_nvpair(nvl, name, &tmp) == 0);
3377         switch (nvpair_type(tmp)) {
3378         case DATA_TYPE_UINT64_ARRAY:
3379                 ret = nvpair_value_uint64_array(tmp, &nva->data, &nva->count);
3380                 break;
3381         case DATA_TYPE_UINT64:
3382                 ret = nvpair_value_uint64(tmp, &nva->__data);
3383                 nva->data = &nva->__data;
3384                 nva->count = 1;
3385                 break;
3386         default:
3387                 /* Not a uint64_t */
3388                 ret = EINVAL;
3389                 break;
3390         }
3391
3392         return (ret);
3393 }
3394
3395 /*
3396  * Given a list of nvlist names, look up the extended stats in newnv and oldnv,
3397  * subtract them, and return the results in a newly allocated stat_array.
3398  * You must free the returned array after you are done with it with
3399  * free_calc_stats().
3400  *
3401  * Additionally, you can set "oldnv" to NULL if you simply want the newnv
3402  * values.
3403  */
3404 static struct stat_array *
3405 calc_and_alloc_stats_ex(const char **names, unsigned int len, nvlist_t *oldnv,
3406     nvlist_t *newnv)
3407 {
3408         nvlist_t *oldnvx = NULL, *newnvx;
3409         struct stat_array *oldnva, *newnva, *calcnva;
3410         int i, j;
3411         unsigned int alloc_size = (sizeof (struct stat_array)) * len;
3412
3413         /* Extract our extended stats nvlist from the main list */
3414         verify(nvlist_lookup_nvlist(newnv, ZPOOL_CONFIG_VDEV_STATS_EX,
3415             &newnvx) == 0);
3416         if (oldnv) {
3417                 verify(nvlist_lookup_nvlist(oldnv, ZPOOL_CONFIG_VDEV_STATS_EX,
3418                     &oldnvx) == 0);
3419         }
3420
3421         newnva = safe_malloc(alloc_size);
3422         oldnva = safe_malloc(alloc_size);
3423         calcnva = safe_malloc(alloc_size);
3424
3425         for (j = 0; j < len; j++) {
3426                 verify(nvpair64_to_stat_array(newnvx, names[j],
3427                     &newnva[j]) == 0);
3428                 calcnva[j].count = newnva[j].count;
3429                 alloc_size = calcnva[j].count * sizeof (calcnva[j].data[0]);
3430                 calcnva[j].data = safe_malloc(alloc_size);
3431                 memcpy(calcnva[j].data, newnva[j].data, alloc_size);
3432
3433                 if (oldnvx) {
3434                         verify(nvpair64_to_stat_array(oldnvx, names[j],
3435                             &oldnva[j]) == 0);
3436                         for (i = 0; i < oldnva[j].count; i++)
3437                                 calcnva[j].data[i] -= oldnva[j].data[i];
3438                 }
3439         }
3440         free(newnva);
3441         free(oldnva);
3442         return (calcnva);
3443 }
3444
3445 static void
3446 free_calc_stats(struct stat_array *nva, unsigned int len)
3447 {
3448         int i;
3449         for (i = 0; i < len; i++)
3450                 free(nva[i].data);
3451
3452         free(nva);
3453 }
3454
3455 static void
3456 print_iostat_histo(struct stat_array *nva, unsigned int len,
3457     iostat_cbdata_t *cb, unsigned int column_width, unsigned int namewidth,
3458     double scale)
3459 {
3460         int i, j;
3461         char buf[6];
3462         uint64_t val;
3463         enum zfs_nicenum_format format;
3464         unsigned int buckets;
3465         unsigned int start_bucket;
3466
3467         if (cb->cb_literal)
3468                 format = ZFS_NICENUM_RAW;
3469         else
3470                 format = ZFS_NICENUM_1024;
3471
3472         /* All these histos are the same size, so just use nva[0].count */
3473         buckets = nva[0].count;
3474
3475         if (cb->cb_flags & IOS_RQ_HISTO_M) {
3476                 /* Start at 512 - req size should never be lower than this */
3477                 start_bucket = 9;
3478         } else {
3479                 start_bucket = 0;
3480         }
3481
3482         for (j = start_bucket; j < buckets; j++) {
3483                 /* Print histogram bucket label */
3484                 if (cb->cb_flags & IOS_L_HISTO_M) {
3485                         /* Ending range of this bucket */
3486                         val = (1UL << (j + 1)) - 1;
3487                         zfs_nicetime(val, buf, sizeof (buf));
3488                 } else {
3489                         /* Request size (starting range of bucket) */
3490                         val = (1UL << j);
3491                         zfs_nicenum(val, buf, sizeof (buf));
3492                 }
3493
3494                 if (cb->cb_scripted)
3495                         printf("%llu", (u_longlong_t)val);
3496                 else
3497                         printf("%-*s", namewidth, buf);
3498
3499                 /* Print the values on the line */
3500                 for (i = 0; i < len; i++) {
3501                         print_one_stat(nva[i].data[j] * scale, format,
3502                             column_width, cb->cb_scripted);
3503                 }
3504                 printf("\n");
3505         }
3506 }
3507
3508 static void
3509 print_solid_separator(unsigned int length)
3510 {
3511         while (length--)
3512                 printf("-");
3513         printf("\n");
3514 }
3515
3516 static void
3517 print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv,
3518     nvlist_t *newnv, double scale, const char *name)
3519 {
3520         unsigned int column_width;
3521         unsigned int namewidth;
3522         unsigned int entire_width;
3523         enum iostat_type type;
3524         struct stat_array *nva;
3525         const char **names;
3526         unsigned int names_len;
3527
3528         /* What type of histo are we? */
3529         type = IOS_HISTO_IDX(cb->cb_flags);
3530
3531         /* Get NULL-terminated array of nvlist names for our histo */
3532         names = vsx_type_to_nvlist[type];
3533         names_len = str_array_len(names); /* num of names */
3534
3535         nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv);
3536
3537         if (cb->cb_literal) {
3538                 column_width = MAX(5,
3539                     (unsigned int) log10(stat_histo_max(nva, names_len)) + 1);
3540         } else {
3541                 column_width = 5;
3542         }
3543
3544         namewidth = MAX(cb->cb_namewidth,
3545             strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]));
3546
3547         /*
3548          * Calculate the entire line width of what we're printing.  The
3549          * +2 is for the two spaces between columns:
3550          */
3551         /*       read  write                            */
3552         /*      -----  -----                            */
3553         /*      |___|  <---------- column_width         */
3554         /*                                              */
3555         /*      |__________|  <--- entire_width         */
3556         /*                                              */
3557         entire_width = namewidth + (column_width + 2) *
3558             label_array_len(iostat_bottom_labels[type]);
3559
3560         if (cb->cb_scripted)
3561                 printf("%s\n", name);
3562         else
3563                 print_iostat_header_impl(cb, column_width, name);
3564
3565         print_iostat_histo(nva, names_len, cb, column_width,
3566             namewidth, scale);
3567
3568         free_calc_stats(nva, names_len);
3569         if (!cb->cb_scripted)
3570                 print_solid_separator(entire_width);
3571 }
3572
3573 /*
3574  * Calculate the average latency of a power-of-two latency histogram
3575  */
3576 static uint64_t
3577 single_histo_average(uint64_t *histo, unsigned int buckets)
3578 {
3579         int i;
3580         uint64_t count = 0, total = 0;
3581
3582         for (i = 0; i < buckets; i++) {
3583                 /*
3584                  * Our buckets are power-of-two latency ranges.  Use the
3585                  * midpoint latency of each bucket to calculate the average.
3586                  * For example:
3587                  *
3588                  * Bucket          Midpoint
3589                  * 8ns-15ns:       12ns
3590                  * 16ns-31ns:      24ns
3591                  * ...
3592                  */
3593                 if (histo[i] != 0) {
3594                         total += histo[i] * (((1UL << i) + ((1UL << i)/2)));
3595                         count += histo[i];
3596                 }
3597         }
3598
3599         /* Prevent divide by zero */
3600         return (count == 0 ? 0 : total / count);
3601 }
3602
3603 static void
3604 print_iostat_queues(iostat_cbdata_t *cb, nvlist_t *oldnv,
3605     nvlist_t *newnv, double scale)
3606 {
3607         int i;
3608         uint64_t val;
3609         const char *names[] = {
3610                 ZPOOL_CONFIG_VDEV_SYNC_R_PEND_QUEUE,
3611                 ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE,
3612                 ZPOOL_CONFIG_VDEV_SYNC_W_PEND_QUEUE,
3613                 ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE,
3614                 ZPOOL_CONFIG_VDEV_ASYNC_R_PEND_QUEUE,
3615                 ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE,
3616                 ZPOOL_CONFIG_VDEV_ASYNC_W_PEND_QUEUE,
3617                 ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE,
3618                 ZPOOL_CONFIG_VDEV_SCRUB_PEND_QUEUE,
3619                 ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE,
3620         };
3621
3622         struct stat_array *nva;
3623
3624         unsigned int column_width = default_column_width(cb, IOS_QUEUES);
3625         enum zfs_nicenum_format format;
3626
3627         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), NULL, newnv);
3628
3629         if (cb->cb_literal)
3630                 format = ZFS_NICENUM_RAW;
3631         else
3632                 format = ZFS_NICENUM_1024;
3633
3634         for (i = 0; i < ARRAY_SIZE(names); i++) {
3635                 val = nva[i].data[0] * scale;
3636                 print_one_stat(val, format, column_width, cb->cb_scripted);
3637         }
3638
3639         free_calc_stats(nva, ARRAY_SIZE(names));
3640 }
3641
3642 static void
3643 print_iostat_latency(iostat_cbdata_t *cb, nvlist_t *oldnv,
3644     nvlist_t *newnv, double scale)
3645 {
3646         int i;
3647         uint64_t val;
3648         const char *names[] = {
3649                 ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO,
3650                 ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO,
3651                 ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO,
3652                 ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO,
3653                 ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO,
3654                 ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO,
3655                 ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO,
3656                 ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO,
3657                 ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO,
3658         };
3659         struct stat_array *nva;
3660
3661         unsigned int column_width = default_column_width(cb, IOS_LATENCY);
3662         enum zfs_nicenum_format format;
3663
3664         nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv);
3665
3666         if (cb->cb_literal)
3667                 format = ZFS_NICENUM_RAWTIME;
3668         else
3669                 format = ZFS_NICENUM_TIME;
3670
3671         /* Print our avg latencies on the line */
3672         for (i = 0; i < ARRAY_SIZE(names); i++) {
3673                 /* Compute average latency for a latency histo */
3674                 val = single_histo_average(nva[i].data, nva[i].count) * scale;
3675                 print_one_stat(val, format, column_width, cb->cb_scripted);
3676         }
3677         free_calc_stats(nva, ARRAY_SIZE(names));
3678 }
3679
3680 /*
3681  * Print default statistics (capacity/operations/bandwidth)
3682  */
3683 static void
3684 print_iostat_default(vdev_stat_t *vs, iostat_cbdata_t *cb, double scale)
3685 {
3686         unsigned int column_width = default_column_width(cb, IOS_DEFAULT);
3687         enum zfs_nicenum_format format;
3688         char na;        /* char to print for "not applicable" values */
3689
3690         if (cb->cb_literal) {
3691                 format = ZFS_NICENUM_RAW;
3692                 na = '0';
3693         } else {
3694                 format = ZFS_NICENUM_1024;
3695                 na = '-';
3696         }
3697
3698         /* only toplevel vdevs have capacity stats */
3699         if (vs->vs_space == 0) {
3700                 if (cb->cb_scripted)
3701                         printf("\t%c\t%c", na, na);
3702                 else
3703                         printf("  %*c  %*c", column_width, na, column_width,
3704                             na);
3705         } else {
3706                 print_one_stat(vs->vs_alloc, format, column_width,
3707                     cb->cb_scripted);
3708                 print_one_stat(vs->vs_space - vs->vs_alloc, format,
3709                     column_width, cb->cb_scripted);
3710         }
3711
3712         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_READ] * scale),
3713             format, column_width, cb->cb_scripted);
3714         print_one_stat((uint64_t)(vs->vs_ops[ZIO_TYPE_WRITE] * scale),
3715             format, column_width, cb->cb_scripted);
3716         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_READ] * scale),
3717             format, column_width, cb->cb_scripted);
3718         print_one_stat((uint64_t)(vs->vs_bytes[ZIO_TYPE_WRITE] * scale),
3719             format, column_width, cb->cb_scripted);
3720 }
3721
3722 /*
3723  * Print out all the statistics for the given vdev.  This can either be the
3724  * toplevel configuration, or called recursively.  If 'name' is NULL, then this
3725  * is a verbose output, and we don't want to display the toplevel pool stats.
3726  *
3727  * Returns the number of stat lines printed.
3728  */
3729 unsigned int
3730 print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv,
3731     nvlist_t *newnv, iostat_cbdata_t *cb, int depth)
3732 {
3733         nvlist_t **oldchild, **newchild;
3734         uint_t c, children, oldchildren;
3735         vdev_stat_t *oldvs, *newvs, *calcvs;
3736         vdev_stat_t zerovs = { 0 };
3737         char *vname;
3738         int i;
3739         int ret = 0;
3740         uint64_t tdelta;
3741         double scale;
3742
3743         calcvs = safe_malloc(sizeof (*calcvs));
3744
3745         if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
3746                 return (ret);
3747
3748         if (oldnv != NULL) {
3749                 verify(nvlist_lookup_uint64_array(oldnv,
3750                     ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&oldvs, &c) == 0);
3751         } else {
3752                 oldvs = &zerovs;
3753         }
3754
3755         /* Do we only want to see a specific vdev? */
3756         for (i = 0; i < cb->cb_vdev_names_count; i++) {
3757                 /* Yes we do.  Is this the vdev? */
3758                 if (strcmp(name, cb->cb_vdev_names[i]) == 0) {
3759                         /*
3760                          * This is our vdev.  Since it is the only vdev we
3761                          * will be displaying, make depth = 0 so that it
3762                          * doesn't get indented.
3763                          */
3764                         depth = 0;
3765                         break;
3766                 }
3767         }
3768
3769         if (cb->cb_vdev_names_count && (i == cb->cb_vdev_names_count)) {
3770                 /* Couldn't match the name */
3771                 goto children;
3772         }
3773
3774
3775         verify(nvlist_lookup_uint64_array(newnv, ZPOOL_CONFIG_VDEV_STATS,
3776             (uint64_t **)&newvs, &c) == 0);
3777
3778         /*
3779          * Print the vdev name unless it's is a histogram.  Histograms
3780          * display the vdev name in the header itself.
3781          */
3782         if (!(cb->cb_flags & IOS_ANYHISTO_M)) {
3783                 if (cb->cb_scripted) {
3784                         printf("%s", name);
3785                 } else {
3786                         if (strlen(name) + depth > cb->cb_namewidth)
3787                                 (void) printf("%*s%s", depth, "", name);
3788                         else
3789                                 (void) printf("%*s%s%*s", depth, "", name,
3790                                     (int)(cb->cb_namewidth - strlen(name) -
3791                                     depth), "");
3792                 }
3793         }
3794
3795         /* Calculate our scaling factor */
3796         tdelta = newvs->vs_timestamp - oldvs->vs_timestamp;
3797         if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) {
3798                 /*
3799                  * If we specify printing histograms with no time interval, then
3800                  * print the histogram numbers over the entire lifetime of the
3801                  * vdev.
3802                  */
3803                 scale = 1;
3804         } else {
3805                 if (tdelta == 0)
3806                         scale = 1.0;
3807                 else
3808                         scale = (double)NANOSEC / tdelta;
3809         }
3810
3811         if (cb->cb_flags & IOS_DEFAULT_M) {
3812                 calc_default_iostats(oldvs, newvs, calcvs);
3813                 print_iostat_default(calcvs, cb, scale);
3814         }
3815         if (cb->cb_flags & IOS_LATENCY_M)
3816                 print_iostat_latency(cb, oldnv, newnv, scale);
3817         if (cb->cb_flags & IOS_QUEUES_M)
3818                 print_iostat_queues(cb, oldnv, newnv, scale);
3819         if (cb->cb_flags & IOS_ANYHISTO_M) {
3820                 printf("\n");
3821                 print_iostat_histos(cb, oldnv, newnv, scale, name);
3822         }
3823
3824         if (cb->vcdl != NULL) {
3825                 char *path;
3826                 if (nvlist_lookup_string(newnv, ZPOOL_CONFIG_PATH,
3827                     &path) == 0) {
3828                         printf("  ");
3829                         zpool_print_cmd(cb->vcdl, zpool_get_name(zhp), path);
3830                 }
3831         }
3832
3833         if (!(cb->cb_flags & IOS_ANYHISTO_M))
3834                 printf("\n");
3835
3836         ret++;
3837
3838 children:
3839
3840         free(calcvs);
3841
3842         if (!cb->cb_verbose)
3843                 return (ret);
3844
3845         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_CHILDREN,
3846             &newchild, &children) != 0)
3847                 return (ret);
3848
3849         if (oldnv) {
3850                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_CHILDREN,
3851                     &oldchild, &oldchildren) != 0)
3852                         return (ret);
3853
3854                 children = MIN(oldchildren, children);
3855         }
3856
3857         for (c = 0; c < children; c++) {
3858                 uint64_t ishole = B_FALSE, islog = B_FALSE;
3859
3860                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_HOLE,
3861                     &ishole);
3862
3863                 (void) nvlist_lookup_uint64(newchild[c], ZPOOL_CONFIG_IS_LOG,
3864                     &islog);
3865
3866                 if (ishole || islog)
3867                         continue;
3868
3869                 vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
3870                     cb->cb_name_flags);
3871                 ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c] : NULL,
3872                     newchild[c], cb, depth + 2);
3873                 free(vname);
3874         }
3875
3876         /*
3877          * Log device section
3878          */
3879
3880         if (num_logs(newnv) > 0) {
3881                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
3882                     !cb->cb_vdev_names) {
3883                         print_iostat_dashes(cb, 0, "logs");
3884                 }
3885                 printf("\n");
3886
3887                 for (c = 0; c < children; c++) {
3888                         uint64_t islog = B_FALSE;
3889                         (void) nvlist_lookup_uint64(newchild[c],
3890                             ZPOOL_CONFIG_IS_LOG, &islog);
3891
3892                         if (islog) {
3893                                 vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
3894                                     cb->cb_name_flags);
3895                                 ret += print_vdev_stats(zhp, vname, oldnv ?
3896                                     oldchild[c] : NULL, newchild[c],
3897                                     cb, depth + 2);
3898                                 free(vname);
3899                         }
3900                 }
3901
3902         }
3903
3904         /*
3905          * Include level 2 ARC devices in iostat output
3906          */
3907         if (nvlist_lookup_nvlist_array(newnv, ZPOOL_CONFIG_L2CACHE,
3908             &newchild, &children) != 0)
3909                 return (ret);
3910
3911         if (oldnv) {
3912                 if (nvlist_lookup_nvlist_array(oldnv, ZPOOL_CONFIG_L2CACHE,
3913                     &oldchild, &oldchildren) != 0)
3914                         return (ret);
3915
3916                 children = MIN(oldchildren, children);
3917         }
3918
3919         if (children > 0) {
3920                 if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted &&
3921                     !cb->cb_vdev_names) {
3922                         print_iostat_dashes(cb, 0, "cache");
3923                 }
3924                 printf("\n");
3925
3926                 for (c = 0; c < children; c++) {
3927                         vname = zpool_vdev_name(g_zfs, zhp, newchild[c],
3928                             cb->cb_name_flags);
3929                         ret += print_vdev_stats(zhp, vname, oldnv ? oldchild[c]
3930                             : NULL, newchild[c], cb, depth + 2);
3931                         free(vname);
3932                 }
3933         }
3934
3935         return (ret);
3936 }
3937
3938 static int
3939 refresh_iostat(zpool_handle_t *zhp, void *data)
3940 {
3941         iostat_cbdata_t *cb = data;
3942         boolean_t missing;
3943
3944         /*
3945          * If the pool has disappeared, remove it from the list and continue.
3946          */
3947         if (zpool_refresh_stats(zhp, &missing) != 0)
3948                 return (-1);
3949
3950         if (missing)
3951                 pool_list_remove(cb->cb_list, zhp);
3952
3953         return (0);
3954 }
3955
3956 /*
3957  * Callback to print out the iostats for the given pool.
3958  */
3959 int
3960 print_iostat(zpool_handle_t *zhp, void *data)
3961 {
3962         iostat_cbdata_t *cb = data;
3963         nvlist_t *oldconfig, *newconfig;
3964         nvlist_t *oldnvroot, *newnvroot;
3965         int ret;
3966
3967         newconfig = zpool_get_config(zhp, &oldconfig);
3968
3969         if (cb->cb_iteration == 1)
3970                 oldconfig = NULL;
3971
3972         verify(nvlist_lookup_nvlist(newconfig, ZPOOL_CONFIG_VDEV_TREE,
3973             &newnvroot) == 0);
3974
3975         if (oldconfig == NULL)
3976                 oldnvroot = NULL;
3977         else
3978                 verify(nvlist_lookup_nvlist(oldconfig, ZPOOL_CONFIG_VDEV_TREE,
3979                     &oldnvroot) == 0);
3980
3981         ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot,
3982             cb, 0);
3983         if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) &&
3984             !cb->cb_scripted && cb->cb_verbose && !cb->cb_vdev_names_count) {
3985                 print_iostat_separator(cb);
3986                 if (cb->vcdl != NULL) {
3987                         print_cmd_columns(cb->vcdl, 1);
3988                 }
3989                 printf("\n");
3990         }
3991
3992         return (ret);
3993 }
3994
3995 static int
3996 get_columns(void)
3997 {
3998         struct winsize ws;
3999         int columns = 80;
4000         int error;
4001
4002         if (isatty(STDOUT_FILENO)) {
4003                 error = ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws);
4004                 if (error == 0)
4005                         columns = ws.ws_col;
4006         } else {
4007                 columns = 999;
4008         }
4009
4010         return (columns);
4011 }
4012
4013 int
4014 get_namewidth(zpool_handle_t *zhp, void *data)
4015 {
4016         iostat_cbdata_t *cb = data;
4017         nvlist_t *config, *nvroot;
4018         int columns;
4019
4020         if ((config = zpool_get_config(zhp, NULL)) != NULL) {
4021                 verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4022                     &nvroot) == 0);
4023                 unsigned int poolname_len = strlen(zpool_get_name(zhp));
4024                 if (!cb->cb_verbose)
4025                         cb->cb_namewidth = MAX(poolname_len, cb->cb_namewidth);
4026                 else
4027                         cb->cb_namewidth = MAX(poolname_len,
4028                             max_width(zhp, nvroot, 0, cb->cb_namewidth,
4029                             cb->cb_name_flags));
4030         }
4031         /*
4032          * The width must be at least 10, but may be as large as the
4033          * column width - 42 so that we can still fit in one line.
4034          */
4035         columns = get_columns();
4036
4037         if (cb->cb_namewidth < 10)
4038                 cb->cb_namewidth = 10;
4039         if (cb->cb_namewidth > columns - 42)
4040                 cb->cb_namewidth = columns - 42;
4041
4042         return (0);
4043 }
4044
4045 /*
4046  * Parse the input string, get the 'interval' and 'count' value if there is one.
4047  */
4048 static void
4049 get_interval_count(int *argcp, char **argv, float *iv,
4050     unsigned long *cnt)
4051 {
4052         float interval = 0;
4053         unsigned long count = 0;
4054         int argc = *argcp;
4055
4056         /*
4057          * Determine if the last argument is an integer or a pool name
4058          */
4059         if (argc > 0 && isnumber(argv[argc - 1])) {
4060                 char *end;
4061
4062                 errno = 0;
4063                 interval = strtof(argv[argc - 1], &end);
4064
4065                 if (*end == '\0' && errno == 0) {
4066                         if (interval == 0) {
4067                                 (void) fprintf(stderr, gettext("interval "
4068                                     "cannot be zero\n"));
4069                                 usage(B_FALSE);
4070                         }
4071                         /*
4072                          * Ignore the last parameter
4073                          */
4074                         argc--;
4075                 } else {
4076                         /*
4077                          * If this is not a valid number, just plow on.  The
4078                          * user will get a more informative error message later
4079                          * on.
4080                          */
4081                         interval = 0;
4082                 }
4083         }
4084
4085         /*
4086          * If the last argument is also an integer, then we have both a count
4087          * and an interval.
4088          */
4089         if (argc > 0 && isnumber(argv[argc - 1])) {
4090                 char *end;
4091
4092                 errno = 0;
4093                 count = interval;
4094                 interval = strtof(argv[argc - 1], &end);
4095
4096                 if (*end == '\0' && errno == 0) {
4097                         if (interval == 0) {
4098                                 (void) fprintf(stderr, gettext("interval "
4099                                     "cannot be zero\n"));
4100                                 usage(B_FALSE);
4101                         }
4102
4103                         /*
4104                          * Ignore the last parameter
4105                          */
4106                         argc--;
4107                 } else {
4108                         interval = 0;
4109                 }
4110         }
4111
4112         *iv = interval;
4113         *cnt = count;
4114         *argcp = argc;
4115 }
4116
4117 static void
4118 get_timestamp_arg(char c)
4119 {
4120         if (c == 'u')
4121                 timestamp_fmt = UDATE;
4122         else if (c == 'd')
4123                 timestamp_fmt = DDATE;
4124         else
4125                 usage(B_FALSE);
4126 }
4127
4128 /*
4129  * Return stat flags that are supported by all pools by both the module and
4130  * zpool iostat.  "*data" should be initialized to all 0xFFs before running.
4131  * It will get ANDed down until only the flags that are supported on all pools
4132  * remain.
4133  */
4134 static int
4135 get_stat_flags_cb(zpool_handle_t *zhp, void *data)
4136 {
4137         uint64_t *mask = data;
4138         nvlist_t *config, *nvroot, *nvx;
4139         uint64_t flags = 0;
4140         int i, j;
4141
4142         config = zpool_get_config(zhp, NULL);
4143         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
4144             &nvroot) == 0);
4145
4146         /* Default stats are always supported, but for completeness.. */
4147         if (nvlist_exists(nvroot, ZPOOL_CONFIG_VDEV_STATS))
4148                 flags |= IOS_DEFAULT_M;
4149
4150         /* Get our extended stats nvlist from the main list */
4151         if (nvlist_lookup_nvlist(nvroot, ZPOOL_CONFIG_VDEV_STATS_EX,
4152             &nvx) != 0) {
4153                 /*
4154                  * No extended stats; they're probably running an older
4155                  * module.  No big deal, we support that too.
4156                  */
4157                 goto end;
4158         }
4159
4160         /* For each extended stat, make sure all its nvpairs are supported */
4161         for (j = 0; j < ARRAY_SIZE(vsx_type_to_nvlist); j++) {
4162                 if (!vsx_type_to_nvlist[j][0])
4163                         continue;
4164
4165                 /* Start off by assuming the flag is supported, then check */
4166                 flags |= (1ULL << j);
4167                 for (i = 0; vsx_type_to_nvlist[j][i]; i++) {
4168                         if (!nvlist_exists(nvx, vsx_type_to_nvlist[j][i])) {
4169                                 /* flag isn't supported */
4170                                 flags = flags & ~(1ULL  << j);
4171                                 break;
4172                         }
4173                 }
4174         }
4175 end:
4176         *mask = *mask & flags;
4177         return (0);
4178 }
4179
4180 /*
4181  * Return a bitmask of stats that are supported on all pools by both the module
4182  * and zpool iostat.
4183  */
4184 static uint64_t
4185 get_stat_flags(zpool_list_t *list)
4186 {
4187         uint64_t mask = -1;
4188
4189         /*
4190          * get_stat_flags_cb() will lop off bits from "mask" until only the
4191          * flags that are supported on all pools remain.
4192          */
4193         pool_list_iter(list, B_FALSE, get_stat_flags_cb, &mask);
4194         return (mask);
4195 }
4196
4197 /*
4198  * Return 1 if cb_data->cb_vdev_names[0] is this vdev's name, 0 otherwise.
4199  */
4200 static int
4201 is_vdev_cb(zpool_handle_t *zhp, nvlist_t *nv, void *cb_data)
4202 {
4203         iostat_cbdata_t *cb = cb_data;
4204         char *name = NULL;
4205         int ret = 0;
4206
4207         name = zpool_vdev_name(g_zfs, zhp, nv, cb->cb_name_flags);
4208
4209         if (strcmp(name, cb->cb_vdev_names[0]) == 0)
4210                 ret = 1; /* match */
4211         free(name);
4212
4213         return (ret);
4214 }
4215
4216 /*
4217  * Returns 1 if cb_data->cb_vdev_names[0] is a vdev name, 0 otherwise.
4218  */
4219 static int
4220 is_vdev(zpool_handle_t *zhp, void *cb_data)
4221 {
4222         return (for_each_vdev(zhp, is_vdev_cb, cb_data));
4223 }
4224
4225 /*
4226  * Check if vdevs are in a pool
4227  *
4228  * Return 1 if all argv[] strings are vdev names in pool "pool_name". Otherwise
4229  * return 0.  If pool_name is NULL, then search all pools.
4230  */
4231 static int
4232 are_vdevs_in_pool(int argc, char **argv, char *pool_name,
4233     iostat_cbdata_t *cb)
4234 {
4235         char **tmp_name;
4236         int ret = 0;
4237         int i;
4238         int pool_count = 0;
4239
4240         if ((argc == 0) || !*argv)
4241                 return (0);
4242
4243         if (pool_name)
4244                 pool_count = 1;
4245
4246         /* Temporarily hijack cb_vdev_names for a second... */
4247         tmp_name = cb->cb_vdev_names;
4248
4249         /* Go though our list of prospective vdev names */
4250         for (i = 0; i < argc; i++) {
4251                 cb->cb_vdev_names = argv + i;
4252
4253                 /* Is this name a vdev in our pools? */
4254                 ret = for_each_pool(pool_count, &pool_name, B_TRUE, NULL,
4255                     is_vdev, cb);
4256                 if (!ret) {
4257                         /* No match */
4258                         break;
4259                 }
4260         }
4261
4262         cb->cb_vdev_names = tmp_name;
4263
4264         return (ret);
4265 }
4266
4267 static int
4268 is_pool_cb(zpool_handle_t *zhp, void *data)
4269 {
4270         char *name = data;
4271         if (strcmp(name, zpool_get_name(zhp)) == 0)
4272                 return (1);
4273
4274         return (0);
4275 }
4276
4277 /*
4278  * Do we have a pool named *name?  If so, return 1, otherwise 0.
4279  */
4280 static int
4281 is_pool(char *name)
4282 {
4283         return (for_each_pool(0, NULL, B_TRUE, NULL,  is_pool_cb, name));
4284 }
4285
4286 /* Are all our argv[] strings pool names?  If so return 1, 0 otherwise. */
4287 static int
4288 are_all_pools(int argc, char **argv)
4289 {
4290         if ((argc == 0) || !*argv)
4291                 return (0);
4292
4293         while (--argc >= 0)
4294                 if (!is_pool(argv[argc]))
4295                         return (0);
4296
4297         return (1);
4298 }
4299
4300 /*
4301  * Helper function to print out vdev/pool names we can't resolve.  Used for an
4302  * error message.
4303  */
4304 static void
4305 error_list_unresolved_vdevs(int argc, char **argv, char *pool_name,
4306     iostat_cbdata_t *cb)
4307 {
4308         int i;
4309         char *name;
4310         char *str;
4311         for (i = 0; i < argc; i++) {
4312                 name = argv[i];
4313
4314                 if (is_pool(name))
4315                         str = gettext("pool");
4316                 else if (are_vdevs_in_pool(1, &name, pool_name, cb))
4317                         str = gettext("vdev in this pool");
4318                 else if (are_vdevs_in_pool(1, &name, NULL, cb))
4319                         str = gettext("vdev in another pool");
4320                 else
4321                         str = gettext("unknown");
4322
4323                 fprintf(stderr, "\t%s (%s)\n", name, str);
4324         }
4325 }
4326
4327 /*
4328  * Same as get_interval_count(), but with additional checks to not misinterpret
4329  * guids as interval/count values.  Assumes VDEV_NAME_GUID is set in
4330  * cb.cb_name_flags.
4331  */
4332 static void
4333 get_interval_count_filter_guids(int *argc, char **argv, float *interval,
4334     unsigned long *count, iostat_cbdata_t *cb)
4335 {
4336         char **tmpargv = argv;
4337         int argc_for_interval = 0;
4338
4339         /* Is the last arg an interval value?  Or a guid? */
4340         if (*argc >= 1 && !are_vdevs_in_pool(1, &argv[*argc - 1], NULL, cb)) {
4341                 /*
4342                  * The last arg is not a guid, so it's probably an
4343                  * interval value.
4344                  */
4345                 argc_for_interval++;
4346
4347                 if (*argc >= 2 &&
4348                     !are_vdevs_in_pool(1, &argv[*argc - 2], NULL, cb)) {
4349                         /*
4350                          * The 2nd to last arg is not a guid, so it's probably
4351                          * an interval value.
4352                          */
4353                         argc_for_interval++;
4354                 }
4355         }
4356
4357         /* Point to our list of possible intervals */
4358         tmpargv = &argv[*argc - argc_for_interval];
4359
4360         *argc = *argc - argc_for_interval;
4361         get_interval_count(&argc_for_interval, tmpargv,
4362             interval, count);
4363 }
4364
4365 /*
4366  * Floating point sleep().  Allows you to pass in a floating point value for
4367  * seconds.
4368  */
4369 static void
4370 fsleep(float sec)
4371 {
4372         struct timespec req;
4373         req.tv_sec = floor(sec);
4374         req.tv_nsec = (sec - (float)req.tv_sec) * NANOSEC;
4375         nanosleep(&req, NULL);
4376 }
4377
4378 /*
4379  * Run one of the zpool status/iostat -c scripts with the help (-h) option and
4380  * print the result.
4381  *
4382  * name:        Short name of the script ('iostat').
4383  * path:        Full path to the script ('/usr/local/etc/zfs/zpool.d/iostat');
4384  */
4385 static void
4386 print_zpool_script_help(char *name, char *path)
4387 {
4388         char *argv[] = {path, "-h", NULL};
4389         char **lines = NULL;
4390         int lines_cnt = 0;
4391         int rc;
4392
4393         rc = libzfs_run_process_get_stdout_nopath(path, argv, NULL, &lines,
4394             &lines_cnt);
4395         if (rc != 0 || lines == NULL || lines_cnt <= 0) {
4396                 if (lines != NULL)
4397                         libzfs_free_str_array(lines, lines_cnt);
4398                 return;
4399         }
4400
4401         for (int i = 0; i < lines_cnt; i++)
4402                 if (!is_blank_str(lines[i]))
4403                         printf("  %-14s  %s\n", name, lines[i]);
4404
4405         libzfs_free_str_array(lines, lines_cnt);
4406 }
4407
4408 /*
4409  * Go though the zpool status/iostat -c scripts in the user's path, run their
4410  * help option (-h), and print out the results.
4411  */
4412 static void
4413 print_zpool_dir_scripts(char *dirpath)
4414 {
4415         DIR *dir;
4416         struct dirent *ent;
4417         char fullpath[MAXPATHLEN];
4418         struct stat dir_stat;
4419
4420         if ((dir = opendir(dirpath)) != NULL) {
4421                 /* print all the files and directories within directory */
4422                 while ((ent = readdir(dir)) != NULL) {
4423                         sprintf(fullpath, "%s/%s", dirpath, ent->d_name);
4424
4425                         /* Print the scripts */
4426                         if (stat(fullpath, &dir_stat) == 0)
4427                                 if (dir_stat.st_mode & S_IXUSR &&
4428                                     S_ISREG(dir_stat.st_mode))
4429                                         print_zpool_script_help(ent->d_name,
4430                                             fullpath);
4431                 }
4432                 closedir(dir);
4433         }
4434 }
4435
4436 /*
4437  * Print out help text for all zpool status/iostat -c scripts.
4438  */
4439 static void
4440 print_zpool_script_list(char *subcommand)
4441 {
4442         char *dir, *sp;
4443
4444         printf(gettext("Available 'zpool %s -c' commands:\n"), subcommand);
4445
4446         sp = zpool_get_cmd_search_path();
4447         if (sp == NULL)
4448                 return;
4449
4450         dir = strtok(sp, ":");
4451         while (dir != NULL) {
4452                 print_zpool_dir_scripts(dir);
4453                 dir = strtok(NULL, ":");
4454         }
4455
4456         free(sp);
4457 }
4458
4459 /*
4460  * zpool iostat [[-c [script1,script2,...]] [-lq]|[-rw]] [-ghHLpPvy] [-n name]
4461  *              [-T d|u] [[ pool ...]|[pool vdev ...]|[vdev ...]]
4462  *              [interval [count]]
4463  *
4464  *      -c CMD  For each vdev, run command CMD
4465  *      -g      Display guid for individual vdev name.
4466  *      -L      Follow links when resolving vdev path name.
4467  *      -P      Display full path for vdev name.
4468  *      -v      Display statistics for individual vdevs
4469  *      -h      Display help
4470  *      -p      Display values in parsable (exact) format.
4471  *      -H      Scripted mode.  Don't display headers, and separate properties
4472  *              by a single tab.
4473  *      -l      Display average latency
4474  *      -q      Display queue depths
4475  *      -w      Display latency histograms
4476  *      -r      Display request size histogram
4477  *      -T      Display a timestamp in date(1) or Unix format
4478  *
4479  * This command can be tricky because we want to be able to deal with pool
4480  * creation/destruction as well as vdev configuration changes.  The bulk of this
4481  * processing is handled by the pool_list_* routines in zpool_iter.c.  We rely
4482  * on pool_list_update() to detect the addition of new pools.  Configuration
4483  * changes are all handled within libzfs.
4484  */
4485 int
4486 zpool_do_iostat(int argc, char **argv)
4487 {
4488         int c;
4489         int ret;
4490         int npools;
4491         float interval = 0;
4492         unsigned long count = 0;
4493         zpool_list_t *list;
4494         boolean_t verbose = B_FALSE;
4495         boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE;
4496         boolean_t queues = B_FALSE, parsable = B_FALSE, scripted = B_FALSE;
4497         boolean_t omit_since_boot = B_FALSE;
4498         boolean_t guid = B_FALSE;
4499         boolean_t follow_links = B_FALSE;
4500         boolean_t full_name = B_FALSE;
4501         iostat_cbdata_t cb = { 0 };
4502         char *cmd = NULL;
4503
4504         /* Used for printing error message */
4505         const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q',
4506             [IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'};
4507
4508         uint64_t unsupported_flags;
4509
4510         /* check options */
4511         while ((c = getopt(argc, argv, "c:gLPT:vyhplqrwH")) != -1) {
4512                 switch (c) {
4513                 case 'c':
4514                         if (cmd != NULL) {
4515                                 fprintf(stderr,
4516                                     gettext("Can't set -c flag twice\n"));
4517                                 exit(1);
4518                         }
4519
4520                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
4521                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
4522                                 fprintf(stderr, gettext(
4523                                     "Can't run -c, disabled by "
4524                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
4525                                 exit(1);
4526                         }
4527
4528                         if ((getuid() <= 0 || geteuid() <= 0) &&
4529                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
4530                                 fprintf(stderr, gettext(
4531                                     "Can't run -c with root privileges "
4532                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
4533                                 exit(1);
4534                         }
4535                         cmd = optarg;
4536                         verbose = B_TRUE;
4537                         break;
4538                 case 'g':
4539                         guid = B_TRUE;
4540                         break;
4541                 case 'L':
4542                         follow_links = B_TRUE;
4543                         break;
4544                 case 'P':
4545                         full_name = B_TRUE;
4546                         break;
4547                 case 'T':
4548                         get_timestamp_arg(*optarg);
4549                         break;
4550                 case 'v':
4551                         verbose = B_TRUE;
4552                         break;
4553                 case 'p':
4554                         parsable = B_TRUE;
4555                         break;
4556                 case 'l':
4557                         latency = B_TRUE;
4558                         break;
4559                 case 'q':
4560                         queues = B_TRUE;
4561                         break;
4562                 case 'H':
4563                         scripted = B_TRUE;
4564                         break;
4565                 case 'w':
4566                         l_histo = B_TRUE;
4567                         break;
4568                 case 'r':
4569                         rq_histo = B_TRUE;
4570                         break;
4571                 case 'y':
4572                         omit_since_boot = B_TRUE;
4573                         break;
4574                 case 'h':
4575                         usage(B_FALSE);
4576                         break;
4577                 case '?':
4578                         if (optopt == 'c') {
4579                                 print_zpool_script_list("iostat");
4580                                 exit(0);
4581                         } else {
4582                                 fprintf(stderr,
4583                                     gettext("invalid option '%c'\n"), optopt);
4584                         }
4585                         usage(B_FALSE);
4586                 }
4587         }
4588
4589         argc -= optind;
4590         argv += optind;
4591
4592         cb.cb_literal = parsable;
4593         cb.cb_scripted = scripted;
4594
4595         if (guid)
4596                 cb.cb_name_flags |= VDEV_NAME_GUID;
4597         if (follow_links)
4598                 cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
4599         if (full_name)
4600                 cb.cb_name_flags |= VDEV_NAME_PATH;
4601         cb.cb_iteration = 0;
4602         cb.cb_namewidth = 0;
4603         cb.cb_verbose = verbose;
4604
4605         /* Get our interval and count values (if any) */
4606         if (guid) {
4607                 get_interval_count_filter_guids(&argc, argv, &interval,
4608                     &count, &cb);
4609         } else {
4610                 get_interval_count(&argc, argv, &interval, &count);
4611         }
4612
4613         if (argc == 0) {
4614                 /* No args, so just print the defaults. */
4615         } else if (are_all_pools(argc, argv)) {
4616                 /* All the args are pool names */
4617         } else if (are_vdevs_in_pool(argc, argv, NULL, &cb)) {
4618                 /* All the args are vdevs */
4619                 cb.cb_vdev_names = argv;
4620                 cb.cb_vdev_names_count = argc;
4621                 argc = 0; /* No pools to process */
4622         } else if (are_all_pools(1, argv)) {
4623                 /* The first arg is a pool name */
4624                 if (are_vdevs_in_pool(argc - 1, argv + 1, argv[0], &cb)) {
4625                         /* ...and the rest are vdev names */
4626                         cb.cb_vdev_names = argv + 1;
4627                         cb.cb_vdev_names_count = argc - 1;
4628                         argc = 1; /* One pool to process */
4629                 } else {
4630                         fprintf(stderr, gettext("Expected either a list of "));
4631                         fprintf(stderr, gettext("pools, or list of vdevs in"));
4632                         fprintf(stderr, " \"%s\", ", argv[0]);
4633                         fprintf(stderr, gettext("but got:\n"));
4634                         error_list_unresolved_vdevs(argc - 1, argv + 1,
4635                             argv[0], &cb);
4636                         fprintf(stderr, "\n");
4637                         usage(B_FALSE);
4638                         return (1);
4639                 }
4640         } else {
4641                 /*
4642                  * The args don't make sense. The first arg isn't a pool name,
4643                  * nor are all the args vdevs.
4644                  */
4645                 fprintf(stderr, gettext("Unable to parse pools/vdevs list.\n"));
4646                 fprintf(stderr, "\n");
4647                 return (1);
4648         }
4649
4650         if (cb.cb_vdev_names_count != 0) {
4651                 /*
4652                  * If user specified vdevs, it implies verbose.
4653                  */
4654                 cb.cb_verbose = B_TRUE;
4655         }
4656
4657         /*
4658          * Construct the list of all interesting pools.
4659          */
4660         ret = 0;
4661         if ((list = pool_list_get(argc, argv, NULL, &ret)) == NULL)
4662                 return (1);
4663
4664         if (pool_list_count(list) == 0 && argc != 0) {
4665                 pool_list_free(list);
4666                 return (1);
4667         }
4668
4669         if (pool_list_count(list) == 0 && interval == 0) {
4670                 pool_list_free(list);
4671                 (void) fprintf(stderr, gettext("no pools available\n"));
4672                 return (1);
4673         }
4674
4675         if ((l_histo || rq_histo) && (cmd != NULL || latency || queues)) {
4676                 pool_list_free(list);
4677                 (void) fprintf(stderr,
4678                     gettext("[-r|-w] isn't allowed with [-c|-l|-q]\n"));
4679                 usage(B_FALSE);
4680                 return (1);
4681         }
4682
4683         if (l_histo && rq_histo) {
4684                 pool_list_free(list);
4685                 (void) fprintf(stderr,
4686                     gettext("Only one of [-r|-w] can be passed at a time\n"));
4687                 usage(B_FALSE);
4688                 return (1);
4689         }
4690
4691         /*
4692          * Enter the main iostat loop.
4693          */
4694         cb.cb_list = list;
4695
4696         if (l_histo) {
4697                 /*
4698                  * Histograms tables look out of place when you try to display
4699                  * them with the other stats, so make a rule that you can only
4700                  * print histograms by themselves.
4701                  */
4702                 cb.cb_flags = IOS_L_HISTO_M;
4703         } else if (rq_histo) {
4704                 cb.cb_flags = IOS_RQ_HISTO_M;
4705         } else {
4706                 cb.cb_flags = IOS_DEFAULT_M;
4707                 if (latency)
4708                         cb.cb_flags |= IOS_LATENCY_M;
4709                 if (queues)
4710                         cb.cb_flags |= IOS_QUEUES_M;
4711         }
4712
4713         /*
4714          * See if the module supports all the stats we want to display.
4715          */
4716         unsupported_flags = cb.cb_flags & ~get_stat_flags(list);
4717         if (unsupported_flags) {
4718                 uint64_t f;
4719                 int idx;
4720                 fprintf(stderr,
4721                     gettext("The loaded zfs module doesn't support:"));
4722
4723                 /* for each bit set in unsupported_flags */
4724                 for (f = unsupported_flags; f; f &= ~(1ULL << idx)) {
4725                         idx = lowbit64(f) - 1;
4726                         fprintf(stderr, " -%c", flag_to_arg[idx]);
4727                 }
4728
4729                 fprintf(stderr, ".  Try running a newer module.\n");
4730                 pool_list_free(list);
4731
4732                 return (1);
4733         }
4734
4735         for (;;) {
4736                 if ((npools = pool_list_count(list)) == 0)
4737                         (void) fprintf(stderr, gettext("no pools available\n"));
4738                 else {
4739                         /*
4740                          * If this is the first iteration and -y was supplied
4741                          * we skip any printing.
4742                          */
4743                         boolean_t skip = (omit_since_boot &&
4744                             cb.cb_iteration == 0);
4745
4746                         /*
4747                          * Refresh all statistics.  This is done as an
4748                          * explicit step before calculating the maximum name
4749                          * width, so that any * configuration changes are
4750                          * properly accounted for.
4751                          */
4752                         (void) pool_list_iter(list, B_FALSE, refresh_iostat,
4753                             &cb);
4754
4755                         /*
4756                          * Iterate over all pools to determine the maximum width
4757                          * for the pool / device name column across all pools.
4758                          */
4759                         cb.cb_namewidth = 0;
4760                         (void) pool_list_iter(list, B_FALSE, get_namewidth,
4761                             &cb);
4762
4763                         if (timestamp_fmt != NODATE)
4764                                 print_timestamp(timestamp_fmt);
4765
4766                         if (cmd != NULL && cb.cb_verbose &&
4767                             !(cb.cb_flags & IOS_ANYHISTO_M)) {
4768                                 cb.vcdl = all_pools_for_each_vdev_run(argc,
4769                                     argv, cmd, g_zfs, cb.cb_vdev_names,
4770                                     cb.cb_vdev_names_count, cb.cb_name_flags);
4771                         } else {
4772                                 cb.vcdl = NULL;
4773                         }
4774
4775                         /*
4776                          * If it's the first time and we're not skipping it,
4777                          * or either skip or verbose mode, print the header.
4778                          *
4779                          * The histogram code explicitly prints its header on
4780                          * every vdev, so skip this for histograms.
4781                          */
4782                         if (((++cb.cb_iteration == 1 && !skip) ||
4783                             (skip != verbose)) &&
4784                             (!(cb.cb_flags & IOS_ANYHISTO_M)) &&
4785                             !cb.cb_scripted)
4786                                 print_iostat_header(&cb);
4787
4788                         if (skip) {
4789                                 (void) fsleep(interval);
4790                                 continue;
4791                         }
4792
4793
4794                         pool_list_iter(list, B_FALSE, print_iostat, &cb);
4795
4796                         /*
4797                          * If there's more than one pool, and we're not in
4798                          * verbose mode (which prints a separator for us),
4799                          * then print a separator.
4800                          *
4801                          * In addition, if we're printing specific vdevs then
4802                          * we also want an ending separator.
4803                          */
4804                         if (((npools > 1 && !verbose &&
4805                             !(cb.cb_flags & IOS_ANYHISTO_M)) ||
4806                             (!(cb.cb_flags & IOS_ANYHISTO_M) &&
4807                             cb.cb_vdev_names_count)) &&
4808                             !cb.cb_scripted) {
4809                                 print_iostat_separator(&cb);
4810                                 if (cb.vcdl != NULL)
4811                                         print_cmd_columns(cb.vcdl, 1);
4812                                 printf("\n");
4813                         }
4814
4815                         if (cb.vcdl != NULL)
4816                                 free_vdev_cmd_data_list(cb.vcdl);
4817
4818                 }
4819
4820                 /*
4821                  * Flush the output so that redirection to a file isn't buffered
4822                  * indefinitely.
4823                  */
4824                 (void) fflush(stdout);
4825
4826                 if (interval == 0)
4827                         break;
4828
4829                 if (count != 0 && --count == 0)
4830                         break;
4831
4832                 (void) fsleep(interval);
4833         }
4834
4835         pool_list_free(list);
4836
4837         return (ret);
4838 }
4839
4840 typedef struct list_cbdata {
4841         boolean_t       cb_verbose;
4842         int             cb_name_flags;
4843         int             cb_namewidth;
4844         boolean_t       cb_scripted;
4845         zprop_list_t    *cb_proplist;
4846         boolean_t       cb_literal;
4847 } list_cbdata_t;
4848
4849 /*
4850  * Given a list of columns to display, output appropriate headers for each one.
4851  */
4852 static void
4853 print_header(list_cbdata_t *cb)
4854 {
4855         zprop_list_t *pl = cb->cb_proplist;
4856         char headerbuf[ZPOOL_MAXPROPLEN];
4857         const char *header;
4858         boolean_t first = B_TRUE;
4859         boolean_t right_justify;
4860         size_t width = 0;
4861
4862         for (; pl != NULL; pl = pl->pl_next) {
4863                 width = pl->pl_width;
4864                 if (first && cb->cb_verbose) {
4865                         /*
4866                          * Reset the width to accommodate the verbose listing
4867                          * of devices.
4868                          */
4869                         width = cb->cb_namewidth;
4870                 }
4871
4872                 if (!first)
4873                         (void) printf("  ");
4874                 else
4875                         first = B_FALSE;
4876
4877                 right_justify = B_FALSE;
4878                 if (pl->pl_prop != ZPROP_INVAL) {
4879                         header = zpool_prop_column_name(pl->pl_prop);
4880                         right_justify = zpool_prop_align_right(pl->pl_prop);
4881                 } else {
4882                         int i;
4883
4884                         for (i = 0; pl->pl_user_prop[i] != '\0'; i++)
4885                                 headerbuf[i] = toupper(pl->pl_user_prop[i]);
4886                         headerbuf[i] = '\0';
4887                         header = headerbuf;
4888                 }
4889
4890                 if (pl->pl_next == NULL && !right_justify)
4891                         (void) printf("%s", header);
4892                 else if (right_justify)
4893                         (void) printf("%*s", (int)width, header);
4894                 else
4895                         (void) printf("%-*s", (int)width, header);
4896         }
4897
4898         (void) printf("\n");
4899 }
4900
4901 /*
4902  * Given a pool and a list of properties, print out all the properties according
4903  * to the described layout.
4904  */
4905 static void
4906 print_pool(zpool_handle_t *zhp, list_cbdata_t *cb)
4907 {
4908         zprop_list_t *pl = cb->cb_proplist;
4909         boolean_t first = B_TRUE;
4910         char property[ZPOOL_MAXPROPLEN];
4911         char *propstr;
4912         boolean_t right_justify;
4913         size_t width;
4914
4915         for (; pl != NULL; pl = pl->pl_next) {
4916
4917                 width = pl->pl_width;
4918                 if (first && cb->cb_verbose) {
4919                         /*
4920                          * Reset the width to accommodate the verbose listing
4921                          * of devices.
4922                          */
4923                         width = cb->cb_namewidth;
4924                 }
4925
4926                 if (!first) {
4927                         if (cb->cb_scripted)
4928                                 (void) printf("\t");
4929                         else
4930                                 (void) printf("  ");
4931                 } else {
4932                         first = B_FALSE;
4933                 }
4934
4935                 right_justify = B_FALSE;
4936                 if (pl->pl_prop != ZPROP_INVAL) {
4937                         if (zpool_get_prop(zhp, pl->pl_prop, property,
4938                             sizeof (property), NULL, cb->cb_literal) != 0)
4939                                 propstr = "-";
4940                         else
4941                                 propstr = property;
4942
4943                         right_justify = zpool_prop_align_right(pl->pl_prop);
4944                 } else if ((zpool_prop_feature(pl->pl_user_prop) ||
4945                     zpool_prop_unsupported(pl->pl_user_prop)) &&
4946                     zpool_prop_get_feature(zhp, pl->pl_user_prop, property,
4947                     sizeof (property)) == 0) {
4948                         propstr = property;
4949                 } else {
4950                         propstr = "-";
4951                 }
4952
4953
4954                 /*
4955                  * If this is being called in scripted mode, or if this is the
4956                  * last column and it is left-justified, don't include a width
4957                  * format specifier.
4958                  */
4959                 if (cb->cb_scripted || (pl->pl_next == NULL && !right_justify))
4960                         (void) printf("%s", propstr);
4961                 else if (right_justify)
4962                         (void) printf("%*s", (int)width, propstr);
4963                 else
4964                         (void) printf("%-*s", (int)width, propstr);
4965         }
4966
4967         (void) printf("\n");
4968 }
4969
4970 static void
4971 print_one_column(zpool_prop_t prop, uint64_t value, boolean_t scripted,
4972     boolean_t valid, enum zfs_nicenum_format format)
4973 {
4974         char propval[64];
4975         boolean_t fixed;
4976         size_t width = zprop_width(prop, &fixed, ZFS_TYPE_POOL);
4977
4978         switch (prop) {
4979         case ZPOOL_PROP_EXPANDSZ:
4980                 if (value == 0)
4981                         (void) strlcpy(propval, "-", sizeof (propval));
4982                 else
4983                         zfs_nicenum_format(value, propval, sizeof (propval),
4984                             format);
4985                 break;
4986         case ZPOOL_PROP_FRAGMENTATION:
4987                 if (value == ZFS_FRAG_INVALID) {
4988                         (void) strlcpy(propval, "-", sizeof (propval));
4989                 } else if (format == ZFS_NICENUM_RAW) {
4990                         (void) snprintf(propval, sizeof (propval), "%llu",
4991                             (unsigned long long)value);
4992                 } else {
4993                         (void) snprintf(propval, sizeof (propval), "%llu%%",
4994                             (unsigned long long)value);
4995                 }
4996                 break;
4997         case ZPOOL_PROP_CAPACITY:
4998                 if (format == ZFS_NICENUM_RAW)
4999                         (void) snprintf(propval, sizeof (propval), "%llu",
5000                             (unsigned long long)value);
5001                 else
5002                         (void) snprintf(propval, sizeof (propval), "%llu%%",
5003                             (unsigned long long)value);
5004                 break;
5005         default:
5006                 zfs_nicenum_format(value, propval, sizeof (propval), format);
5007         }
5008
5009         if (!valid)
5010                 (void) strlcpy(propval, "-", sizeof (propval));
5011
5012         if (scripted)
5013                 (void) printf("\t%s", propval);
5014         else
5015                 (void) printf("  %*s", (int)width, propval);
5016 }
5017
5018 void
5019 print_list_stats(zpool_handle_t *zhp, const char *name, nvlist_t *nv,
5020     list_cbdata_t *cb, int depth)
5021 {
5022         nvlist_t **child;
5023         vdev_stat_t *vs;
5024         uint_t c, children;
5025         char *vname;
5026         boolean_t scripted = cb->cb_scripted;
5027         uint64_t islog = B_FALSE;
5028         boolean_t haslog = B_FALSE;
5029         char *dashes = "%-*s      -      -      -         -      -      -\n";
5030
5031         verify(nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
5032             (uint64_t **)&vs, &c) == 0);
5033
5034         if (name != NULL) {
5035                 boolean_t toplevel = (vs->vs_space != 0);
5036                 uint64_t cap;
5037                 enum zfs_nicenum_format format;
5038
5039                 if (cb->cb_literal)
5040                         format = ZFS_NICENUM_RAW;
5041                 else
5042                         format = ZFS_NICENUM_1024;
5043
5044                 if (strcmp(name, VDEV_TYPE_INDIRECT) == 0)
5045                         return;
5046
5047                 if (scripted)
5048                         (void) printf("\t%s", name);
5049                 else if (strlen(name) + depth > cb->cb_namewidth)
5050                         (void) printf("%*s%s", depth, "", name);
5051                 else
5052                         (void) printf("%*s%s%*s", depth, "", name,
5053                             (int)(cb->cb_namewidth - strlen(name) - depth), "");
5054
5055                 /*
5056                  * Print the properties for the individual vdevs. Some
5057                  * properties are only applicable to toplevel vdevs. The
5058                  * 'toplevel' boolean value is passed to the print_one_column()
5059                  * to indicate that the value is valid.
5060                  */
5061                 print_one_column(ZPOOL_PROP_SIZE, vs->vs_space, scripted,
5062                     toplevel, format);
5063                 print_one_column(ZPOOL_PROP_ALLOCATED, vs->vs_alloc, scripted,
5064                     toplevel, format);
5065                 print_one_column(ZPOOL_PROP_FREE, vs->vs_space - vs->vs_alloc,
5066                     scripted, toplevel, format);
5067                 print_one_column(ZPOOL_PROP_EXPANDSZ, vs->vs_esize, scripted,
5068                     B_TRUE, format);
5069                 print_one_column(ZPOOL_PROP_FRAGMENTATION,
5070                     vs->vs_fragmentation, scripted,
5071                     (vs->vs_fragmentation != ZFS_FRAG_INVALID && toplevel),
5072                     format);
5073                 cap = (vs->vs_space == 0) ? 0 :
5074                     (vs->vs_alloc * 100 / vs->vs_space);
5075                 print_one_column(ZPOOL_PROP_CAPACITY, cap, scripted, toplevel,
5076                     format);
5077                 (void) printf("\n");
5078         }
5079
5080         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
5081             &child, &children) != 0)
5082                 return;
5083
5084         for (c = 0; c < children; c++) {
5085                 uint64_t ishole = B_FALSE;
5086
5087                 if (nvlist_lookup_uint64(child[c],
5088                     ZPOOL_CONFIG_IS_HOLE, &ishole) == 0 && ishole)
5089                         continue;
5090
5091                 if (nvlist_lookup_uint64(child[c],
5092                     ZPOOL_CONFIG_IS_LOG, &islog) == 0 && islog) {
5093                         haslog = B_TRUE;
5094                         continue;
5095                 }
5096
5097                 vname = zpool_vdev_name(g_zfs, zhp, child[c],
5098                     cb->cb_name_flags);
5099                 print_list_stats(zhp, vname, child[c], cb, depth + 2);
5100                 free(vname);
5101         }
5102
5103         if (haslog == B_TRUE) {
5104                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5105                 (void) printf(dashes, cb->cb_namewidth, "log");
5106                 for (c = 0; c < children; c++) {
5107                         if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_LOG,
5108                             &islog) != 0 || !islog)
5109                                 continue;
5110                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5111                             cb->cb_name_flags);
5112                         print_list_stats(zhp, vname, child[c], cb, depth + 2);
5113                         free(vname);
5114                 }
5115         }
5116
5117         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
5118             &child, &children) == 0 && children > 0) {
5119                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5120                 (void) printf(dashes, cb->cb_namewidth, "cache");
5121                 for (c = 0; c < children; c++) {
5122                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5123                             cb->cb_name_flags);
5124                         print_list_stats(zhp, vname, child[c], cb, depth + 2);
5125                         free(vname);
5126                 }
5127         }
5128
5129         if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_SPARES, &child,
5130             &children) == 0 && children > 0) {
5131                 /* LINTED E_SEC_PRINTF_VAR_FMT */
5132                 (void) printf(dashes, cb->cb_namewidth, "spare");
5133                 for (c = 0; c < children; c++) {
5134                         vname = zpool_vdev_name(g_zfs, zhp, child[c],
5135                             cb->cb_name_flags);
5136                         print_list_stats(zhp, vname, child[c], cb, depth + 2);
5137                         free(vname);
5138                 }
5139         }
5140 }
5141
5142
5143 /*
5144  * Generic callback function to list a pool.
5145  */
5146 int
5147 list_callback(zpool_handle_t *zhp, void *data)
5148 {
5149         list_cbdata_t *cbp = data;
5150         nvlist_t *config;
5151         nvlist_t *nvroot;
5152
5153         config = zpool_get_config(zhp, NULL);
5154
5155         print_pool(zhp, cbp);
5156         if (!cbp->cb_verbose)
5157                 return (0);
5158
5159         verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
5160             &nvroot) == 0);
5161         print_list_stats(zhp, NULL, nvroot, cbp, 0);
5162
5163         return (0);
5164 }
5165
5166 /*
5167  * zpool list [-gHLpP] [-o prop[,prop]*] [-T d|u] [pool] ... [interval [count]]
5168  *
5169  *      -g      Display guid for individual vdev name.
5170  *      -H      Scripted mode.  Don't display headers, and separate properties
5171  *              by a single tab.
5172  *      -L      Follow links when resolving vdev path name.
5173  *      -o      List of properties to display.  Defaults to
5174  *              "name,size,allocated,free,expandsize,fragmentation,capacity,"
5175  *              "dedupratio,health,altroot"
5176  *      -p      Display values in parsable (exact) format.
5177  *      -P      Display full path for vdev name.
5178  *      -T      Display a timestamp in date(1) or Unix format
5179  *
5180  * List all pools in the system, whether or not they're healthy.  Output space
5181  * statistics for each one, as well as health status summary.
5182  */
5183 int
5184 zpool_do_list(int argc, char **argv)
5185 {
5186         int c;
5187         int ret = 0;
5188         list_cbdata_t cb = { 0 };
5189         static char default_props[] =
5190             "name,size,allocated,free,expandsize,fragmentation,capacity,"
5191             "dedupratio,health,altroot";
5192         char *props = default_props;
5193         float interval = 0;
5194         unsigned long count = 0;
5195         zpool_list_t *list;
5196         boolean_t first = B_TRUE;
5197
5198         /* check options */
5199         while ((c = getopt(argc, argv, ":gHLo:pPT:v")) != -1) {
5200                 switch (c) {
5201                 case 'g':
5202                         cb.cb_name_flags |= VDEV_NAME_GUID;
5203                         break;
5204                 case 'H':
5205                         cb.cb_scripted = B_TRUE;
5206                         break;
5207                 case 'L':
5208                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
5209                         break;
5210                 case 'o':
5211                         props = optarg;
5212                         break;
5213                 case 'P':
5214                         cb.cb_name_flags |= VDEV_NAME_PATH;
5215                         break;
5216                 case 'p':
5217                         cb.cb_literal = B_TRUE;
5218                         break;
5219                 case 'T':
5220                         get_timestamp_arg(*optarg);
5221                         break;
5222                 case 'v':
5223                         cb.cb_verbose = B_TRUE;
5224                         break;
5225                 case ':':
5226                         (void) fprintf(stderr, gettext("missing argument for "
5227                             "'%c' option\n"), optopt);
5228                         usage(B_FALSE);
5229                         break;
5230                 case '?':
5231                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5232                             optopt);
5233                         usage(B_FALSE);
5234                 }
5235         }
5236
5237         argc -= optind;
5238         argv += optind;
5239
5240         get_interval_count(&argc, argv, &interval, &count);
5241
5242         if (zprop_get_list(g_zfs, props, &cb.cb_proplist, ZFS_TYPE_POOL) != 0)
5243                 usage(B_FALSE);
5244
5245         for (;;) {
5246                 if ((list = pool_list_get(argc, argv, &cb.cb_proplist,
5247                     &ret)) == NULL)
5248                         return (1);
5249
5250                 if (pool_list_count(list) == 0)
5251                         break;
5252
5253                 if (timestamp_fmt != NODATE)
5254                         print_timestamp(timestamp_fmt);
5255
5256                 if (!cb.cb_scripted && (first || cb.cb_verbose)) {
5257                         print_header(&cb);
5258                         first = B_FALSE;
5259                 }
5260                 ret = pool_list_iter(list, B_TRUE, list_callback, &cb);
5261
5262                 if (interval == 0)
5263                         break;
5264
5265                 if (count != 0 && --count == 0)
5266                         break;
5267
5268                 pool_list_free(list);
5269                 (void) fsleep(interval);
5270         }
5271
5272         if (argc == 0 && !cb.cb_scripted && pool_list_count(list) == 0) {
5273                 (void) printf(gettext("no pools available\n"));
5274                 ret = 0;
5275         }
5276
5277         pool_list_free(list);
5278         zprop_free_list(cb.cb_proplist);
5279         return (ret);
5280 }
5281
5282 static int
5283 zpool_do_attach_or_replace(int argc, char **argv, int replacing)
5284 {
5285         boolean_t force = B_FALSE;
5286         int c;
5287         nvlist_t *nvroot;
5288         char *poolname, *old_disk, *new_disk;
5289         zpool_handle_t *zhp;
5290         nvlist_t *props = NULL;
5291         char *propval;
5292         int ret;
5293
5294         /* check options */
5295         while ((c = getopt(argc, argv, "fo:")) != -1) {
5296                 switch (c) {
5297                 case 'f':
5298                         force = B_TRUE;
5299                         break;
5300                 case 'o':
5301                         if ((propval = strchr(optarg, '=')) == NULL) {
5302                                 (void) fprintf(stderr, gettext("missing "
5303                                     "'=' for -o option\n"));
5304                                 usage(B_FALSE);
5305                         }
5306                         *propval = '\0';
5307                         propval++;
5308
5309                         if ((strcmp(optarg, ZPOOL_CONFIG_ASHIFT) != 0) ||
5310                             (add_prop_list(optarg, propval, &props, B_TRUE)))
5311                                 usage(B_FALSE);
5312                         break;
5313                 case '?':
5314                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5315                             optopt);
5316                         usage(B_FALSE);
5317                 }
5318         }
5319
5320         argc -= optind;
5321         argv += optind;
5322
5323         /* get pool name and check number of arguments */
5324         if (argc < 1) {
5325                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
5326                 usage(B_FALSE);
5327         }
5328
5329         poolname = argv[0];
5330
5331         if (argc < 2) {
5332                 (void) fprintf(stderr,
5333                     gettext("missing <device> specification\n"));
5334                 usage(B_FALSE);
5335         }
5336
5337         old_disk = argv[1];
5338
5339         if (argc < 3) {
5340                 if (!replacing) {
5341                         (void) fprintf(stderr,
5342                             gettext("missing <new_device> specification\n"));
5343                         usage(B_FALSE);
5344                 }
5345                 new_disk = old_disk;
5346                 argc -= 1;
5347                 argv += 1;
5348         } else {
5349                 new_disk = argv[2];
5350                 argc -= 2;
5351                 argv += 2;
5352         }
5353
5354         if (argc > 1) {
5355                 (void) fprintf(stderr, gettext("too many arguments\n"));
5356                 usage(B_FALSE);
5357         }
5358
5359         if ((zhp = zpool_open(g_zfs, poolname)) == NULL) {
5360                 nvlist_free(props);
5361                 return (1);
5362         }
5363
5364         if (zpool_get_config(zhp, NULL) == NULL) {
5365                 (void) fprintf(stderr, gettext("pool '%s' is unavailable\n"),
5366                     poolname);
5367                 zpool_close(zhp);
5368                 nvlist_free(props);
5369                 return (1);
5370         }
5371
5372         /* unless manually specified use "ashift" pool property (if set) */
5373         if (!nvlist_exists(props, ZPOOL_CONFIG_ASHIFT)) {
5374                 int intval;
5375                 zprop_source_t src;
5376                 char strval[ZPOOL_MAXPROPLEN];
5377
5378                 intval = zpool_get_prop_int(zhp, ZPOOL_PROP_ASHIFT, &src);
5379                 if (src != ZPROP_SRC_DEFAULT) {
5380                         (void) sprintf(strval, "%" PRId32, intval);
5381                         verify(add_prop_list(ZPOOL_CONFIG_ASHIFT, strval,
5382                             &props, B_TRUE) == 0);
5383                 }
5384         }
5385
5386         nvroot = make_root_vdev(zhp, props, force, B_FALSE, replacing, B_FALSE,
5387             argc, argv);
5388         if (nvroot == NULL) {
5389                 zpool_close(zhp);
5390                 nvlist_free(props);
5391                 return (1);
5392         }
5393
5394         ret = zpool_vdev_attach(zhp, old_disk, new_disk, nvroot, replacing);
5395
5396         nvlist_free(props);
5397         nvlist_free(nvroot);
5398         zpool_close(zhp);
5399
5400         return (ret);
5401 }
5402
5403 /*
5404  * zpool replace [-f] <pool> <device> <new_device>
5405  *
5406  *      -f      Force attach, even if <new_device> appears to be in use.
5407  *
5408  * Replace <device> with <new_device>.
5409  */
5410 /* ARGSUSED */
5411 int
5412 zpool_do_replace(int argc, char **argv)
5413 {
5414         return (zpool_do_attach_or_replace(argc, argv, B_TRUE));
5415 }
5416
5417 /*
5418  * zpool attach [-f] [-o property=value] <pool> <device> <new_device>
5419  *
5420  *      -f      Force attach, even if <new_device> appears to be in use.
5421  *      -o      Set property=value.
5422  *
5423  * Attach <new_device> to the mirror containing <device>.  If <device> is not
5424  * part of a mirror, then <device> will be transformed into a mirror of
5425  * <device> and <new_device>.  In either case, <new_device> will begin life
5426  * with a DTL of [0, now], and will immediately begin to resilver itself.
5427  */
5428 int
5429 zpool_do_attach(int argc, char **argv)
5430 {
5431         return (zpool_do_attach_or_replace(argc, argv, B_FALSE));
5432 }
5433
5434 /*
5435  * zpool detach [-f] <pool> <device>
5436  *
5437  *      -f      Force detach of <device>, even if DTLs argue against it
5438  *              (not supported yet)
5439  *
5440  * Detach a device from a mirror.  The operation will be refused if <device>
5441  * is the last device in the mirror, or if the DTLs indicate that this device
5442  * has the only valid copy of some data.
5443  */
5444 /* ARGSUSED */
5445 int
5446 zpool_do_detach(int argc, char **argv)
5447 {
5448         int c;
5449         char *poolname, *path;
5450         zpool_handle_t *zhp;
5451         int ret;
5452
5453         /* check options */
5454         while ((c = getopt(argc, argv, "f")) != -1) {
5455                 switch (c) {
5456                 case 'f':
5457                 case '?':
5458                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5459                             optopt);
5460                         usage(B_FALSE);
5461                 }
5462         }
5463
5464         argc -= optind;
5465         argv += optind;
5466
5467         /* get pool name and check number of arguments */
5468         if (argc < 1) {
5469                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
5470                 usage(B_FALSE);
5471         }
5472
5473         if (argc < 2) {
5474                 (void) fprintf(stderr,
5475                     gettext("missing <device> specification\n"));
5476                 usage(B_FALSE);
5477         }
5478
5479         poolname = argv[0];
5480         path = argv[1];
5481
5482         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5483                 return (1);
5484
5485         ret = zpool_vdev_detach(zhp, path);
5486
5487         zpool_close(zhp);
5488
5489         return (ret);
5490 }
5491
5492 /*
5493  * zpool split [-gLnP] [-o prop=val] ...
5494  *              [-o mntopt] ...
5495  *              [-R altroot] <pool> <newpool> [<device> ...]
5496  *
5497  *      -g      Display guid for individual vdev name.
5498  *      -L      Follow links when resolving vdev path name.
5499  *      -n      Do not split the pool, but display the resulting layout if
5500  *              it were to be split.
5501  *      -o      Set property=value, or set mount options.
5502  *      -P      Display full path for vdev name.
5503  *      -R      Mount the split-off pool under an alternate root.
5504  *      -l      Load encryption keys while importing.
5505  *
5506  * Splits the named pool and gives it the new pool name.  Devices to be split
5507  * off may be listed, provided that no more than one device is specified
5508  * per top-level vdev mirror.  The newly split pool is left in an exported
5509  * state unless -R is specified.
5510  *
5511  * Restrictions: the top-level of the pool pool must only be made up of
5512  * mirrors; all devices in the pool must be healthy; no device may be
5513  * undergoing a resilvering operation.
5514  */
5515 int
5516 zpool_do_split(int argc, char **argv)
5517 {
5518         char *srcpool, *newpool, *propval;
5519         char *mntopts = NULL;
5520         splitflags_t flags;
5521         int c, ret = 0;
5522         boolean_t loadkeys = B_FALSE;
5523         zpool_handle_t *zhp;
5524         nvlist_t *config, *props = NULL;
5525
5526         flags.dryrun = B_FALSE;
5527         flags.import = B_FALSE;
5528         flags.name_flags = 0;
5529
5530         /* check options */
5531         while ((c = getopt(argc, argv, ":gLR:lno:P")) != -1) {
5532                 switch (c) {
5533                 case 'g':
5534                         flags.name_flags |= VDEV_NAME_GUID;
5535                         break;
5536                 case 'L':
5537                         flags.name_flags |= VDEV_NAME_FOLLOW_LINKS;
5538                         break;
5539                 case 'R':
5540                         flags.import = B_TRUE;
5541                         if (add_prop_list(
5542                             zpool_prop_to_name(ZPOOL_PROP_ALTROOT), optarg,
5543                             &props, B_TRUE) != 0) {
5544                                 nvlist_free(props);
5545                                 usage(B_FALSE);
5546                         }
5547                         break;
5548                 case 'l':
5549                         loadkeys = B_TRUE;
5550                         break;
5551                 case 'n':
5552                         flags.dryrun = B_TRUE;
5553                         break;
5554                 case 'o':
5555                         if ((propval = strchr(optarg, '=')) != NULL) {
5556                                 *propval = '\0';
5557                                 propval++;
5558                                 if (add_prop_list(optarg, propval,
5559                                     &props, B_TRUE) != 0) {
5560                                         nvlist_free(props);
5561                                         usage(B_FALSE);
5562                                 }
5563                         } else {
5564                                 mntopts = optarg;
5565                         }
5566                         break;
5567                 case 'P':
5568                         flags.name_flags |= VDEV_NAME_PATH;
5569                         break;
5570                 case ':':
5571                         (void) fprintf(stderr, gettext("missing argument for "
5572                             "'%c' option\n"), optopt);
5573                         usage(B_FALSE);
5574                         break;
5575                 case '?':
5576                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5577                             optopt);
5578                         usage(B_FALSE);
5579                         break;
5580                 }
5581         }
5582
5583         if (!flags.import && mntopts != NULL) {
5584                 (void) fprintf(stderr, gettext("setting mntopts is only "
5585                     "valid when importing the pool\n"));
5586                 usage(B_FALSE);
5587         }
5588
5589         if (!flags.import && loadkeys) {
5590                 (void) fprintf(stderr, gettext("loading keys is only "
5591                     "valid when importing the pool\n"));
5592                 usage(B_FALSE);
5593         }
5594
5595         argc -= optind;
5596         argv += optind;
5597
5598         if (argc < 1) {
5599                 (void) fprintf(stderr, gettext("Missing pool name\n"));
5600                 usage(B_FALSE);
5601         }
5602         if (argc < 2) {
5603                 (void) fprintf(stderr, gettext("Missing new pool name\n"));
5604                 usage(B_FALSE);
5605         }
5606
5607         srcpool = argv[0];
5608         newpool = argv[1];
5609
5610         argc -= 2;
5611         argv += 2;
5612
5613         if ((zhp = zpool_open(g_zfs, srcpool)) == NULL) {
5614                 nvlist_free(props);
5615                 return (1);
5616         }
5617
5618         config = split_mirror_vdev(zhp, newpool, props, flags, argc, argv);
5619         if (config == NULL) {
5620                 ret = 1;
5621         } else {
5622                 if (flags.dryrun) {
5623                         (void) printf(gettext("would create '%s' with the "
5624                             "following layout:\n\n"), newpool);
5625                         print_vdev_tree(NULL, newpool, config, 0, B_FALSE,
5626                             flags.name_flags);
5627                 }
5628         }
5629
5630         zpool_close(zhp);
5631
5632         if (ret != 0 || flags.dryrun || !flags.import) {
5633                 nvlist_free(config);
5634                 nvlist_free(props);
5635                 return (ret);
5636         }
5637
5638         /*
5639          * The split was successful. Now we need to open the new
5640          * pool and import it.
5641          */
5642         if ((zhp = zpool_open_canfail(g_zfs, newpool)) == NULL) {
5643                 nvlist_free(config);
5644                 nvlist_free(props);
5645                 return (1);
5646         }
5647
5648         if (loadkeys) {
5649                 ret = zfs_crypto_attempt_load_keys(g_zfs, newpool);
5650                 if (ret != 0)
5651                         ret = 1;
5652         }
5653
5654         if (zpool_get_state(zhp) != POOL_STATE_UNAVAIL &&
5655             zpool_enable_datasets(zhp, mntopts, 0) != 0) {
5656                 ret = 1;
5657                 (void) fprintf(stderr, gettext("Split was successful, but "
5658                     "the datasets could not all be mounted\n"));
5659                 (void) fprintf(stderr, gettext("Try doing '%s' with a "
5660                     "different altroot\n"), "zpool import");
5661         }
5662         zpool_close(zhp);
5663         nvlist_free(config);
5664         nvlist_free(props);
5665
5666         return (ret);
5667 }
5668
5669
5670
5671 /*
5672  * zpool online <pool> <device> ...
5673  */
5674 int
5675 zpool_do_online(int argc, char **argv)
5676 {
5677         int c, i;
5678         char *poolname;
5679         zpool_handle_t *zhp;
5680         int ret = 0;
5681         vdev_state_t newstate;
5682         int flags = 0;
5683
5684         /* check options */
5685         while ((c = getopt(argc, argv, "et")) != -1) {
5686                 switch (c) {
5687                 case 'e':
5688                         flags |= ZFS_ONLINE_EXPAND;
5689                         break;
5690                 case 't':
5691                 case '?':
5692                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5693                             optopt);
5694                         usage(B_FALSE);
5695                 }
5696         }
5697
5698         argc -= optind;
5699         argv += optind;
5700
5701         /* get pool name and check number of arguments */
5702         if (argc < 1) {
5703                 (void) fprintf(stderr, gettext("missing pool name\n"));
5704                 usage(B_FALSE);
5705         }
5706         if (argc < 2) {
5707                 (void) fprintf(stderr, gettext("missing device name\n"));
5708                 usage(B_FALSE);
5709         }
5710
5711         poolname = argv[0];
5712
5713         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5714                 return (1);
5715
5716         for (i = 1; i < argc; i++) {
5717                 if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) {
5718                         if (newstate != VDEV_STATE_HEALTHY) {
5719                                 (void) printf(gettext("warning: device '%s' "
5720                                     "onlined, but remains in faulted state\n"),
5721                                     argv[i]);
5722                                 if (newstate == VDEV_STATE_FAULTED)
5723                                         (void) printf(gettext("use 'zpool "
5724                                             "clear' to restore a faulted "
5725                                             "device\n"));
5726                                 else
5727                                         (void) printf(gettext("use 'zpool "
5728                                             "replace' to replace devices "
5729                                             "that are no longer present\n"));
5730                         }
5731                 } else {
5732                         ret = 1;
5733                 }
5734         }
5735
5736         zpool_close(zhp);
5737
5738         return (ret);
5739 }
5740
5741 /*
5742  * zpool offline [-ft] <pool> <device> ...
5743  *
5744  *      -f      Force the device into a faulted state.
5745  *
5746  *      -t      Only take the device off-line temporarily.  The offline/faulted
5747  *              state will not be persistent across reboots.
5748  */
5749 /* ARGSUSED */
5750 int
5751 zpool_do_offline(int argc, char **argv)
5752 {
5753         int c, i;
5754         char *poolname;
5755         zpool_handle_t *zhp;
5756         int ret = 0;
5757         boolean_t istmp = B_FALSE;
5758         boolean_t fault = B_FALSE;
5759
5760         /* check options */
5761         while ((c = getopt(argc, argv, "ft")) != -1) {
5762                 switch (c) {
5763                 case 'f':
5764                         fault = B_TRUE;
5765                         break;
5766                 case 't':
5767                         istmp = B_TRUE;
5768                         break;
5769                 case '?':
5770                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5771                             optopt);
5772                         usage(B_FALSE);
5773                 }
5774         }
5775
5776         argc -= optind;
5777         argv += optind;
5778
5779         /* get pool name and check number of arguments */
5780         if (argc < 1) {
5781                 (void) fprintf(stderr, gettext("missing pool name\n"));
5782                 usage(B_FALSE);
5783         }
5784         if (argc < 2) {
5785                 (void) fprintf(stderr, gettext("missing device name\n"));
5786                 usage(B_FALSE);
5787         }
5788
5789         poolname = argv[0];
5790
5791         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5792                 return (1);
5793
5794         for (i = 1; i < argc; i++) {
5795                 if (fault) {
5796                         uint64_t guid = zpool_vdev_path_to_guid(zhp, argv[i]);
5797                         vdev_aux_t aux;
5798                         if (istmp == B_FALSE) {
5799                                 /* Force the fault to persist across imports */
5800                                 aux = VDEV_AUX_EXTERNAL_PERSIST;
5801                         } else {
5802                                 aux = VDEV_AUX_EXTERNAL;
5803                         }
5804
5805                         if (guid == 0 || zpool_vdev_fault(zhp, guid, aux) != 0)
5806                                 ret = 1;
5807                 } else {
5808                         if (zpool_vdev_offline(zhp, argv[i], istmp) != 0)
5809                                 ret = 1;
5810                 }
5811         }
5812
5813         zpool_close(zhp);
5814
5815         return (ret);
5816 }
5817
5818 /*
5819  * zpool clear <pool> [device]
5820  *
5821  * Clear all errors associated with a pool or a particular device.
5822  */
5823 int
5824 zpool_do_clear(int argc, char **argv)
5825 {
5826         int c;
5827         int ret = 0;
5828         boolean_t dryrun = B_FALSE;
5829         boolean_t do_rewind = B_FALSE;
5830         boolean_t xtreme_rewind = B_FALSE;
5831         uint32_t rewind_policy = ZPOOL_NO_REWIND;
5832         nvlist_t *policy = NULL;
5833         zpool_handle_t *zhp;
5834         char *pool, *device;
5835
5836         /* check options */
5837         while ((c = getopt(argc, argv, "FnX")) != -1) {
5838                 switch (c) {
5839                 case 'F':
5840                         do_rewind = B_TRUE;
5841                         break;
5842                 case 'n':
5843                         dryrun = B_TRUE;
5844                         break;
5845                 case 'X':
5846                         xtreme_rewind = B_TRUE;
5847                         break;
5848                 case '?':
5849                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5850                             optopt);
5851                         usage(B_FALSE);
5852                 }
5853         }
5854
5855         argc -= optind;
5856         argv += optind;
5857
5858         if (argc < 1) {
5859                 (void) fprintf(stderr, gettext("missing pool name\n"));
5860                 usage(B_FALSE);
5861         }
5862
5863         if (argc > 2) {
5864                 (void) fprintf(stderr, gettext("too many arguments\n"));
5865                 usage(B_FALSE);
5866         }
5867
5868         if ((dryrun || xtreme_rewind) && !do_rewind) {
5869                 (void) fprintf(stderr,
5870                     gettext("-n or -X only meaningful with -F\n"));
5871                 usage(B_FALSE);
5872         }
5873         if (dryrun)
5874                 rewind_policy = ZPOOL_TRY_REWIND;
5875         else if (do_rewind)
5876                 rewind_policy = ZPOOL_DO_REWIND;
5877         if (xtreme_rewind)
5878                 rewind_policy |= ZPOOL_EXTREME_REWIND;
5879
5880         /* In future, further rewind policy choices can be passed along here */
5881         if (nvlist_alloc(&policy, NV_UNIQUE_NAME, 0) != 0 ||
5882             nvlist_add_uint32(policy, ZPOOL_LOAD_REWIND_POLICY,
5883             rewind_policy) != 0) {
5884                 return (1);
5885         }
5886
5887         pool = argv[0];
5888         device = argc == 2 ? argv[1] : NULL;
5889
5890         if ((zhp = zpool_open_canfail(g_zfs, pool)) == NULL) {
5891                 nvlist_free(policy);
5892                 return (1);
5893         }
5894
5895         if (zpool_clear(zhp, device, policy) != 0)
5896                 ret = 1;
5897
5898         zpool_close(zhp);
5899
5900         nvlist_free(policy);
5901
5902         return (ret);
5903 }
5904
5905 /*
5906  * zpool reguid <pool>
5907  */
5908 int
5909 zpool_do_reguid(int argc, char **argv)
5910 {
5911         int c;
5912         char *poolname;
5913         zpool_handle_t *zhp;
5914         int ret = 0;
5915
5916         /* check options */
5917         while ((c = getopt(argc, argv, "")) != -1) {
5918                 switch (c) {
5919                 case '?':
5920                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5921                             optopt);
5922                         usage(B_FALSE);
5923                 }
5924         }
5925
5926         argc -= optind;
5927         argv += optind;
5928
5929         /* get pool name and check number of arguments */
5930         if (argc < 1) {
5931                 (void) fprintf(stderr, gettext("missing pool name\n"));
5932                 usage(B_FALSE);
5933         }
5934
5935         if (argc > 1) {
5936                 (void) fprintf(stderr, gettext("too many arguments\n"));
5937                 usage(B_FALSE);
5938         }
5939
5940         poolname = argv[0];
5941         if ((zhp = zpool_open(g_zfs, poolname)) == NULL)
5942                 return (1);
5943
5944         ret = zpool_reguid(zhp);
5945
5946         zpool_close(zhp);
5947         return (ret);
5948 }
5949
5950
5951 /*
5952  * zpool reopen <pool>
5953  *
5954  * Reopen the pool so that the kernel can update the sizes of all vdevs.
5955  */
5956 int
5957 zpool_do_reopen(int argc, char **argv)
5958 {
5959         int c;
5960         int ret = 0;
5961         boolean_t scrub_restart = B_TRUE;
5962
5963         /* check options */
5964         while ((c = getopt(argc, argv, "n")) != -1) {
5965                 switch (c) {
5966                 case 'n':
5967                         scrub_restart = B_FALSE;
5968                         break;
5969                 case '?':
5970                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
5971                             optopt);
5972                         usage(B_FALSE);
5973                 }
5974         }
5975
5976         argc -= optind;
5977         argv += optind;
5978
5979         /* if argc == 0 we will execute zpool_reopen_one on all pools */
5980         ret = for_each_pool(argc, argv, B_TRUE, NULL, zpool_reopen_one,
5981             &scrub_restart);
5982
5983         return (ret);
5984 }
5985
5986 typedef struct scrub_cbdata {
5987         int     cb_type;
5988         int     cb_argc;
5989         char    **cb_argv;
5990         pool_scrub_cmd_t cb_scrub_cmd;
5991 } scrub_cbdata_t;
5992
5993 int
5994 scrub_callback(zpool_handle_t *zhp, void *data)
5995 {
5996         scrub_cbdata_t *cb = data;
5997         int err;
5998
5999         /*
6000          * Ignore faulted pools.
6001          */
6002         if (zpool_get_state(zhp) == POOL_STATE_UNAVAIL) {
6003                 (void) fprintf(stderr, gettext("cannot scrub '%s': pool is "
6004                     "currently unavailable\n"), zpool_get_name(zhp));
6005                 return (1);
6006         }
6007
6008         err = zpool_scan(zhp, cb->cb_type, cb->cb_scrub_cmd);
6009
6010         return (err != 0);
6011 }
6012
6013 /*
6014  * zpool scrub [-s | -p] <pool> ...
6015  *
6016  *      -s      Stop.  Stops any in-progress scrub.
6017  *      -p      Pause. Pause in-progress scrub.
6018  */
6019 int
6020 zpool_do_scrub(int argc, char **argv)
6021 {
6022         int c;
6023         scrub_cbdata_t cb;
6024
6025         cb.cb_type = POOL_SCAN_SCRUB;
6026         cb.cb_scrub_cmd = POOL_SCRUB_NORMAL;
6027
6028         /* check options */
6029         while ((c = getopt(argc, argv, "sp")) != -1) {
6030                 switch (c) {
6031                 case 's':
6032                         cb.cb_type = POOL_SCAN_NONE;
6033                         break;
6034                 case 'p':
6035                         cb.cb_scrub_cmd = POOL_SCRUB_PAUSE;
6036                         break;
6037                 case '?':
6038                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
6039                             optopt);
6040                         usage(B_FALSE);
6041                 }
6042         }
6043
6044         if (cb.cb_type == POOL_SCAN_NONE &&
6045             cb.cb_scrub_cmd == POOL_SCRUB_PAUSE) {
6046                 (void) fprintf(stderr, gettext("invalid option combination: "
6047                     "-s and -p are mutually exclusive\n"));
6048                 usage(B_FALSE);
6049         }
6050
6051         cb.cb_argc = argc;
6052         cb.cb_argv = argv;
6053         argc -= optind;
6054         argv += optind;
6055
6056         if (argc < 1) {
6057                 (void) fprintf(stderr, gettext("missing pool name argument\n"));
6058                 usage(B_FALSE);
6059         }
6060
6061         return (for_each_pool(argc, argv, B_TRUE, NULL, scrub_callback, &cb));
6062 }
6063
6064 /*
6065  * Print out detailed scrub status.
6066  */
6067 static void
6068 print_scan_status(pool_scan_stat_t *ps)
6069 {
6070         time_t start, end, pause;
6071         uint64_t total_secs_left;
6072         uint64_t elapsed, secs_left, mins_left, hours_left, days_left;
6073         uint64_t pass_scanned, scanned, pass_issued, issued, total;
6074         uint64_t scan_rate, issue_rate;
6075         double fraction_done;
6076         char processed_buf[7], scanned_buf[7], issued_buf[7], total_buf[7];
6077         char srate_buf[7], irate_buf[7];
6078
6079         (void) printf(gettext("  scan: "));
6080
6081         /* If there's never been a scan, there's not much to say. */
6082         if (ps == NULL || ps->pss_func == POOL_SCAN_NONE ||
6083             ps->pss_func >= POOL_SCAN_FUNCS) {
6084                 (void) printf(gettext("none requested\n"));
6085                 return;
6086         }
6087
6088         start = ps->pss_start_time;
6089         end = ps->pss_end_time;
6090         pause = ps->pss_pass_scrub_pause;
6091
6092         zfs_nicebytes(ps->pss_processed, processed_buf, sizeof (processed_buf));
6093
6094         assert(ps->pss_func == POOL_SCAN_SCRUB ||
6095             ps->pss_func == POOL_SCAN_RESILVER);
6096
6097         /* Scan is finished or canceled. */
6098         if (ps->pss_state == DSS_FINISHED) {
6099                 total_secs_left = end - start;
6100                 days_left = total_secs_left / 60 / 60 / 24;
6101                 hours_left = (total_secs_left / 60 / 60) % 24;
6102                 mins_left = (total_secs_left / 60) % 60;
6103                 secs_left = (total_secs_left % 60);
6104
6105                 if (ps->pss_func == POOL_SCAN_SCRUB) {
6106                         (void) printf(gettext("scrub repaired %s "
6107                             "in %llu days %02llu:%02llu:%02llu "
6108                             "with %llu errors on %s"), processed_buf,
6109                             (u_longlong_t)days_left, (u_longlong_t)hours_left,
6110                             (u_longlong_t)mins_left, (u_longlong_t)secs_left,
6111                             (u_longlong_t)ps->pss_errors, ctime(&end));
6112                 } else if (ps->pss_func == POOL_SCAN_RESILVER) {
6113                         (void) printf(gettext("resilvered %s "
6114                             "in %llu days %02llu:%02llu:%02llu "
6115                             "with %llu errors on %s"), processed_buf,
6116                             (u_longlong_t)days_left, (u_longlong_t)hours_left,
6117                             (u_longlong_t)mins_left, (u_longlong_t)secs_left,
6118                             (u_longlong_t)ps->pss_errors, ctime(&end));
6119                 }
6120                 return;
6121         } else if (ps->pss_state == DSS_CANCELED) {
6122                 if (ps->pss_func == POOL_SCAN_SCRUB) {
6123                         (void) printf(gettext("scrub canceled on %s"),
6124                             ctime(&end));
6125                 } else if (ps->pss_func == POOL_SCAN_RESILVER) {
6126                         (void) printf(gettext("resilver canceled on %s"),
6127                             ctime(&end));
6128                 }
6129                 return;
6130         }
6131
6132         assert(ps->pss_state == DSS_SCANNING);
6133
6134         /* Scan is in progress. Resilvers can't be paused. */
6135         if (ps->pss_func == POOL_SCAN_SCRUB) {
6136                 if (pause == 0) {
6137                         (void) printf(gettext("scrub in progress since %s"),
6138                             ctime(&start));
6139                 } else {
6140                         (void) printf(gettext("scrub paused since %s"),
6141                             ctime(&pause));
6142                         (void) printf(gettext("\tscrub started on %s"),
6143                             ctime(&start));
6144                 }
6145         } else if (ps->pss_func == POOL_SCAN_RESILVER) {
6146                 (void) printf(gettext("resilver in progress since %s"),
6147                     ctime(&start));
6148         }
6149
6150         scanned = ps->pss_examined;
6151         pass_scanned = ps->pss_pass_exam;
6152         issued = ps->pss_issued;
6153         pass_issued = ps->pss_pass_issued;
6154         total = ps->pss_to_examine;
6155
6156         /* we are only done with a block once we have issued the IO for it */
6157         fraction_done = (double)issued / total;
6158
6159         /* elapsed time for this pass, rounding up to 1 if it's 0 */
6160         elapsed = time(NULL) - ps->pss_pass_start;
6161         elapsed -= ps->pss_pass_scrub_spent_paused;
6162         elapsed = (elapsed != 0) ? elapsed : 1;
6163
6164         scan_rate = pass_scanned / elapsed;
6165         issue_rate = pass_issued / elapsed;
6166         total_secs_left = (issue_rate != 0) ?
6167             ((total - issued) / issue_rate) : UINT64_MAX;
6168
6169         days_left = total_secs_left / 60 / 60 / 24;
6170         hours_left = (total_secs_left / 60 / 60) % 24;
6171         mins_left = (total_secs_left / 60) % 60;
6172         secs_left = (total_secs_left % 60);
6173
6174         /* format all of the numbers we will be reporting */
6175         zfs_nicebytes(scanned, scanned_buf, sizeof (scanned_buf));
6176         zfs_nicebytes(issued, issued_buf, sizeof (issued_buf));
6177         zfs_nicebytes(total, total_buf, sizeof (total_buf));
6178         zfs_nicebytes(scan_rate, srate_buf, sizeof (srate_buf));
6179         zfs_nicebytes(issue_rate, irate_buf, sizeof (irate_buf));
6180
6181         /* do not print estimated time if we have a paused scrub */
6182         if (pause == 0) {
6183                 (void) printf(gettext("\t%s scanned at %s/s, "
6184                     "%s issued at %s/s, %s total\n"),
6185                     scanned_buf, srate_buf, issued_buf, irate_buf, total_buf);
6186         } else {
6187                 (void) printf(gettext("\t%s scanned, %s issued, %s total\n"),
6188                     scanned_buf, issued_buf, total_buf);
6189         }
6190
6191         if (ps->pss_func == POOL_SCAN_RESILVER) {
6192                 (void) printf(gettext("\t%s resilvered, %.2f%% done"),
6193                     processed_buf, 100 * fraction_done);
6194         } else if (ps->pss_func == POOL_SCAN_SCRUB) {
6195                 (void) printf(gettext("\t%s repaired, %.2f%% done"),
6196                     processed_buf, 100 * fraction_done);
6197         }
6198
6199         if (pause == 0) {
6200                 if (issue_rate >= 10 * 1024 * 1024) {
6201                         (void) printf(gettext(", %llu days "
6202                             "%02llu:%02llu:%02llu to go\n"),
6203                             (u_longlong_t)days_left, (u_longlong_t)hours_left,
6204                             (u_longlong_t)mins_left, (u_longlong_t)secs_left);
6205                 } else {
6206                         (void) printf(gettext(", no estimated "
6207                             "completion time\n"));
6208                 }
6209         } else {
6210                 (void) printf(gettext("\n"));
6211         }
6212 }
6213
6214 /*
6215  * Print out detailed removal status.
6216  */
6217 static void
6218 print_removal_status(zpool_handle_t *zhp, pool_removal_stat_t *prs)
6219 {
6220         char copied_buf[7], examined_buf[7], total_buf[7], rate_buf[7];
6221         time_t start, end;
6222         nvlist_t *config, *nvroot;
6223         nvlist_t **child;
6224         uint_t children;
6225         char *vdev_name;
6226
6227         if (prs == NULL || prs->prs_state == DSS_NONE)
6228                 return;
6229
6230         /*
6231          * Determine name of vdev.
6232          */
6233         config = zpool_get_config(zhp, NULL);
6234         nvroot = fnvlist_lookup_nvlist(config,
6235             ZPOOL_CONFIG_VDEV_TREE);
6236         verify(nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN,
6237             &child, &children) == 0);
6238         assert(prs->prs_removing_vdev < children);
6239         vdev_name = zpool_vdev_name(g_zfs, zhp,
6240             child[prs->prs_removing_vdev], B_TRUE);
6241
6242         (void) printf(gettext("remove: "));
6243
6244         start = prs->prs_start_time;
6245         end = prs->prs_end_time;
6246         zfs_nicenum(prs->prs_copied, copied_buf, sizeof (copied_buf));
6247
6248         /*
6249          * Removal is finished or canceled.
6250          */
6251         if (prs->prs_state == DSS_FINISHED) {
6252                 uint64_t minutes_taken = (end - start) / 60;
6253
6254                 (void) printf(gettext("Removal of vdev %llu copied %s "
6255                     "in %lluh%um, completed on %s"),
6256                     (longlong_t)prs->prs_removing_vdev,
6257                     copied_buf,
6258                     (u_longlong_t)(minutes_taken / 60),
6259                     (uint_t)(minutes_taken % 60),
6260                     ctime((time_t *)&end));
6261         } else if (prs->prs_state == DSS_CANCELED) {
6262                 (void) printf(gettext("Removal of %s canceled on %s"),
6263                     vdev_name, ctime(&end));
6264         } else {
6265                 uint64_t copied, total, elapsed, mins_left, hours_left;
6266                 double fraction_done;
6267                 uint_t rate;
6268
6269                 assert(prs->prs_state == DSS_SCANNING);
6270
6271                 /*
6272                  * Removal is in progress.
6273                  */
6274                 (void) printf(gettext(
6275                     "Evacuation of %s in progress since %s"),
6276                     vdev_name, ctime(&start));
6277
6278                 copied = prs->prs_copied > 0 ? prs->prs_copied : 1;
6279                 total = prs->prs_to_copy;
6280                 fraction_done = (double)copied / total;
6281
6282                 /* elapsed time for this pass */
6283                 elapsed = time(NULL) - prs->prs_start_time;
6284                 elapsed = elapsed > 0 ? elapsed : 1;
6285                 rate = copied / elapsed;
6286                 rate = rate > 0 ? rate : 1;
6287                 mins_left = ((total - copied) / rate) / 60;
6288                 hours_left = mins_left / 60;
6289
6290                 zfs_nicenum(copied, examined_buf, sizeof (examined_buf));
6291                 zfs_nicenum(total, total_buf, sizeof (total_buf));
6292                 zfs_nicenum(rate, rate_buf, sizeof (rate_buf));
6293
6294                 /*
6295                  * do not print estimated time if hours_left is more than
6296                  * 30 days
6297                  */
6298                 (void) printf(gettext("    %s copied out of %s at %s/s, "
6299                     "%.2f%% done"),
6300                     examined_buf, total_buf, rate_buf, 100 * fraction_done);
6301                 if (hours_left < (30 * 24)) {
6302                         (void) printf(gettext(", %lluh%um to go\n"),
6303                             (u_longlong_t)hours_left, (uint_t)(mins_left % 60));
6304                 } else {
6305                         (void) printf(gettext(
6306                             ", (copy is slow, no estimated time)\n"));
6307                 }
6308         }
6309
6310         if (prs->prs_mapping_memory > 0) {
6311                 char mem_buf[7];
6312                 zfs_nicenum(prs->prs_mapping_memory, mem_buf, sizeof (mem_buf));
6313                 (void) printf(gettext("    %s memory used for "
6314                     "removed device mappings\n"),
6315                     mem_buf);
6316         }
6317 }
6318
6319 static void
6320 print_error_log(zpool_handle_t *zhp)
6321 {
6322         nvlist_t *nverrlist = NULL;
6323         nvpair_t *elem;
6324         char *pathname;
6325         size_t len = MAXPATHLEN * 2;
6326
6327         if (zpool_get_errlog(zhp, &nverrlist) != 0)
6328                 return;
6329
6330         (void) printf("errors: Permanent errors have been "
6331             "detected in the following files:\n\n");
6332
6333         pathname = safe_malloc(len);
6334         elem = NULL;
6335         while ((elem = nvlist_next_nvpair(nverrlist, elem)) != NULL) {
6336                 nvlist_t *nv;
6337                 uint64_t dsobj, obj;
6338
6339                 verify(nvpair_value_nvlist(elem, &nv) == 0);
6340                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_DATASET,
6341                     &dsobj) == 0);
6342                 verify(nvlist_lookup_uint64(nv, ZPOOL_ERR_OBJECT,
6343                     &obj) == 0);
6344                 zpool_obj_to_path(zhp, dsobj, obj, pathname, len);
6345                 (void) printf("%7s %s\n", "", pathname);
6346         }
6347         free(pathname);
6348         nvlist_free(nverrlist);
6349 }
6350
6351 static void
6352 print_spares(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **spares,
6353     uint_t nspares)
6354 {
6355         uint_t i;
6356         char *name;
6357
6358         if (nspares == 0)
6359                 return;
6360
6361         (void) printf(gettext("\tspares\n"));
6362
6363         for (i = 0; i < nspares; i++) {
6364                 name = zpool_vdev_name(g_zfs, zhp, spares[i],
6365                     cb->cb_name_flags);
6366                 print_status_config(zhp, cb, name, spares[i], 2, B_TRUE);
6367                 free(name);
6368         }
6369 }
6370
6371 static void
6372 print_l2cache(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t **l2cache,
6373     uint_t nl2cache)
6374 {
6375         uint_t i;
6376         char *name;
6377
6378         if (nl2cache == 0)
6379                 return;
6380
6381         (void) printf(gettext("\tcache\n"));
6382
6383         for (i = 0; i < nl2cache; i++) {
6384                 name = zpool_vdev_name(g_zfs, zhp, l2cache[i],
6385                     cb->cb_name_flags);
6386                 print_status_config(zhp, cb, name, l2cache[i], 2, B_FALSE);
6387                 free(name);
6388         }
6389 }
6390
6391 static void
6392 print_dedup_stats(nvlist_t *config)
6393 {
6394         ddt_histogram_t *ddh;
6395         ddt_stat_t *dds;
6396         ddt_object_t *ddo;
6397         uint_t c;
6398         char dspace[6], mspace[6];
6399
6400         /*
6401          * If the pool was faulted then we may not have been able to
6402          * obtain the config. Otherwise, if we have anything in the dedup
6403          * table continue processing the stats.
6404          */
6405         if (nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS,
6406             (uint64_t **)&ddo, &c) != 0)
6407                 return;
6408
6409         (void) printf("\n");
6410         (void) printf(gettext(" dedup: "));
6411         if (ddo->ddo_count == 0) {
6412                 (void) printf(gettext("no DDT entries\n"));
6413                 return;
6414         }
6415
6416         zfs_nicebytes(ddo->ddo_dspace, dspace, sizeof (dspace));
6417         zfs_nicebytes(ddo->ddo_mspace, mspace, sizeof (mspace));
6418         (void) printf("DDT entries %llu, size %s on disk, %s in core\n",
6419             (u_longlong_t)ddo->ddo_count,
6420             dspace,
6421             mspace);
6422
6423         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_STATS,
6424             (uint64_t **)&dds, &c) == 0);
6425         verify(nvlist_lookup_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM,
6426             (uint64_t **)&ddh, &c) == 0);
6427         zpool_dump_ddt(dds, ddh);
6428 }
6429
6430 /*
6431  * Display a summary of pool status.  Displays a summary such as:
6432  *
6433  *        pool: tank
6434  *      status: DEGRADED
6435  *      reason: One or more devices ...
6436  *         see: http://zfsonlinux.org/msg/ZFS-xxxx-01
6437  *      config:
6438  *              mirror          DEGRADED
6439  *                c1t0d0        OK
6440  *                c2t0d0        UNAVAIL
6441  *
6442  * When given the '-v' option, we print out the complete config.  If the '-e'
6443  * option is specified, then we print out error rate information as well.
6444  */
6445 int
6446 status_callback(zpool_handle_t *zhp, void *data)
6447 {
6448         status_cbdata_t *cbp = data;
6449         nvlist_t *config, *nvroot;
6450         char *msgid;
6451         zpool_status_t reason;
6452         zpool_errata_t errata;
6453         const char *health;
6454         uint_t c;
6455         vdev_stat_t *vs;
6456
6457         config = zpool_get_config(zhp, NULL);
6458         reason = zpool_get_status(zhp, &msgid, &errata);
6459
6460         cbp->cb_count++;
6461
6462         /*
6463          * If we were given 'zpool status -x', only report those pools with
6464          * problems.
6465          */
6466         if (cbp->cb_explain &&
6467             (reason == ZPOOL_STATUS_OK ||
6468             reason == ZPOOL_STATUS_VERSION_OLDER ||
6469             reason == ZPOOL_STATUS_FEAT_DISABLED)) {
6470                 if (!cbp->cb_allpools) {
6471                         (void) printf(gettext("pool '%s' is healthy\n"),
6472                             zpool_get_name(zhp));
6473                         if (cbp->cb_first)
6474                                 cbp->cb_first = B_FALSE;
6475                 }
6476                 return (0);
6477         }
6478
6479         if (cbp->cb_first)
6480                 cbp->cb_first = B_FALSE;
6481         else
6482                 (void) printf("\n");
6483
6484         nvroot = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE);
6485         verify(nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS,
6486             (uint64_t **)&vs, &c) == 0);
6487
6488         health = zpool_get_state_str(zhp);
6489
6490         (void) printf(gettext("  pool: %s\n"), zpool_get_name(zhp));
6491         (void) printf(gettext(" state: %s\n"), health);
6492
6493         switch (reason) {
6494         case ZPOOL_STATUS_MISSING_DEV_R:
6495                 (void) printf(gettext("status: One or more devices could not "
6496                     "be opened.  Sufficient replicas exist for\n\tthe pool to "
6497                     "continue functioning in a degraded state.\n"));
6498                 (void) printf(gettext("action: Attach the missing device and "
6499                     "online it using 'zpool online'.\n"));
6500                 break;
6501
6502         case ZPOOL_STATUS_MISSING_DEV_NR:
6503                 (void) printf(gettext("status: One or more devices could not "
6504                     "be opened.  There are insufficient\n\treplicas for the "
6505                     "pool to continue functioning.\n"));
6506                 (void) printf(gettext("action: Attach the missing device and "
6507                     "online it using 'zpool online'.\n"));
6508                 break;
6509
6510         case ZPOOL_STATUS_CORRUPT_LABEL_R:
6511                 (void) printf(gettext("status: One or more devices could not "
6512                     "be used because the label is missing or\n\tinvalid.  "
6513                     "Sufficient replicas exist for the pool to continue\n\t"
6514                     "functioning in a degraded state.\n"));
6515                 (void) printf(gettext("action: Replace the device using "
6516                     "'zpool replace'.\n"));
6517                 break;
6518
6519         case ZPOOL_STATUS_CORRUPT_LABEL_NR:
6520                 (void) printf(gettext("status: One or more devices could not "
6521                     "be used because the label is missing \n\tor invalid.  "
6522                     "There are insufficient replicas for the pool to "
6523                     "continue\n\tfunctioning.\n"));
6524                 zpool_explain_recover(zpool_get_handle(zhp),
6525                     zpool_get_name(zhp), reason, config);
6526                 break;
6527
6528         case ZPOOL_STATUS_FAILING_DEV:
6529                 (void) printf(gettext("status: One or more devices has "
6530                     "experienced an unrecoverable error.  An\n\tattempt was "
6531                     "made to correct the error.  Applications are "
6532                     "unaffected.\n"));
6533                 (void) printf(gettext("action: Determine if the device needs "
6534                     "to be replaced, and clear the errors\n\tusing "
6535                     "'zpool clear' or replace the device with 'zpool "
6536                     "replace'.\n"));
6537                 break;
6538
6539         case ZPOOL_STATUS_OFFLINE_DEV:
6540                 (void) printf(gettext("status: One or more devices has "
6541                     "been taken offline by the administrator.\n\tSufficient "
6542                     "replicas exist for the pool to continue functioning in "
6543                     "a\n\tdegraded state.\n"));
6544                 (void) printf(gettext("action: Online the device using "
6545                     "'zpool online' or replace the device with\n\t'zpool "
6546                     "replace'.\n"));
6547                 break;
6548
6549         case ZPOOL_STATUS_REMOVED_DEV:
6550                 (void) printf(gettext("status: One or more devices has "
6551                     "been removed by the administrator.\n\tSufficient "
6552                     "replicas exist for the pool to continue functioning in "
6553                     "a\n\tdegraded state.\n"));
6554                 (void) printf(gettext("action: Online the device using "
6555                     "'zpool online' or replace the device with\n\t'zpool "
6556                     "replace'.\n"));
6557                 break;
6558
6559         case ZPOOL_STATUS_RESILVERING:
6560                 (void) printf(gettext("status: One or more devices is "
6561                     "currently being resilvered.  The pool will\n\tcontinue "
6562                     "to function, possibly in a degraded state.\n"));
6563                 (void) printf(gettext("action: Wait for the resilver to "
6564                     "complete.\n"));
6565                 break;
6566
6567         case ZPOOL_STATUS_CORRUPT_DATA:
6568                 (void) printf(gettext("status: One or more devices has "
6569                     "experienced an error resulting in data\n\tcorruption.  "
6570                     "Applications may be affected.\n"));
6571                 (void) printf(gettext("action: Restore the file in question "
6572                     "if possible.  Otherwise restore the\n\tentire pool from "
6573                     "backup.\n"));
6574                 break;
6575
6576         case ZPOOL_STATUS_CORRUPT_POOL:
6577                 (void) printf(gettext("status: The pool metadata is corrupted "
6578                     "and the pool cannot be opened.\n"));
6579                 zpool_explain_recover(zpool_get_handle(zhp),
6580                     zpool_get_name(zhp), reason, config);
6581                 break;
6582
6583         case ZPOOL_STATUS_VERSION_OLDER:
6584                 (void) printf(gettext("status: The pool is formatted using a "
6585                     "legacy on-disk format.  The pool can\n\tstill be used, "
6586                     "but some features are unavailable.\n"));
6587                 (void) printf(gettext("action: Upgrade the pool using 'zpool "
6588                     "upgrade'.  Once this is done, the\n\tpool will no longer "
6589                     "be accessible on software that does not support\n\t"
6590                     "feature flags.\n"));
6591                 break;
6592
6593         case ZPOOL_STATUS_VERSION_NEWER:
6594                 (void) printf(gettext("status: The pool has been upgraded to a "
6595                     "newer, incompatible on-disk version.\n\tThe pool cannot "
6596                     "be accessed on this system.\n"));
6597                 (void) printf(gettext("action: Access the pool from a system "
6598                     "running more recent software, or\n\trestore the pool from "
6599                     "backup.\n"));
6600                 break;
6601
6602         case ZPOOL_STATUS_FEAT_DISABLED:
6603                 (void) printf(gettext("status: Some supported features are not "
6604                     "enabled on the pool. The pool can\n\tstill be used, but "
6605                     "some features are unavailable.\n"));
6606                 (void) printf(gettext("action: Enable all features using "
6607                     "'zpool upgrade'. Once this is done,\n\tthe pool may no "
6608                     "longer be accessible by software that does not support\n\t"
6609                     "the features. See zpool-features(5) for details.\n"));
6610                 break;
6611
6612         case ZPOOL_STATUS_UNSUP_FEAT_READ:
6613                 (void) printf(gettext("status: The pool cannot be accessed on "
6614                     "this system because it uses the\n\tfollowing feature(s) "
6615                     "not supported on this system:\n"));
6616                 zpool_print_unsup_feat(config);
6617                 (void) printf("\n");
6618                 (void) printf(gettext("action: Access the pool from a system "
6619                     "that supports the required feature(s),\n\tor restore the "
6620                     "pool from backup.\n"));
6621                 break;
6622
6623         case ZPOOL_STATUS_UNSUP_FEAT_WRITE:
6624                 (void) printf(gettext("status: The pool can only be accessed "
6625                     "in read-only mode on this system. It\n\tcannot be "
6626                     "accessed in read-write mode because it uses the "
6627                     "following\n\tfeature(s) not supported on this system:\n"));
6628                 zpool_print_unsup_feat(config);
6629                 (void) printf("\n");
6630                 (void) printf(gettext("action: The pool cannot be accessed in "
6631                     "read-write mode. Import the pool with\n"
6632                     "\t\"-o readonly=on\", access the pool from a system that "
6633                     "supports the\n\trequired feature(s), or restore the "
6634                     "pool from backup.\n"));
6635                 break;
6636
6637         case ZPOOL_STATUS_FAULTED_DEV_R:
6638                 (void) printf(gettext("status: One or more devices are "
6639                     "faulted in response to persistent errors.\n\tSufficient "
6640                     "replicas exist for the pool to continue functioning "
6641                     "in a\n\tdegraded state.\n"));
6642                 (void) printf(gettext("action: Replace the faulted device, "
6643                     "or use 'zpool clear' to mark the device\n\trepaired.\n"));
6644                 break;
6645
6646         case ZPOOL_STATUS_FAULTED_DEV_NR:
6647                 (void) printf(gettext("status: One or more devices are "
6648                     "faulted in response to persistent errors.  There are "
6649                     "insufficient replicas for the pool to\n\tcontinue "
6650                     "functioning.\n"));
6651                 (void) printf(gettext("action: Destroy and re-create the pool "
6652                     "from a backup source.  Manually marking the device\n"
6653                     "\trepaired using 'zpool clear' may allow some data "
6654                     "to be recovered.\n"));
6655                 break;
6656
6657         case ZPOOL_STATUS_IO_FAILURE_MMP:
6658                 (void) printf(gettext("status: The pool is suspended because "
6659                     "multihost writes failed or were delayed;\n\tanother "
6660                     "system could import the pool undetected.\n"));
6661                 (void) printf(gettext("action: Make sure the pool's devices "
6662                     "are connected, then reboot your system and\n\timport the "
6663                     "pool.\n"));
6664                 break;
6665
6666         case ZPOOL_STATUS_IO_FAILURE_WAIT:
6667         case ZPOOL_STATUS_IO_FAILURE_CONTINUE:
6668                 (void) printf(gettext("status: One or more devices are "
6669                     "faulted in response to IO failures.\n"));
6670                 (void) printf(gettext("action: Make sure the affected devices "
6671                     "are connected, then run 'zpool clear'.\n"));
6672                 break;
6673
6674         case ZPOOL_STATUS_BAD_LOG:
6675                 (void) printf(gettext("status: An intent log record "
6676                     "could not be read.\n"
6677                     "\tWaiting for administrator intervention to fix the "
6678                     "faulted pool.\n"));
6679                 (void) printf(gettext("action: Either restore the affected "
6680                     "device(s) and run 'zpool online',\n"
6681                     "\tor ignore the intent log records by running "
6682                     "'zpool clear'.\n"));
6683                 break;
6684
6685         case ZPOOL_STATUS_HOSTID_MISMATCH:
6686                 (void) printf(gettext("status: Mismatch between pool hostid "
6687                     "and system hostid on imported pool.\n\tThis pool was "
6688                     "previously imported into a system with a different "
6689                     "hostid,\n\tand then was verbatim imported into this "
6690                     "system.\n"));
6691                 (void) printf(gettext("action: Export this pool on all systems "
6692                     "on which it is imported.\n"
6693                     "\tThen import it to correct the mismatch.\n"));
6694                 break;
6695
6696         case ZPOOL_STATUS_ERRATA:
6697                 (void) printf(gettext("status: Errata #%d detected.\n"),
6698                     errata);
6699
6700                 switch (errata) {
6701                 case ZPOOL_ERRATA_NONE:
6702                         break;
6703
6704                 case ZPOOL_ERRATA_ZOL_2094_SCRUB:
6705                         (void) printf(gettext("action: To correct the issue "
6706                             "run 'zpool scrub'.\n"));
6707                         break;
6708
6709                 case ZPOOL_ERRATA_ZOL_6845_ENCRYPTION:
6710                         (void) printf(gettext("\tExisting encrypted datasets "
6711                             "contain an on-disk incompatibility\n\twhich "
6712                             "needs to be corrected.\n"));
6713                         (void) printf(gettext("action: To correct the issue "
6714                             "backup existing encrypted datasets to new\n\t"
6715                             "encrypted datasets and destroy the old ones. "
6716                             "'zfs mount -o ro' can\n\tbe used to temporarily "
6717                             "mount existing encrypted datasets readonly.\n"));
6718                         break;
6719
6720                 default:
6721                         /*
6722                          * All errata which allow the pool to be imported
6723                          * must contain an action message.
6724                          */
6725                         assert(0);
6726                 }
6727                 break;
6728
6729         default:
6730                 /*
6731                  * The remaining errors can't actually be generated, yet.
6732                  */
6733                 assert(reason == ZPOOL_STATUS_OK);
6734         }
6735
6736         if (msgid != NULL)
6737                 (void) printf(gettext("   see: http://zfsonlinux.org/msg/%s\n"),
6738                     msgid);
6739
6740         if (config != NULL) {
6741                 uint64_t nerr;
6742                 nvlist_t **spares, **l2cache;
6743                 uint_t nspares, nl2cache;
6744                 pool_scan_stat_t *ps = NULL;
6745                 pool_removal_stat_t *prs = NULL;
6746
6747                 (void) nvlist_lookup_uint64_array(nvroot,
6748                     ZPOOL_CONFIG_SCAN_STATS, (uint64_t **)&ps, &c);
6749                 print_scan_status(ps);
6750
6751                 (void) nvlist_lookup_uint64_array(nvroot,
6752                     ZPOOL_CONFIG_REMOVAL_STATS, (uint64_t **)&prs, &c);
6753                 print_removal_status(zhp, prs);
6754
6755                 cbp->cb_namewidth = max_width(zhp, nvroot, 0, 0,
6756                     cbp->cb_name_flags | VDEV_NAME_TYPE_ID);
6757                 if (cbp->cb_namewidth < 10)
6758                         cbp->cb_namewidth = 10;
6759
6760                 (void) printf(gettext("config:\n\n"));
6761                 (void) printf(gettext("\t%-*s  %-8s %5s %5s %5s"),
6762                     cbp->cb_namewidth, "NAME", "STATE", "READ", "WRITE",
6763                     "CKSUM");
6764
6765                 if (cbp->vcdl != NULL)
6766                         print_cmd_columns(cbp->vcdl, 0);
6767
6768                 printf("\n");
6769                 print_status_config(zhp, cbp, zpool_get_name(zhp), nvroot, 0,
6770                     B_FALSE);
6771
6772                 if (num_logs(nvroot) > 0)
6773                         print_logs(zhp, cbp, nvroot);
6774                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE,
6775                     &l2cache, &nl2cache) == 0)
6776                         print_l2cache(zhp, cbp, l2cache, nl2cache);
6777
6778                 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
6779                     &spares, &nspares) == 0)
6780                         print_spares(zhp, cbp, spares, nspares);
6781
6782                 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_ERRCOUNT,
6783                     &nerr) == 0) {
6784                         nvlist_t *nverrlist = NULL;
6785
6786                         /*
6787                          * If the approximate error count is small, get a
6788                          * precise count by fetching the entire log and
6789                          * uniquifying the results.
6790                          */
6791                         if (nerr > 0 && nerr < 100 && !cbp->cb_verbose &&
6792                             zpool_get_errlog(zhp, &nverrlist) == 0) {
6793                                 nvpair_t *elem;
6794
6795                                 elem = NULL;
6796                                 nerr = 0;
6797                                 while ((elem = nvlist_next_nvpair(nverrlist,
6798                                     elem)) != NULL) {
6799                                         nerr++;
6800                                 }
6801                         }
6802                         nvlist_free(nverrlist);
6803
6804                         (void) printf("\n");
6805
6806                         if (nerr == 0)
6807                                 (void) printf(gettext("errors: No known data "
6808                                     "errors\n"));
6809                         else if (!cbp->cb_verbose)
6810                                 (void) printf(gettext("errors: %llu data "
6811                                     "errors, use '-v' for a list\n"),
6812                                     (u_longlong_t)nerr);
6813                         else
6814                                 print_error_log(zhp);
6815                 }
6816
6817                 if (cbp->cb_dedup_stats)
6818                         print_dedup_stats(config);
6819         } else {
6820                 (void) printf(gettext("config: The configuration cannot be "
6821                     "determined.\n"));
6822         }
6823
6824         return (0);
6825 }
6826
6827 /*
6828  * zpool status [-c [script1,script2,...]] [-gLPvx] [-T d|u] [pool] ...
6829  *              [interval [count]]
6830  *
6831  *      -c CMD  For each vdev, run command CMD
6832  *      -g      Display guid for individual vdev name.
6833  *      -L      Follow links when resolving vdev path name.
6834  *      -P      Display full path for vdev name.
6835  *      -v      Display complete error logs
6836  *      -x      Display only pools with potential problems
6837  *      -D      Display dedup status (undocumented)
6838  *      -T      Display a timestamp in date(1) or Unix format
6839  *
6840  * Describes the health status of all pools or some subset.
6841  */
6842 int
6843 zpool_do_status(int argc, char **argv)
6844 {
6845         int c;
6846         int ret;
6847         float interval = 0;
6848         unsigned long count = 0;
6849         status_cbdata_t cb = { 0 };
6850         char *cmd = NULL;
6851
6852         /* check options */
6853         while ((c = getopt(argc, argv, "c:gLPvxDT:")) != -1) {
6854                 switch (c) {
6855                 case 'c':
6856                         if (cmd != NULL) {
6857                                 fprintf(stderr,
6858                                     gettext("Can't set -c flag twice\n"));
6859                                 exit(1);
6860                         }
6861
6862                         if (getenv("ZPOOL_SCRIPTS_ENABLED") != NULL &&
6863                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_ENABLED")) {
6864                                 fprintf(stderr, gettext(
6865                                     "Can't run -c, disabled by "
6866                                     "ZPOOL_SCRIPTS_ENABLED.\n"));
6867                                 exit(1);
6868                         }
6869
6870                         if ((getuid() <= 0 || geteuid() <= 0) &&
6871                             !libzfs_envvar_is_set("ZPOOL_SCRIPTS_AS_ROOT")) {
6872                                 fprintf(stderr, gettext(
6873                                     "Can't run -c with root privileges "
6874                                     "unless ZPOOL_SCRIPTS_AS_ROOT is set.\n"));
6875                                 exit(1);
6876                         }
6877                         cmd = optarg;
6878                         break;
6879                 case 'g':
6880                         cb.cb_name_flags |= VDEV_NAME_GUID;
6881                         break;
6882                 case 'L':
6883                         cb.cb_name_flags |= VDEV_NAME_FOLLOW_LINKS;
6884                         break;
6885                 case 'P':
6886                         cb.cb_name_flags |= VDEV_NAME_PATH;
6887                         break;
6888                 case 'v':
6889                         cb.cb_verbose = B_TRUE;
6890                         break;
6891                 case 'x':
6892                         cb.cb_explain = B_TRUE;
6893                         break;
6894                 case 'D':
6895                         cb.cb_dedup_stats = B_TRUE;
6896                         break;
6897                 case 'T':
6898                         get_timestamp_arg(*optarg);
6899                         break;
6900                 case '?':
6901                         if (optopt == 'c') {
6902                                 print_zpool_script_list("status");
6903                                 exit(0);
6904                         } else {
6905                                 fprintf(stderr,
6906                                     gettext("invalid option '%c'\n"), optopt);
6907                         }
6908                         usage(B_FALSE);
6909                 }
6910         }
6911
6912         argc -= optind;
6913         argv += optind;
6914
6915         get_interval_count(&argc, argv, &interval, &count);
6916
6917         if (argc == 0)
6918                 cb.cb_allpools = B_TRUE;
6919
6920         cb.cb_first = B_TRUE;
6921         cb.cb_print_status = B_TRUE;
6922
6923         for (;;) {
6924                 if (timestamp_fmt != NODATE)
6925                         print_timestamp(timestamp_fmt);
6926
6927                 if (cmd != NULL)
6928                         cb.vcdl = all_pools_for_each_vdev_run(argc, argv, cmd,
6929                             NULL, NULL, 0, 0);
6930
6931                 ret = for_each_pool(argc, argv, B_TRUE, NULL,
6932                     status_callback, &cb);
6933
6934                 if (cb.vcdl != NULL)
6935                         free_vdev_cmd_data_list(cb.vcdl);
6936
6937                 if (argc == 0 && cb.cb_count == 0)
6938                         (void) fprintf(stderr, gettext("no pools available\n"));
6939                 else if (cb.cb_explain && cb.cb_first && cb.cb_allpools)
6940                         (void) printf(gettext("all pools are healthy\n"));
6941
6942                 if (ret != 0)
6943                         return (ret);
6944
6945                 if (interval == 0)
6946                         break;
6947
6948                 if (count != 0 && --count == 0)
6949                         break;
6950
6951                 (void) fsleep(interval);
6952         }
6953
6954         return (0);
6955 }
6956
6957 typedef struct upgrade_cbdata {
6958         int     cb_first;
6959         int     cb_argc;
6960         uint64_t cb_version;
6961         char    **cb_argv;
6962 } upgrade_cbdata_t;
6963
6964 static int
6965 check_unsupp_fs(zfs_handle_t *zhp, void *unsupp_fs)
6966 {
6967         int zfs_version = (int)zfs_prop_get_int(zhp, ZFS_PROP_VERSION);
6968         int *count = (int *)unsupp_fs;
6969
6970         if (zfs_version > ZPL_VERSION) {
6971                 (void) printf(gettext("%s (v%d) is not supported by this "
6972                     "implementation of ZFS.\n"),
6973                     zfs_get_name(zhp), zfs_version);
6974                 (*count)++;
6975         }
6976
6977         zfs_iter_filesystems(zhp, check_unsupp_fs, unsupp_fs);
6978
6979         zfs_close(zhp);
6980
6981         return (0);
6982 }
6983
6984 static int
6985 upgrade_version(zpool_handle_t *zhp, uint64_t version)
6986 {
6987         int ret;
6988         nvlist_t *config;
6989         uint64_t oldversion;
6990         int unsupp_fs = 0;
6991
6992         config = zpool_get_config(zhp, NULL);
6993         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
6994             &oldversion) == 0);
6995
6996         assert(SPA_VERSION_IS_SUPPORTED(oldversion));
6997         assert(oldversion < version);
6998
6999         ret = zfs_iter_root(zpool_get_handle(zhp), check_unsupp_fs, &unsupp_fs);
7000         if (ret != 0)
7001                 return (ret);
7002
7003         if (unsupp_fs) {
7004                 (void) fprintf(stderr, gettext("Upgrade not performed due "
7005                     "to %d unsupported filesystems (max v%d).\n"),
7006                     unsupp_fs, (int)ZPL_VERSION);
7007                 return (1);
7008         }
7009
7010         ret = zpool_upgrade(zhp, version);
7011         if (ret != 0)
7012                 return (ret);
7013
7014         if (version >= SPA_VERSION_FEATURES) {
7015                 (void) printf(gettext("Successfully upgraded "
7016                     "'%s' from version %llu to feature flags.\n"),
7017                     zpool_get_name(zhp), (u_longlong_t)oldversion);
7018         } else {
7019                 (void) printf(gettext("Successfully upgraded "
7020                     "'%s' from version %llu to version %llu.\n"),
7021                     zpool_get_name(zhp), (u_longlong_t)oldversion,
7022                     (u_longlong_t)version);
7023         }
7024
7025         return (0);
7026 }
7027
7028 static int
7029 upgrade_enable_all(zpool_handle_t *zhp, int *countp)
7030 {
7031         int i, ret, count;
7032         boolean_t firstff = B_TRUE;
7033         nvlist_t *enabled = zpool_get_features(zhp);
7034
7035         count = 0;
7036         for (i = 0; i < SPA_FEATURES; i++) {
7037                 const char *fname = spa_feature_table[i].fi_uname;
7038                 const char *fguid = spa_feature_table[i].fi_guid;
7039                 if (!nvlist_exists(enabled, fguid)) {
7040                         char *propname;
7041                         verify(-1 != asprintf(&propname, "feature@%s", fname));
7042                         ret = zpool_set_prop(zhp, propname,
7043                             ZFS_FEATURE_ENABLED);
7044                         if (ret != 0) {
7045                                 free(propname);
7046                                 return (ret);
7047                         }
7048                         count++;
7049
7050                         if (firstff) {
7051                                 (void) printf(gettext("Enabled the "
7052                                     "following features on '%s':\n"),
7053                                     zpool_get_name(zhp));
7054                                 firstff = B_FALSE;
7055                         }
7056                         (void) printf(gettext("  %s\n"), fname);
7057                         free(propname);
7058                 }
7059         }
7060
7061         if (countp != NULL)
7062                 *countp = count;
7063         return (0);
7064 }
7065
7066 static int
7067 upgrade_cb(zpool_handle_t *zhp, void *arg)
7068 {
7069         upgrade_cbdata_t *cbp = arg;
7070         nvlist_t *config;
7071         uint64_t version;
7072         boolean_t printnl = B_FALSE;
7073         int ret;
7074
7075         config = zpool_get_config(zhp, NULL);
7076         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7077             &version) == 0);
7078
7079         assert(SPA_VERSION_IS_SUPPORTED(version));
7080
7081         if (version < cbp->cb_version) {
7082                 cbp->cb_first = B_FALSE;
7083                 ret = upgrade_version(zhp, cbp->cb_version);
7084                 if (ret != 0)
7085                         return (ret);
7086                 printnl = B_TRUE;
7087
7088                 /*
7089                  * If they did "zpool upgrade -a", then we could
7090                  * be doing ioctls to different pools.  We need
7091                  * to log this history once to each pool, and bypass
7092                  * the normal history logging that happens in main().
7093                  */
7094                 (void) zpool_log_history(g_zfs, history_str);
7095                 log_history = B_FALSE;
7096         }
7097
7098         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
7099                 int count;
7100                 ret = upgrade_enable_all(zhp, &count);
7101                 if (ret != 0)
7102                         return (ret);
7103
7104                 if (count > 0) {
7105                         cbp->cb_first = B_FALSE;
7106                         printnl = B_TRUE;
7107                 }
7108         }
7109
7110         if (printnl) {
7111                 (void) printf(gettext("\n"));
7112         }
7113
7114         return (0);
7115 }
7116
7117 static int
7118 upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
7119 {
7120         upgrade_cbdata_t *cbp = arg;
7121         nvlist_t *config;
7122         uint64_t version;
7123
7124         config = zpool_get_config(zhp, NULL);
7125         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7126             &version) == 0);
7127
7128         assert(SPA_VERSION_IS_SUPPORTED(version));
7129
7130         if (version < SPA_VERSION_FEATURES) {
7131                 if (cbp->cb_first) {
7132                         (void) printf(gettext("The following pools are "
7133                             "formatted with legacy version numbers and can\n"
7134                             "be upgraded to use feature flags.  After "
7135                             "being upgraded, these pools\nwill no "
7136                             "longer be accessible by software that does not "
7137                             "support feature\nflags.\n\n"));
7138                         (void) printf(gettext("VER  POOL\n"));
7139                         (void) printf(gettext("---  ------------\n"));
7140                         cbp->cb_first = B_FALSE;
7141                 }
7142
7143                 (void) printf("%2llu   %s\n", (u_longlong_t)version,
7144                     zpool_get_name(zhp));
7145         }
7146
7147         return (0);
7148 }
7149
7150 static int
7151 upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
7152 {
7153         upgrade_cbdata_t *cbp = arg;
7154         nvlist_t *config;
7155         uint64_t version;
7156
7157         config = zpool_get_config(zhp, NULL);
7158         verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
7159             &version) == 0);
7160
7161         if (version >= SPA_VERSION_FEATURES) {
7162                 int i;
7163                 boolean_t poolfirst = B_TRUE;
7164                 nvlist_t *enabled = zpool_get_features(zhp);
7165
7166                 for (i = 0; i < SPA_FEATURES; i++) {
7167                         const char *fguid = spa_feature_table[i].fi_guid;
7168                         const char *fname = spa_feature_table[i].fi_uname;
7169                         if (!nvlist_exists(enabled, fguid)) {
7170                                 if (cbp->cb_first) {
7171                                         (void) printf(gettext("\nSome "
7172                                             "supported features are not "
7173                                             "enabled on the following pools. "
7174                                             "Once a\nfeature is enabled the "
7175                                             "pool may become incompatible with "
7176                                             "software\nthat does not support "
7177                                             "the feature. See "
7178                                             "zpool-features(5) for "
7179                                             "details.\n\n"));
7180                                         (void) printf(gettext("POOL  "
7181                                             "FEATURE\n"));
7182                                         (void) printf(gettext("------"
7183                                             "---------\n"));
7184                                         cbp->cb_first = B_FALSE;
7185                                 }
7186
7187                                 if (poolfirst) {
7188                                         (void) printf(gettext("%s\n"),
7189                                             zpool_get_name(zhp));
7190                                         poolfirst = B_FALSE;
7191                                 }
7192
7193                                 (void) printf(gettext("      %s\n"), fname);
7194                         }
7195                         /*
7196                          * If they did "zpool upgrade -a", then we could
7197                          * be doing ioctls to different pools.  We need
7198                          * to log this history once to each pool, and bypass
7199                          * the normal history logging that happens in main().
7200                          */
7201                         (void) zpool_log_history(g_zfs, history_str);
7202                         log_history = B_FALSE;
7203                 }
7204         }
7205
7206         return (0);
7207 }
7208
7209 /* ARGSUSED */
7210 static int
7211 upgrade_one(zpool_handle_t *zhp, void *data)
7212 {
7213         boolean_t printnl = B_FALSE;
7214         upgrade_cbdata_t *cbp = data;
7215         uint64_t cur_version;
7216         int ret;
7217
7218         if (strcmp("log", zpool_get_name(zhp)) == 0) {
7219                 (void) fprintf(stderr, gettext("'log' is now a reserved word\n"
7220                     "Pool 'log' must be renamed using export and import"
7221                     " to upgrade.\n"));
7222                 return (1);
7223         }
7224
7225         cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
7226         if (cur_version > cbp->cb_version) {
7227                 (void) printf(gettext("Pool '%s' is already formatted "
7228                     "using more current version '%llu'.\n\n"),
7229                     zpool_get_name(zhp), (u_longlong_t)cur_version);
7230                 return (0);
7231         }
7232
7233         if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
7234                 (void) printf(gettext("Pool '%s' is already formatted "
7235                     "using version %llu.\n\n"), zpool_get_name(zhp),
7236                     (u_longlong_t)cbp->cb_version);
7237                 return (0);
7238         }
7239
7240         if (cur_version != cbp->cb_version) {
7241                 printnl = B_TRUE;
7242                 ret = upgrade_version(zhp, cbp->cb_version);
7243                 if (ret != 0)
7244                         return (ret);
7245         }
7246
7247         if (cbp->cb_version >= SPA_VERSION_FEATURES) {
7248                 int count = 0;
7249                 ret = upgrade_enable_all(zhp, &count);
7250                 if (ret != 0)
7251                         return (ret);
7252
7253                 if (count != 0) {
7254                         printnl = B_TRUE;
7255                 } else if (cur_version == SPA_VERSION) {
7256                         (void) printf(gettext("Pool '%s' already has all "
7257                             "supported features enabled.\n"),
7258                             zpool_get_name(zhp));
7259                 }
7260         }
7261
7262         if (printnl) {
7263                 (void) printf(gettext("\n"));
7264         }
7265
7266         return (0);
7267 }
7268
7269 /*
7270  * zpool upgrade
7271  * zpool upgrade -v
7272  * zpool upgrade [-V version] <-a | pool ...>
7273  *
7274  * With no arguments, display downrev'd ZFS pool available for upgrade.
7275  * Individual pools can be upgraded by specifying the pool, and '-a' will
7276  * upgrade all pools.
7277  */
7278 int
7279 zpool_do_upgrade(int argc, char **argv)
7280 {
7281         int c;
7282         upgrade_cbdata_t cb = { 0 };
7283         int ret = 0;
7284         boolean_t showversions = B_FALSE;
7285         boolean_t upgradeall = B_FALSE;
7286         char *end;
7287
7288
7289         /* check options */
7290         while ((c = getopt(argc, argv, ":avV:")) != -1) {
7291                 switch (c) {
7292                 case 'a':
7293                         upgradeall = B_TRUE;
7294                         break;
7295                 case 'v':
7296                         showversions = B_TRUE;
7297                         break;
7298                 case 'V':
7299                         cb.cb_version = strtoll(optarg, &end, 10);
7300                         if (*end != '\0' ||
7301                             !SPA_VERSION_IS_SUPPORTED(cb.cb_version)) {
7302                                 (void) fprintf(stderr,
7303                                     gettext("invalid version '%s'\n"), optarg);
7304                                 usage(B_FALSE);
7305                         }
7306                         break;
7307                 case ':':
7308                         (void) fprintf(stderr, gettext("missing argument for "
7309                             "'%c' option\n"), optopt);
7310                         usage(B_FALSE);
7311                         break;
7312                 case '?':
7313                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7314                             optopt);
7315                         usage(B_FALSE);
7316                 }
7317         }
7318
7319         cb.cb_argc = argc;
7320         cb.cb_argv = argv;
7321         argc -= optind;
7322         argv += optind;
7323
7324         if (cb.cb_version == 0) {
7325                 cb.cb_version = SPA_VERSION;
7326         } else if (!upgradeall && argc == 0) {
7327                 (void) fprintf(stderr, gettext("-V option is "
7328                     "incompatible with other arguments\n"));
7329                 usage(B_FALSE);
7330         }
7331
7332         if (showversions) {
7333                 if (upgradeall || argc != 0) {
7334                         (void) fprintf(stderr, gettext("-v option is "
7335                             "incompatible with other arguments\n"));
7336                         usage(B_FALSE);
7337                 }
7338         } else if (upgradeall) {
7339                 if (argc != 0) {
7340                         (void) fprintf(stderr, gettext("-a option should not "
7341                             "be used along with a pool name\n"));
7342                         usage(B_FALSE);
7343                 }
7344         }
7345
7346         (void) printf(gettext("This system supports ZFS pool feature "
7347             "flags.\n\n"));
7348         if (showversions) {
7349                 int i;
7350
7351                 (void) printf(gettext("The following features are "
7352                     "supported:\n\n"));
7353                 (void) printf(gettext("FEAT DESCRIPTION\n"));
7354                 (void) printf("----------------------------------------------"
7355                     "---------------\n");
7356                 for (i = 0; i < SPA_FEATURES; i++) {
7357                         zfeature_info_t *fi = &spa_feature_table[i];
7358                         const char *ro =
7359                             (fi->fi_flags & ZFEATURE_FLAG_READONLY_COMPAT) ?
7360                             " (read-only compatible)" : "";
7361
7362                         (void) printf("%-37s%s\n", fi->fi_uname, ro);
7363                         (void) printf("     %s\n", fi->fi_desc);
7364                 }
7365                 (void) printf("\n");
7366
7367                 (void) printf(gettext("The following legacy versions are also "
7368                     "supported:\n\n"));
7369                 (void) printf(gettext("VER  DESCRIPTION\n"));
7370                 (void) printf("---  -----------------------------------------"
7371                     "---------------\n");
7372                 (void) printf(gettext(" 1   Initial ZFS version\n"));
7373                 (void) printf(gettext(" 2   Ditto blocks "
7374                     "(replicated metadata)\n"));
7375                 (void) printf(gettext(" 3   Hot spares and double parity "
7376                     "RAID-Z\n"));
7377                 (void) printf(gettext(" 4   zpool history\n"));
7378                 (void) printf(gettext(" 5   Compression using the gzip "
7379                     "algorithm\n"));
7380                 (void) printf(gettext(" 6   bootfs pool property\n"));
7381                 (void) printf(gettext(" 7   Separate intent log devices\n"));
7382                 (void) printf(gettext(" 8   Delegated administration\n"));
7383                 (void) printf(gettext(" 9   refquota and refreservation "
7384                     "properties\n"));
7385                 (void) printf(gettext(" 10  Cache devices\n"));
7386                 (void) printf(gettext(" 11  Improved scrub performance\n"));
7387                 (void) printf(gettext(" 12  Snapshot properties\n"));
7388                 (void) printf(gettext(" 13  snapused property\n"));
7389                 (void) printf(gettext(" 14  passthrough-x aclinherit\n"));
7390                 (void) printf(gettext(" 15  user/group space accounting\n"));
7391                 (void) printf(gettext(" 16  stmf property support\n"));
7392                 (void) printf(gettext(" 17  Triple-parity RAID-Z\n"));
7393                 (void) printf(gettext(" 18  Snapshot user holds\n"));
7394                 (void) printf(gettext(" 19  Log device removal\n"));
7395                 (void) printf(gettext(" 20  Compression using zle "
7396                     "(zero-length encoding)\n"));
7397                 (void) printf(gettext(" 21  Deduplication\n"));
7398                 (void) printf(gettext(" 22  Received properties\n"));
7399                 (void) printf(gettext(" 23  Slim ZIL\n"));
7400                 (void) printf(gettext(" 24  System attributes\n"));
7401                 (void) printf(gettext(" 25  Improved scrub stats\n"));
7402                 (void) printf(gettext(" 26  Improved snapshot deletion "
7403                     "performance\n"));
7404                 (void) printf(gettext(" 27  Improved snapshot creation "
7405                     "performance\n"));
7406                 (void) printf(gettext(" 28  Multiple vdev replacements\n"));
7407                 (void) printf(gettext("\nFor more information on a particular "
7408                     "version, including supported releases,\n"));
7409                 (void) printf(gettext("see the ZFS Administration Guide.\n\n"));
7410         } else if (argc == 0 && upgradeall) {
7411                 cb.cb_first = B_TRUE;
7412                 ret = zpool_iter(g_zfs, upgrade_cb, &cb);
7413                 if (ret == 0 && cb.cb_first) {
7414                         if (cb.cb_version == SPA_VERSION) {
7415                                 (void) printf(gettext("All pools are already "
7416                                     "formatted using feature flags.\n\n"));
7417                                 (void) printf(gettext("Every feature flags "
7418                                     "pool already has all supported features "
7419                                     "enabled.\n"));
7420                         } else {
7421                                 (void) printf(gettext("All pools are already "
7422                                     "formatted with version %llu or higher.\n"),
7423                                     (u_longlong_t)cb.cb_version);
7424                         }
7425                 }
7426         } else if (argc == 0) {
7427                 cb.cb_first = B_TRUE;
7428                 ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
7429                 assert(ret == 0);
7430
7431                 if (cb.cb_first) {
7432                         (void) printf(gettext("All pools are formatted "
7433                             "using feature flags.\n\n"));
7434                 } else {
7435                         (void) printf(gettext("\nUse 'zpool upgrade -v' "
7436                             "for a list of available legacy versions.\n"));
7437                 }
7438
7439                 cb.cb_first = B_TRUE;
7440                 ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
7441                 assert(ret == 0);
7442
7443                 if (cb.cb_first) {
7444                         (void) printf(gettext("Every feature flags pool has "
7445                             "all supported features enabled.\n"));
7446                 } else {
7447                         (void) printf(gettext("\n"));
7448                 }
7449         } else {
7450                 ret = for_each_pool(argc, argv, B_FALSE, NULL,
7451                     upgrade_one, &cb);
7452         }
7453
7454         return (ret);
7455 }
7456
7457 typedef struct hist_cbdata {
7458         boolean_t first;
7459         boolean_t longfmt;
7460         boolean_t internal;
7461 } hist_cbdata_t;
7462
7463 /*
7464  * Print out the command history for a specific pool.
7465  */
7466 static int
7467 get_history_one(zpool_handle_t *zhp, void *data)
7468 {
7469         nvlist_t *nvhis;
7470         nvlist_t **records;
7471         uint_t numrecords;
7472         int ret, i;
7473         hist_cbdata_t *cb = (hist_cbdata_t *)data;
7474
7475         cb->first = B_FALSE;
7476
7477         (void) printf(gettext("History for '%s':\n"), zpool_get_name(zhp));
7478
7479         if ((ret = zpool_get_history(zhp, &nvhis)) != 0)
7480                 return (ret);
7481
7482         verify(nvlist_lookup_nvlist_array(nvhis, ZPOOL_HIST_RECORD,
7483             &records, &numrecords) == 0);
7484         for (i = 0; i < numrecords; i++) {
7485                 nvlist_t *rec = records[i];
7486                 char tbuf[30] = "";
7487
7488                 if (nvlist_exists(rec, ZPOOL_HIST_TIME)) {
7489                         time_t tsec;
7490                         struct tm t;
7491
7492                         tsec = fnvlist_lookup_uint64(records[i],
7493                             ZPOOL_HIST_TIME);
7494                         (void) localtime_r(&tsec, &t);
7495                         (void) strftime(tbuf, sizeof (tbuf), "%F.%T", &t);
7496                 }
7497
7498                 if (nvlist_exists(rec, ZPOOL_HIST_CMD)) {
7499                         (void) printf("%s %s", tbuf,
7500                             fnvlist_lookup_string(rec, ZPOOL_HIST_CMD));
7501                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_EVENT)) {
7502                         int ievent =
7503                             fnvlist_lookup_uint64(rec, ZPOOL_HIST_INT_EVENT);
7504                         if (!cb->internal)
7505                                 continue;
7506                         if (ievent >= ZFS_NUM_LEGACY_HISTORY_EVENTS) {
7507                                 (void) printf("%s unrecognized record:\n",
7508                                     tbuf);
7509                                 dump_nvlist(rec, 4);
7510                                 continue;
7511                         }
7512                         (void) printf("%s [internal %s txg:%lld] %s", tbuf,
7513                             zfs_history_event_names[ievent],
7514                             (longlong_t)fnvlist_lookup_uint64(
7515                             rec, ZPOOL_HIST_TXG),
7516                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_STR));
7517                 } else if (nvlist_exists(rec, ZPOOL_HIST_INT_NAME)) {
7518                         if (!cb->internal)
7519                                 continue;
7520                         (void) printf("%s [txg:%lld] %s", tbuf,
7521                             (longlong_t)fnvlist_lookup_uint64(
7522                             rec, ZPOOL_HIST_TXG),
7523                             fnvlist_lookup_string(rec, ZPOOL_HIST_INT_NAME));
7524                         if (nvlist_exists(rec, ZPOOL_HIST_DSNAME)) {
7525                                 (void) printf(" %s (%llu)",
7526                                     fnvlist_lookup_string(rec,
7527                                     ZPOOL_HIST_DSNAME),
7528                                     (u_longlong_t)fnvlist_lookup_uint64(rec,
7529                                     ZPOOL_HIST_DSID));
7530                         }
7531                         (void) printf(" %s", fnvlist_lookup_string(rec,
7532                             ZPOOL_HIST_INT_STR));
7533                 } else if (nvlist_exists(rec, ZPOOL_HIST_IOCTL)) {
7534                         if (!cb->internal)
7535                                 continue;
7536                         (void) printf("%s ioctl %s\n", tbuf,
7537                             fnvlist_lookup_string(rec, ZPOOL_HIST_IOCTL));
7538                         if (nvlist_exists(rec, ZPOOL_HIST_INPUT_NVL)) {
7539                                 (void) printf("    input:\n");
7540                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
7541                                     ZPOOL_HIST_INPUT_NVL), 8);
7542                         }
7543                         if (nvlist_exists(rec, ZPOOL_HIST_OUTPUT_NVL)) {
7544                                 (void) printf("    output:\n");
7545                                 dump_nvlist(fnvlist_lookup_nvlist(rec,
7546                                     ZPOOL_HIST_OUTPUT_NVL), 8);
7547                         }
7548                         if (nvlist_exists(rec, ZPOOL_HIST_ERRNO)) {
7549                                 (void) printf("    errno: %lld\n",
7550                                     (longlong_t)fnvlist_lookup_int64(rec,
7551                                     ZPOOL_HIST_ERRNO));
7552                         }
7553                 } else {
7554                         if (!cb->internal)
7555                                 continue;
7556                         (void) printf("%s unrecognized record:\n", tbuf);
7557                         dump_nvlist(rec, 4);
7558                 }
7559
7560                 if (!cb->longfmt) {
7561                         (void) printf("\n");
7562                         continue;
7563                 }
7564                 (void) printf(" [");
7565                 if (nvlist_exists(rec, ZPOOL_HIST_WHO)) {
7566                         uid_t who = fnvlist_lookup_uint64(rec, ZPOOL_HIST_WHO);
7567                         struct passwd *pwd = getpwuid(who);
7568                         (void) printf("user %d ", (int)who);
7569                         if (pwd != NULL)
7570                                 (void) printf("(%s) ", pwd->pw_name);
7571                 }
7572                 if (nvlist_exists(rec, ZPOOL_HIST_HOST)) {
7573                         (void) printf("on %s",
7574                             fnvlist_lookup_string(rec, ZPOOL_HIST_HOST));
7575                 }
7576                 if (nvlist_exists(rec, ZPOOL_HIST_ZONE)) {
7577                         (void) printf(":%s",
7578                             fnvlist_lookup_string(rec, ZPOOL_HIST_ZONE));
7579                 }
7580
7581                 (void) printf("]");
7582                 (void) printf("\n");
7583         }
7584         (void) printf("\n");
7585         nvlist_free(nvhis);
7586
7587         return (ret);
7588 }
7589
7590 /*
7591  * zpool history <pool>
7592  *
7593  * Displays the history of commands that modified pools.
7594  */
7595 int
7596 zpool_do_history(int argc, char **argv)
7597 {
7598         hist_cbdata_t cbdata = { 0 };
7599         int ret;
7600         int c;
7601
7602         cbdata.first = B_TRUE;
7603         /* check options */
7604         while ((c = getopt(argc, argv, "li")) != -1) {
7605                 switch (c) {
7606                 case 'l':
7607                         cbdata.longfmt = B_TRUE;
7608                         break;
7609                 case 'i':
7610                         cbdata.internal = B_TRUE;
7611                         break;
7612                 case '?':
7613                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7614                             optopt);
7615                         usage(B_FALSE);
7616                 }
7617         }
7618         argc -= optind;
7619         argv += optind;
7620
7621         ret = for_each_pool(argc, argv, B_FALSE,  NULL, get_history_one,
7622             &cbdata);
7623
7624         if (argc == 0 && cbdata.first == B_TRUE) {
7625                 (void) fprintf(stderr, gettext("no pools available\n"));
7626                 return (0);
7627         }
7628
7629         return (ret);
7630 }
7631
7632 typedef struct ev_opts {
7633         int verbose;
7634         int scripted;
7635         int follow;
7636         int clear;
7637         char poolname[ZFS_MAX_DATASET_NAME_LEN];
7638 } ev_opts_t;
7639
7640 static void
7641 zpool_do_events_short(nvlist_t *nvl, ev_opts_t *opts)
7642 {
7643         char ctime_str[26], str[32], *ptr;
7644         int64_t *tv;
7645         uint_t n;
7646
7647         verify(nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0);
7648         memset(str, ' ', 32);
7649         (void) ctime_r((const time_t *)&tv[0], ctime_str);
7650         (void) strncpy(str, ctime_str+4,  6);           /* 'Jun 30' */
7651         (void) strncpy(str+7, ctime_str+20, 4);         /* '1993' */
7652         (void) strncpy(str+12, ctime_str+11, 8);        /* '21:49:08' */
7653         (void) sprintf(str+20, ".%09lld", (longlong_t)tv[1]); /* '.123456789' */
7654         if (opts->scripted)
7655                 (void) printf(gettext("%s\t"), str);
7656         else
7657                 (void) printf(gettext("%s "), str);
7658
7659         verify(nvlist_lookup_string(nvl, FM_CLASS, &ptr) == 0);
7660         (void) printf(gettext("%s\n"), ptr);
7661 }
7662
7663 static void
7664 zpool_do_events_nvprint(nvlist_t *nvl, int depth)
7665 {
7666         nvpair_t *nvp;
7667
7668         for (nvp = nvlist_next_nvpair(nvl, NULL);
7669             nvp != NULL; nvp = nvlist_next_nvpair(nvl, nvp)) {
7670
7671                 data_type_t type = nvpair_type(nvp);
7672                 const char *name = nvpair_name(nvp);
7673
7674                 boolean_t b;
7675                 uint8_t i8;
7676                 uint16_t i16;
7677                 uint32_t i32;
7678                 uint64_t i64;
7679                 char *str;
7680                 nvlist_t *cnv;
7681
7682                 printf(gettext("%*s%s = "), depth, "", name);
7683
7684                 switch (type) {
7685                 case DATA_TYPE_BOOLEAN:
7686                         printf(gettext("%s"), "1");
7687                         break;
7688
7689                 case DATA_TYPE_BOOLEAN_VALUE:
7690                         (void) nvpair_value_boolean_value(nvp, &b);
7691                         printf(gettext("%s"), b ? "1" : "0");
7692                         break;
7693
7694                 case DATA_TYPE_BYTE:
7695                         (void) nvpair_value_byte(nvp, &i8);
7696                         printf(gettext("0x%x"), i8);
7697                         break;
7698
7699                 case DATA_TYPE_INT8:
7700                         (void) nvpair_value_int8(nvp, (void *)&i8);
7701                         printf(gettext("0x%x"), i8);
7702                         break;
7703
7704                 case DATA_TYPE_UINT8:
7705                         (void) nvpair_value_uint8(nvp, &i8);
7706                         printf(gettext("0x%x"), i8);
7707                         break;
7708
7709                 case DATA_TYPE_INT16:
7710                         (void) nvpair_value_int16(nvp, (void *)&i16);
7711                         printf(gettext("0x%x"), i16);
7712                         break;
7713
7714                 case DATA_TYPE_UINT16:
7715                         (void) nvpair_value_uint16(nvp, &i16);
7716                         printf(gettext("0x%x"), i16);
7717                         break;
7718
7719                 case DATA_TYPE_INT32:
7720                         (void) nvpair_value_int32(nvp, (void *)&i32);
7721                         printf(gettext("0x%x"), i32);
7722                         break;
7723
7724                 case DATA_TYPE_UINT32:
7725                         (void) nvpair_value_uint32(nvp, &i32);
7726                         printf(gettext("0x%x"), i32);
7727                         break;
7728
7729                 case DATA_TYPE_INT64:
7730                         (void) nvpair_value_int64(nvp, (void *)&i64);
7731                         printf(gettext("0x%llx"), (u_longlong_t)i64);
7732                         break;
7733
7734                 case DATA_TYPE_UINT64:
7735                         (void) nvpair_value_uint64(nvp, &i64);
7736                         /*
7737                          * translate vdev state values to readable
7738                          * strings to aide zpool events consumers
7739                          */
7740                         if (strcmp(name,
7741                             FM_EREPORT_PAYLOAD_ZFS_VDEV_STATE) == 0 ||
7742                             strcmp(name,
7743                             FM_EREPORT_PAYLOAD_ZFS_VDEV_LASTSTATE) == 0) {
7744                                 printf(gettext("\"%s\" (0x%llx)"),
7745                                     zpool_state_to_name(i64, VDEV_AUX_NONE),
7746                                     (u_longlong_t)i64);
7747                         } else {
7748                                 printf(gettext("0x%llx"), (u_longlong_t)i64);
7749                         }
7750                         break;
7751
7752                 case DATA_TYPE_HRTIME:
7753                         (void) nvpair_value_hrtime(nvp, (void *)&i64);
7754                         printf(gettext("0x%llx"), (u_longlong_t)i64);
7755                         break;
7756
7757                 case DATA_TYPE_STRING:
7758                         (void) nvpair_value_string(nvp, &str);
7759                         printf(gettext("\"%s\""), str ? str : "<NULL>");
7760                         break;
7761
7762                 case DATA_TYPE_NVLIST:
7763                         printf(gettext("(embedded nvlist)\n"));
7764                         (void) nvpair_value_nvlist(nvp, &cnv);
7765                         zpool_do_events_nvprint(cnv, depth + 8);
7766                         printf(gettext("%*s(end %s)"), depth, "", name);
7767                         break;
7768
7769                 case DATA_TYPE_NVLIST_ARRAY: {
7770                         nvlist_t **val;
7771                         uint_t i, nelem;
7772
7773                         (void) nvpair_value_nvlist_array(nvp, &val, &nelem);
7774                         printf(gettext("(%d embedded nvlists)\n"), nelem);
7775                         for (i = 0; i < nelem; i++) {
7776                                 printf(gettext("%*s%s[%d] = %s\n"),
7777                                     depth, "", name, i, "(embedded nvlist)");
7778                                 zpool_do_events_nvprint(val[i], depth + 8);
7779                                 printf(gettext("%*s(end %s[%i])\n"),
7780                                     depth, "", name, i);
7781                         }
7782                         printf(gettext("%*s(end %s)\n"), depth, "", name);
7783                         }
7784                         break;
7785
7786                 case DATA_TYPE_INT8_ARRAY: {
7787                         int8_t *val;
7788                         uint_t i, nelem;
7789
7790                         (void) nvpair_value_int8_array(nvp, &val, &nelem);
7791                         for (i = 0; i < nelem; i++)
7792                                 printf(gettext("0x%x "), val[i]);
7793
7794                         break;
7795                         }
7796
7797                 case DATA_TYPE_UINT8_ARRAY: {
7798                         uint8_t *val;
7799                         uint_t i, nelem;
7800
7801                         (void) nvpair_value_uint8_array(nvp, &val, &nelem);
7802                         for (i = 0; i < nelem; i++)
7803                                 printf(gettext("0x%x "), val[i]);
7804
7805                         break;
7806                         }
7807
7808                 case DATA_TYPE_INT16_ARRAY: {
7809                         int16_t *val;
7810                         uint_t i, nelem;
7811
7812                         (void) nvpair_value_int16_array(nvp, &val, &nelem);
7813                         for (i = 0; i < nelem; i++)
7814                                 printf(gettext("0x%x "), val[i]);
7815
7816                         break;
7817                         }
7818
7819                 case DATA_TYPE_UINT16_ARRAY: {
7820                         uint16_t *val;
7821                         uint_t i, nelem;
7822
7823                         (void) nvpair_value_uint16_array(nvp, &val, &nelem);
7824                         for (i = 0; i < nelem; i++)
7825                                 printf(gettext("0x%x "), val[i]);
7826
7827                         break;
7828                         }
7829
7830                 case DATA_TYPE_INT32_ARRAY: {
7831                         int32_t *val;
7832                         uint_t i, nelem;
7833
7834                         (void) nvpair_value_int32_array(nvp, &val, &nelem);
7835                         for (i = 0; i < nelem; i++)
7836                                 printf(gettext("0x%x "), val[i]);
7837
7838                         break;
7839                         }
7840
7841                 case DATA_TYPE_UINT32_ARRAY: {
7842                         uint32_t *val;
7843                         uint_t i, nelem;
7844
7845                         (void) nvpair_value_uint32_array(nvp, &val, &nelem);
7846                         for (i = 0; i < nelem; i++)
7847                                 printf(gettext("0x%x "), val[i]);
7848
7849                         break;
7850                         }
7851
7852                 case DATA_TYPE_INT64_ARRAY: {
7853                         int64_t *val;
7854                         uint_t i, nelem;
7855
7856                         (void) nvpair_value_int64_array(nvp, &val, &nelem);
7857                         for (i = 0; i < nelem; i++)
7858                                 printf(gettext("0x%llx "),
7859                                     (u_longlong_t)val[i]);
7860
7861                         break;
7862                         }
7863
7864                 case DATA_TYPE_UINT64_ARRAY: {
7865                         uint64_t *val;
7866                         uint_t i, nelem;
7867
7868                         (void) nvpair_value_uint64_array(nvp, &val, &nelem);
7869                         for (i = 0; i < nelem; i++)
7870                                 printf(gettext("0x%llx "),
7871                                     (u_longlong_t)val[i]);
7872
7873                         break;
7874                         }
7875
7876                 case DATA_TYPE_STRING_ARRAY: {
7877                         char **str;
7878                         uint_t i, nelem;
7879
7880                         (void) nvpair_value_string_array(nvp, &str, &nelem);
7881                         for (i = 0; i < nelem; i++)
7882                                 printf(gettext("\"%s\" "),
7883                                     str[i] ? str[i] : "<NULL>");
7884
7885                         break;
7886                         }
7887
7888                 case DATA_TYPE_BOOLEAN_ARRAY:
7889                 case DATA_TYPE_BYTE_ARRAY:
7890                 case DATA_TYPE_DOUBLE:
7891                 case DATA_TYPE_UNKNOWN:
7892                         printf(gettext("<unknown>"));
7893                         break;
7894                 }
7895
7896                 printf(gettext("\n"));
7897         }
7898 }
7899
7900 static int
7901 zpool_do_events_next(ev_opts_t *opts)
7902 {
7903         nvlist_t *nvl;
7904         int zevent_fd, ret, dropped;
7905         char *pool;
7906
7907         zevent_fd = open(ZFS_DEV, O_RDWR);
7908         VERIFY(zevent_fd >= 0);
7909
7910         if (!opts->scripted)
7911                 (void) printf(gettext("%-30s %s\n"), "TIME", "CLASS");
7912
7913         while (1) {
7914                 ret = zpool_events_next(g_zfs, &nvl, &dropped,
7915                     (opts->follow ? ZEVENT_NONE : ZEVENT_NONBLOCK), zevent_fd);
7916                 if (ret || nvl == NULL)
7917                         break;
7918
7919                 if (dropped > 0)
7920                         (void) printf(gettext("dropped %d events\n"), dropped);
7921
7922                 if (strlen(opts->poolname) > 0 &&
7923                     nvlist_lookup_string(nvl, FM_FMRI_ZFS_POOL, &pool) == 0 &&
7924                     strcmp(opts->poolname, pool) != 0)
7925                         continue;
7926
7927                 zpool_do_events_short(nvl, opts);
7928
7929                 if (opts->verbose) {
7930                         zpool_do_events_nvprint(nvl, 8);
7931                         printf(gettext("\n"));
7932                 }
7933                 (void) fflush(stdout);
7934
7935                 nvlist_free(nvl);
7936         }
7937
7938         VERIFY(0 == close(zevent_fd));
7939
7940         return (ret);
7941 }
7942
7943 static int
7944 zpool_do_events_clear(ev_opts_t *opts)
7945 {
7946         int count, ret;
7947
7948         ret = zpool_events_clear(g_zfs, &count);
7949         if (!ret)
7950                 (void) printf(gettext("cleared %d events\n"), count);
7951
7952         return (ret);
7953 }
7954
7955 /*
7956  * zpool events [-vHf [pool] | -c]
7957  *
7958  * Displays events logs by ZFS.
7959  */
7960 int
7961 zpool_do_events(int argc, char **argv)
7962 {
7963         ev_opts_t opts = { 0 };
7964         int ret;
7965         int c;
7966
7967         /* check options */
7968         while ((c = getopt(argc, argv, "vHfc")) != -1) {
7969                 switch (c) {
7970                 case 'v':
7971                         opts.verbose = 1;
7972                         break;
7973                 case 'H':
7974                         opts.scripted = 1;
7975                         break;
7976                 case 'f':
7977                         opts.follow = 1;
7978                         break;
7979                 case 'c':
7980                         opts.clear = 1;
7981                         break;
7982                 case '?':
7983                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
7984                             optopt);
7985                         usage(B_FALSE);
7986                 }
7987         }
7988         argc -= optind;
7989         argv += optind;
7990
7991         if (argc > 1) {
7992                 (void) fprintf(stderr, gettext("too many arguments\n"));
7993                 usage(B_FALSE);
7994         } else if (argc == 1) {
7995                 (void) strlcpy(opts.poolname, argv[0], sizeof (opts.poolname));
7996                 if (!zfs_name_valid(opts.poolname, ZFS_TYPE_POOL)) {
7997                         (void) fprintf(stderr,
7998                             gettext("invalid pool name '%s'\n"), opts.poolname);
7999                         usage(B_FALSE);
8000                 }
8001         }
8002
8003         if ((argc == 1 || opts.verbose || opts.scripted || opts.follow) &&
8004             opts.clear) {
8005                 (void) fprintf(stderr,
8006                     gettext("invalid options combined with -c\n"));
8007                 usage(B_FALSE);
8008         }
8009
8010         if (opts.clear)
8011                 ret = zpool_do_events_clear(&opts);
8012         else
8013                 ret = zpool_do_events_next(&opts);
8014
8015         return (ret);
8016 }
8017
8018 static int
8019 get_callback(zpool_handle_t *zhp, void *data)
8020 {
8021         zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data;
8022         char value[MAXNAMELEN];
8023         zprop_source_t srctype;
8024         zprop_list_t *pl;
8025
8026         for (pl = cbp->cb_proplist; pl != NULL; pl = pl->pl_next) {
8027
8028                 /*
8029                  * Skip the special fake placeholder. This will also skip
8030                  * over the name property when 'all' is specified.
8031                  */
8032                 if (pl->pl_prop == ZPOOL_PROP_NAME &&
8033                     pl == cbp->cb_proplist)
8034                         continue;
8035
8036                 if (pl->pl_prop == ZPROP_INVAL &&
8037                     (zpool_prop_feature(pl->pl_user_prop) ||
8038                     zpool_prop_unsupported(pl->pl_user_prop))) {
8039                         srctype = ZPROP_SRC_LOCAL;
8040
8041                         if (zpool_prop_get_feature(zhp, pl->pl_user_prop,
8042                             value, sizeof (value)) == 0) {
8043                                 zprop_print_one_property(zpool_get_name(zhp),
8044                                     cbp, pl->pl_user_prop, value, srctype,
8045                                     NULL, NULL);
8046                         }
8047                 } else {
8048                         if (zpool_get_prop(zhp, pl->pl_prop, value,
8049                             sizeof (value), &srctype, cbp->cb_literal) != 0)
8050                                 continue;
8051
8052                         zprop_print_one_property(zpool_get_name(zhp), cbp,
8053                             zpool_prop_to_name(pl->pl_prop), value, srctype,
8054                             NULL, NULL);
8055                 }
8056         }
8057         return (0);
8058 }
8059
8060 /*
8061  * zpool get [-Hp] [-o "all" | field[,...]] <"all" | property[,...]> <pool> ...
8062  *
8063  *      -H      Scripted mode.  Don't display headers, and separate properties
8064  *              by a single tab.
8065  *      -o      List of columns to display.  Defaults to
8066  *              "name,property,value,source".
8067  *      -p      Display values in parsable (exact) format.
8068  *
8069  * Get properties of pools in the system. Output space statistics
8070  * for each one as well as other attributes.
8071  */
8072 int
8073 zpool_do_get(int argc, char **argv)
8074 {
8075         zprop_get_cbdata_t cb = { 0 };
8076         zprop_list_t fake_name = { 0 };
8077         int ret;
8078         int c, i;
8079         char *value;
8080
8081         cb.cb_first = B_TRUE;
8082
8083         /*
8084          * Set up default columns and sources.
8085          */
8086         cb.cb_sources = ZPROP_SRC_ALL;
8087         cb.cb_columns[0] = GET_COL_NAME;
8088         cb.cb_columns[1] = GET_COL_PROPERTY;
8089         cb.cb_columns[2] = GET_COL_VALUE;
8090         cb.cb_columns[3] = GET_COL_SOURCE;
8091         cb.cb_type = ZFS_TYPE_POOL;
8092
8093         /* check options */
8094         while ((c = getopt(argc, argv, ":Hpo:")) != -1) {
8095                 switch (c) {
8096                 case 'p':
8097                         cb.cb_literal = B_TRUE;
8098                         break;
8099                 case 'H':
8100                         cb.cb_scripted = B_TRUE;
8101                         break;
8102                 case 'o':
8103                         bzero(&cb.cb_columns, sizeof (cb.cb_columns));
8104                         i = 0;
8105                         while (*optarg != '\0') {
8106                                 static char *col_subopts[] =
8107                                 { "name", "property", "value", "source",
8108                                 "all", NULL };
8109
8110                                 if (i == ZFS_GET_NCOLS) {
8111                                         (void) fprintf(stderr, gettext("too "
8112                                         "many fields given to -o "
8113                                         "option\n"));
8114                                         usage(B_FALSE);
8115                                 }
8116
8117                                 switch (getsubopt(&optarg, col_subopts,
8118                                     &value)) {
8119                                 case 0:
8120                                         cb.cb_columns[i++] = GET_COL_NAME;
8121                                         break;
8122                                 case 1:
8123                                         cb.cb_columns[i++] = GET_COL_PROPERTY;
8124                                         break;
8125                                 case 2:
8126                                         cb.cb_columns[i++] = GET_COL_VALUE;
8127                                         break;
8128                                 case 3:
8129                                         cb.cb_columns[i++] = GET_COL_SOURCE;
8130                                         break;
8131                                 case 4:
8132                                         if (i > 0) {
8133                                                 (void) fprintf(stderr,
8134                                                     gettext("\"all\" conflicts "
8135                                                     "with specific fields "
8136                                                     "given to -o option\n"));
8137                                                 usage(B_FALSE);
8138                                         }
8139                                         cb.cb_columns[0] = GET_COL_NAME;
8140                                         cb.cb_columns[1] = GET_COL_PROPERTY;
8141                                         cb.cb_columns[2] = GET_COL_VALUE;
8142                                         cb.cb_columns[3] = GET_COL_SOURCE;
8143                                         i = ZFS_GET_NCOLS;
8144                                         break;
8145                                 default:
8146                                         (void) fprintf(stderr,
8147                                             gettext("invalid column name "
8148                                             "'%s'\n"), value);
8149                                         usage(B_FALSE);
8150                                 }
8151                         }
8152                         break;
8153                 case '?':
8154                         (void) fprintf(stderr, gettext("invalid option '%c'\n"),
8155                             optopt);
8156                         usage(B_FALSE);
8157                 }
8158         }
8159
8160         argc -= optind;
8161         argv += optind;
8162
8163         if (argc < 1) {
8164                 (void) fprintf(stderr, gettext("missing property "
8165                     "argument\n"));
8166                 usage(B_FALSE);
8167         }
8168
8169         if (zprop_get_list(g_zfs, argv[0], &cb.cb_proplist,
8170             ZFS_TYPE_POOL) != 0)
8171                 usage(B_FALSE);
8172
8173         argc--;
8174         argv++;
8175
8176         if (cb.cb_proplist != NULL) {
8177                 fake_name.pl_prop = ZPOOL_PROP_NAME;
8178                 fake_name.pl_width = strlen(gettext("NAME"));
8179                 fake_name.pl_next = cb.cb_proplist;
8180                 cb.cb_proplist = &fake_name;
8181         }
8182
8183         ret = for_each_pool(argc, argv, B_TRUE, &cb.cb_proplist,
8184             get_callback, &cb);
8185
8186         if (cb.cb_proplist == &fake_name)
8187                 zprop_free_list(fake_name.pl_next);
8188         else
8189                 zprop_free_list(cb.cb_proplist);
8190
8191         return (ret);
8192 }
8193
8194 typedef struct set_cbdata {
8195         char *cb_propname;
8196         char *cb_value;
8197         boolean_t cb_any_successful;
8198 } set_cbdata_t;
8199
8200 int
8201 set_callback(zpool_handle_t *zhp, void *data)
8202 {
8203         int error;
8204         set_cbdata_t *cb = (set_cbdata_t *)data;
8205
8206         error = zpool_set_prop(zhp, cb->cb_propname, cb->cb_value);
8207
8208         if (!error)
8209                 cb->cb_any_successful = B_TRUE;
8210
8211         return (error);
8212 }
8213
8214 int
8215 zpool_do_set(int argc, char **argv)
8216 {
8217         set_cbdata_t cb = { 0 };
8218         int error;
8219
8220         if (argc > 1 && argv[1][0] == '-') {
8221                 (void) fprintf(stderr, gettext("invalid option '%c'\n"),
8222                     argv[1][1]);
8223                 usage(B_FALSE);
8224         }
8225
8226         if (argc < 2) {
8227                 (void) fprintf(stderr, gettext("missing property=value "
8228                     "argument\n"));
8229                 usage(B_FALSE);
8230         }
8231
8232         if (argc < 3) {
8233                 (void) fprintf(stderr, gettext("missing pool name\n"));
8234                 usage(B_FALSE);
8235         }
8236
8237         if (argc > 3) {
8238                 (void) fprintf(stderr, gettext("too many pool names\n"));
8239                 usage(B_FALSE);
8240         }
8241
8242         cb.cb_propname = argv[1];
8243         cb.cb_value = strchr(cb.cb_propname, '=');
8244         if (cb.cb_value == NULL) {
8245                 (void) fprintf(stderr, gettext("missing value in "
8246                     "property=value argument\n"));
8247                 usage(B_FALSE);
8248         }
8249
8250         *(cb.cb_value) = '\0';
8251         cb.cb_value++;
8252
8253         error = for_each_pool(argc - 2, argv + 2, B_TRUE, NULL,
8254             set_callback, &cb);
8255
8256         return (error);
8257 }
8258
8259 static int
8260 find_command_idx(char *command, int *idx)
8261 {
8262         int i;
8263
8264         for (i = 0; i < NCOMMAND; i++) {
8265                 if (command_table[i].name == NULL)
8266                         continue;
8267
8268                 if (strcmp(command, command_table[i].name) == 0) {
8269                         *idx = i;
8270                         return (0);
8271                 }
8272         }
8273         return (1);
8274 }
8275
8276 int
8277 main(int argc, char **argv)
8278 {
8279         int ret = 0;
8280         int i = 0;
8281         char *cmdname;
8282
8283         (void) setlocale(LC_ALL, "");
8284         (void) textdomain(TEXT_DOMAIN);
8285         srand(time(NULL));
8286
8287         opterr = 0;
8288
8289         /*
8290          * Make sure the user has specified some command.
8291          */
8292         if (argc < 2) {
8293                 (void) fprintf(stderr, gettext("missing command\n"));
8294                 usage(B_FALSE);
8295         }
8296
8297         cmdname = argv[1];
8298
8299         /*
8300          * Special case '-?'
8301          */
8302         if ((strcmp(cmdname, "-?") == 0) || strcmp(cmdname, "--help") == 0)
8303                 usage(B_TRUE);
8304
8305         if ((g_zfs = libzfs_init()) == NULL) {
8306                 (void) fprintf(stderr, "%s", libzfs_error_init(errno));
8307                 return (1);
8308         }
8309
8310         libzfs_print_on_error(g_zfs, B_TRUE);
8311
8312         zfs_save_arguments(argc, argv, history_str, sizeof (history_str));
8313
8314         /*
8315          * Run the appropriate command.
8316          */
8317         if (find_command_idx(cmdname, &i) == 0) {
8318                 current_command = &command_table[i];
8319                 ret = command_table[i].func(argc - 1, argv + 1);
8320         } else if (strchr(cmdname, '=')) {
8321                 verify(find_command_idx("set", &i) == 0);
8322                 current_command = &command_table[i];
8323                 ret = command_table[i].func(argc, argv);
8324         } else if (strcmp(cmdname, "freeze") == 0 && argc == 3) {
8325                 /*
8326                  * 'freeze' is a vile debugging abomination, so we treat
8327                  * it as such.
8328                  */
8329                 zfs_cmd_t zc = {"\0"};
8330
8331                 (void) strlcpy(zc.zc_name, argv[2], sizeof (zc.zc_name));
8332                 ret = zfs_ioctl(g_zfs, ZFS_IOC_POOL_FREEZE, &zc);
8333                 if (ret != 0) {
8334                         (void) fprintf(stderr,
8335                         gettext("failed to freeze pool: %d\n"), errno);
8336                         ret = 1;
8337                 }
8338
8339                 log_history = 0;
8340         } else {
8341                 (void) fprintf(stderr, gettext("unrecognized "
8342                     "command '%s'\n"), cmdname);
8343                 usage(B_FALSE);
8344                 ret = 1;
8345         }
8346
8347         if (ret == 0 && log_history)
8348                 (void) zpool_log_history(g_zfs, history_str);
8349
8350         libzfs_fini(g_zfs);
8351
8352         /*
8353          * The 'ZFS_ABORT' environment variable causes us to dump core on exit
8354          * for the purposes of running ::findleaks.
8355          */
8356         if (getenv("ZFS_ABORT") != NULL) {
8357                 (void) printf("dumping core by request\n");
8358                 abort();
8359         }
8360
8361         return (ret);
8362 }