Fix taskq creation failure in vdev_open_children()

When creating and destroying pools in tight loop it's possible to
exhaust the number of allowed threads on a system.  This results
in taskq_create() failling and a NULL dereference.

Resolve the issue by falling back to opening the vdevs all
synchronously.

Reviewed-by: Denys Rtveliashvili <denys@rtveliashvili.name>
Reviewed-by: Håkan Johansson <f96hajo@chalmers.se>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes zfsonlinux/spl#521
Closes #4637

diff --git a/config/user-commands.m4 b/config/user-commands.m4
index 6e9c3d10383ab4c118b680454031cb4f17628010..d53bec4ff9326e31b76b49d157ef90df24398d03 100644 (file)
--- a/config/user-commands.m4
+++ b/config/user-commands.m4
@@ -89,6 +89,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_COMMON], [
        AC_PATH_TOOL(USERADD, useradd, "/usr/sbin/useradd")
        AC_PATH_TOOL(USERDEL, userdel, "/usr/sbin/userdel")
        AC_PATH_TOOL(USERMOD, usermod, "/usr/sbin/usermod")
+       AC_PATH_TOOL(UUIDGEN, uuidgen, "")
        AC_PATH_TOOL(WAIT, wait, "wait") dnl # Builtin in bash
        AC_PATH_TOOL(WC, wc, "")
 ])

diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c
index 8a4d48a1dc42d11df91c249b4b509b9b1cdeea7a..e3a9da2d5d7b8a585dcd9f5babc3cd801dea6dbc 100644 (file)
--- a/module/zfs/vdev.c
+++ b/module/zfs/vdev.c
@@ -1179,12 +1179,15 @@ vdev_open_children(vdev_t *vd)
         * spa_namespace_lock
         */
        if (vdev_uses_zvols(vd)) {
+retry_sync:
                for (c = 0; c < children; c++)
                        vd->vdev_child[c]->vdev_open_error =
                            vdev_open(vd->vdev_child[c]);
        } else {
                tq = taskq_create("vdev_open", children, minclsyspri,
                    children, children, TASKQ_PREPOPULATE);
+               if (tq == NULL)
+                       goto retry_sync;
 
                for (c = 0; c < children; c++)
                        VERIFY(taskq_dispatch(tq, vdev_open_child,

diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run
index 086ea2b273eaf45259b0096227a37dcb394a2b6d..f7ca9d5a7c8db06778121de8ef7fa0ba423668fe 100644 (file)
--- a/tests/runfiles/linux.run
+++ b/tests/runfiles/linux.run
@@ -248,6 +248,7 @@ tests = [
     'zpool_create_009_neg', 'zpool_create_010_neg', 'zpool_create_017_neg',
     'zpool_create_018_pos', 'zpool_create_019_pos',
     'zpool_create_021_pos', 'zpool_create_022_pos', 'zpool_create_023_neg',
+    'zpool_create_024_pos',
     'zpool_create_features_001_pos', 'zpool_create_features_002_pos',
     'zpool_create_features_003_pos', 'zpool_create_features_004_neg']
 

diff --git a/tests/zfs-tests/include/commands.cfg.in b/tests/zfs-tests/include/commands.cfg.in
index cf201c7e4341c6f184dd702ea9e2688a4a98a8d4..e7fb5ff135ae09deb6ca8fc98d2611aa501c7a04 100644 (file)
--- a/tests/zfs-tests/include/commands.cfg.in
+++ b/tests/zfs-tests/include/commands.cfg.in
@@ -121,6 +121,7 @@ export UNSHARE="@UNSHARE@"
 export USERADD="@USERADD@"
 export USERDEL="@USERDEL@"
 export USERMOD="@USERMOD@"
+export UUIDGEN="@UUIDGEN@"
 export VMSTAT="@VMSTAT@"
 export WAIT="@WAIT@"
 export WC="@WC@"

diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_create/Makefile.am b/tests/zfs-tests/tests/functional/cli_root/zpool_create/Makefile.am
index 0530dec1a8be66043f9d62fb73c8aefa371e4d46..a9ffbee03abad80c6c81b738ceda22b5eaa8d75e 100644 (file)
--- a/tests/zfs-tests/tests/functional/cli_root/zpool_create/Makefile.am
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_create/Makefile.am
@@ -26,6 +26,7 @@ dist_pkgdata_SCRIPTS = \
        zpool_create_021_pos.ksh \
        zpool_create_022_pos.ksh \
        zpool_create_023_neg.ksh \
+       zpool_create_024_pos.ksh \
        zpool_create_features_001_pos.ksh \
        zpool_create_features_002_pos.ksh \
        zpool_create_features_003_pos.ksh \

diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_024_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_024_pos.ksh
new file mode 100755 (executable)
index 0000000..1fa22d6
--- /dev/null
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_create/zpool_create_024_pos.ksh
@@ -0,0 +1,148 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2016, Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/cli_root/zpool_create/zpool_create.cfg
+
+#
+# DESCRIPTION:
+# Many 'zpool create' and 'zpool destroy' must succeed concurrently.
+#
+# STRATEGY:
+# 1. Create N process each of which create/destroy a pool M times.
+# 2. Allow all process to run to completion.
+# 3. Verify all pools and their vdevs were destroyed.
+#
+
+verify_runnable "global"
+
+function cleanup
+{
+       if [[ -n "$child_pids" ]]; then
+               for wait_pid in $child_pids; do
+                       $KILL $wait_pid 2>/dev/null
+               done
+       fi
+
+       if [[ -n "$child_pools" ]]; then
+               for pool in $child_pools; do
+                       typeset vdev0="$TEST_BASE_DIR/$pool-vdev0.img"
+                       typeset vdev1="$TEST_BASE_DIR/$pool-vdev1.img"
+
+                       if poolexists $pool; then
+                               destroy_pool $pool
+                       fi
+
+                       $RM -f $vdev0 $vdev1
+               done
+       fi
+}
+
+log_onexit cleanup
+
+log_assert "Many 'zpool create' and 'zpool destroy' must succeed concurrently."
+
+child_pids=""
+child_pools=""
+
+function zpool_stress
+{
+       typeset pool=$1
+       typeset vdev0="$TEST_BASE_DIR/$pool-vdev0.img"
+       typeset vdev1="$TEST_BASE_DIR/$pool-vdev1.img"
+       typeset -i iters=$2
+       typeset retry=10
+       typeset j=0
+
+       $TRUNCATE -s $FILESIZE $vdev0
+       $TRUNCATE -s $FILESIZE $vdev1
+
+       while [[ $j -lt $iters ]]; do
+               ((j = j + 1))
+               $SLEEP 1
+
+               $ZPOOL create $pool $vdev0 $vdev1
+               if [ $? -ne 0 ]; then
+                       return 1;
+               fi
+
+               # The 'zfs destroy' command is retried because it can
+               # transiently return EBUSY when blkid is concurrently
+               # probing new volumes and therefore has them open.
+               typeset k=0;
+               while [[ $k -lt $retry ]]; do
+                       ((k = k + 1))
+
+                       $ZPOOL destroy $pool
+                       if [ $? -eq 0 ]; then
+                               break;
+                       elif [ $k -eq $retry ]; then
+                               return 1;
+                       fi
+
+                       $SLEEP 3
+               done
+       done
+
+       $RM -f $vdev0 $vdev1
+       return 0
+}
+
+# 1. Create 128 process each of which create/destroy a pool 5 times.
+typeset i=0
+while [[ $i -lt 128 ]]; do
+       typeset uuid=$($UUIDGEN | $CUT -c1-13)
+
+       zpool_stress $TESTPOOL-$uuid 5 &
+       typeset pid=$!
+
+       child_pids="$child_pids $pid"
+       child_pools="$child_pools $TESTPOOL-$uuid"
+       ((i = i + 1))
+done
+
+# 2. Allow all process to run to completion.
+wait
+
+# 3. Verify all pools and their vdevs were destroyed.
+for pool in $child_pools; do
+       typeset vdev0="$TEST_BASE_DIR/$pool-vdev0.img"
+       typeset vdev1="$TEST_BASE_DIR/$pool-vdev1.img"
+
+       if poolexists $pool; then
+               log_fail "pool $pool exists"
+       fi
+
+       if [ -e $vdev0 ]; then
+               log_fail "pool vdev $vdev0 exists"
+       fi
+
+       if [ -e $vdev1 ]; then
+               log_fail "pool vdev $vdev1 exists"
+       fi
+done
+
+log_pass "Many 'zpool create' and 'zpool destroy' must succeed concurrently."