Retry zfs destroy when busy in rsend tests

rsend tests in the test suite frequently create and
destroy datasets. It is possible for zfs destroy to
return an error code indicating the dataset is busy.
Simply use a log_must_busy in these cases to retry
destroying those datasets. Other fixes to rsend test
cases to avoid unmounting and remounting filesystems
and some cleanup.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Giuseppe Di Natale <dinatale2@llnl.gov>
Closes #6418

diff --git a/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh b/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh
index 2d0054b2c18303553a5cb0384d7d329912106d3a..a396aae4b07d292e134e45e40e84cfb81971cc7f 100755 (executable)
--- a/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh
+++ b/tests/zfs-tests/tests/functional/reservation/reservation_004_pos.sh
@@ -56,7 +56,7 @@ verify_runnable "both"
 function cleanup {
 
        for obj in $OBJ_LIST; do
-               datasetexists $obj && log_must zfs destroy -f $obj
+               datasetexists $obj && log_must_busy zfs destroy -f $obj
        done
 }
 
@@ -112,7 +112,7 @@ for obj in $OBJ_LIST ; do
                "($resv_size_set != $resv_size_get)"
        fi
 
-       log_must zfs destroy -f $obj
+       log_must_busy zfs destroy -f $obj
 
        new_space_avail=`get_prop available $TESTPOOL`
        new_space_used=`get_prop used $TESTPOOL`

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib
index fde4123be872e60bde573f58942db36b2c4ce9c7..6e2f2ce6fafe96383e0431989a8986416c2d095d 100644 (file)
--- a/tests/zfs-tests/tests/functional/rsend/rsend.kshlib
+++ b/tests/zfs-tests/tests/functional/rsend/rsend.kshlib
@@ -119,13 +119,13 @@ function cleanup_pool
        log_must rm -rf $BACKDIR/*
 
        if is_global_zone ; then
-               log_must zfs destroy -Rf $pool
+               log_must_busy zfs destroy -Rf $pool
        else
                typeset list=$(zfs list -H -r -t all -o name $pool)
                for ds in $list ; do
                        if [[ $ds != $pool ]] ; then
                                if datasetexists $ds ; then
-                                       log_must zfs destroy -Rf $ds
+                                       log_must_busy zfs destroy -Rf $ds
                                fi
                        fi
                done
@@ -296,24 +296,16 @@ function destroy_tree
        typeset -i ret=0
        typeset snap
        for snap in "$@" ; do
-               zfs destroy $snap
-               ret=$?
+               log_must_busy zfs destroy $snap
 
                typeset ds=${snap%%@*}
                typeset type=$(get_prop "type" $ds)
                if [[ $type == "filesystem" ]]; then
                        typeset mntpnt=$(get_prop mountpoint $ds)
-                       ((ret |= $?))
-
-                       if ((ret != 0)); then
-                               rm -r $mntpnt/$snap
-                               ((ret |= $?))
+                       if [[ -n $mntpnt ]]; then
+                               rm -rf $mntpnt/$snap
                        fi
                fi
-
-               if ((ret != 0)); then
-                       return $ret
-               fi
        done
 
        return 0
@@ -516,8 +508,8 @@ function test_fs_setup
        typeset sendpool=${sendfs%%/*}
        typeset recvpool=${recvfs%%/*}
 
-       datasetexists $sendfs && log_must zfs destroy -r $sendpool
-       datasetexists $recvfs && log_must zfs destroy -r $recvpool
+       datasetexists $sendfs && log_must_busy zfs destroy -r $sendpool
+       datasetexists $recvfs && log_must_busy zfs destroy -r $recvpool
 
        if $(datasetexists $sendfs || zfs create -o compress=lz4 $sendfs); then
                mk_files 1000 256 0 $sendfs &
@@ -547,7 +539,7 @@ function test_fs_setup
        fi
 
        if datasetexists $streamfs; then
-               log_must zfs destroy -r $streamfs
+               log_must_busy zfs destroy -r $streamfs
        fi
        log_must zfs create -o compress=lz4 $sendpool/stream
 }
@@ -664,8 +656,8 @@ function resume_cleanup
        typeset sendfs=$1
        typeset streamfs=$2
 
-       datasetexists $sendfs && log_must zfs destroy -r $sendfs
-       datasetexists $streamfs && log_must zfs destroy -r $streamfs
+       datasetexists $sendfs && log_must_busy zfs destroy -r $sendfs
+       datasetexists $streamfs && log_must_busy zfs destroy -r $streamfs
        cleanup_pool $POOL2
        rm -f /$POOL/initial.zsend /$POOL/incremental.zsend
 }

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh
index 11d48ef0dc8db5f10ecbb6f638b6ccd36be18060..2ace6737b14e3a260a0bb82b561e8040cda92725 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_003_pos.ksh
@@ -59,7 +59,7 @@ if is_global_zone ; then
        #
        log_must eval "zfs send -I $POOL2@psnap $POOL2/pclone@final > " \
                "$BACKDIR/pool-clone-I"
-       log_must zfs destroy -rf $POOL2/pclone
+       log_must_busy zfs destroy -rf $POOL2/pclone
        log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/pool-clone-I"
        log_must cmp_ds_subs $POOL $POOL2
        log_must cmp_ds_cont $POOL $POOL2
@@ -73,7 +73,7 @@ dstds=$(get_dst_ds $POOL $POOL2)
 ds=$dstds/$FS/fs1
 log_must eval "zfs send -I $ds/fs2@fsnap $ds/fclone@final > " \
        "$BACKDIR/fs-clone-I"
-log_must zfs destroy -rf $ds/fclone
+log_must_busy zfs destroy -rf $ds/fclone
 log_must eval "zfs receive -F $ds/fclone < $BACKDIR/fs-clone-I"
 
 log_must cmp_ds_subs $POOL $dstds
@@ -86,7 +86,7 @@ if is_global_zone ; then
        ds=$POOL2/$FS
        log_must eval "zfs send -I $ds/vol@vsnap $ds/vclone@final > " \
                "$BACKDIR/vol-clone-I"
-       log_must zfs destroy -rf $ds/vclone
+       log_must_busy zfs destroy -rf $ds/vclone
        log_must eval "zfs receive -d -F $POOL2 < $BACKDIR/vol-clone-I"
        log_must cmp_ds_subs $POOL $POOL2
        log_must cmp_ds_cont $POOL $POOL2

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh
index 1578283fc707de2fc9952646a44b5f3238c1ccc0..7d858d92d5118b89bf1c513489170f7b3be42237 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_009_pos.ksh
@@ -48,10 +48,10 @@ verify_runnable "global"
 function cleanup
 {
        if datasetexists bpool ; then
-               log_must zpool destroy -f bpool
+               log_must_busy zpool destroy -f bpool
        fi
        if datasetexists spool ; then
-               log_must zpool destroy -f spool
+               log_must_busy zpool destroy -f spool
        fi
 }
 
@@ -80,7 +80,7 @@ log_must ismounted spool
 # Test out of space on top filesystem
 #
 log_must mv /bpool/fs/file /bpool
-log_must zfs destroy -rf bpool/fs
+log_must_busy zfs destroy -rf bpool/fs
 
 log_must zfs snapshot bpool@snap
 log_must eval "zfs send -R bpool@snap > $BACKDIR/bpool-R"

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh
index 3b6cc29372c0a76bb59286592d8dc9b286dc51e9..9ecd18d87da677680dedb9b30bf0e2a2ef9a10a3 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_011_pos.ksh
@@ -45,9 +45,6 @@ verify_runnable "both"
 
 function cleanup
 {
-       export __ZFS_POOL_RESTRICT="$POOL $POOL2"
-       log_must zfs unmount -a
-       unset __ZFS_POOL_RESTRICT
        log_must cleanup_pool $POOL
        log_must cleanup_pool $POOL2
 

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh
index 80644ee2093037f29806d4562b8a1b8328dacd49..110e56144674abc313f0e5ad0d2c3d6a4b22788f 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_013_pos.ksh
@@ -66,11 +66,11 @@ cleanup
 log_must eval "zfs send -R $POOL@final > $BACKDIR/pool-final-R"
 log_must eval "zfs receive -dF $POOL2 < $BACKDIR/pool-final-R"
 
-log_must zfs destroy -Rf $POOL/$FS
-log_must zfs destroy -Rf $POOL/pclone
+log_must_busy zfs destroy -Rf $POOL/$FS
+log_must_busy zfs destroy -Rf $POOL/pclone
 
 if is_global_zone ; then
-       log_must zfs destroy -Rf $POOL/vol
+       log_must_busy zfs destroy -Rf $POOL/vol
 fi
 log_must zfs snapshot -r $POOL@destroy
 

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh
index 29bdc4f840ddb2d6c7ed34e7ceebdf27d0c5ac93..9f1530b45500e6457528508e7b21c8fe43bf9c63 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_020_pos.ksh
@@ -33,6 +33,11 @@
 
 verify_runnable "both"
 
+# See issue: https://github.com/zfsonlinux/zfs/issues/6446
+if is_linux; then
+       log_unsupported "Test often hangs. Skipping."
+fi
+
 log_assert "Verify resumability of full ZFS send/receive with the -D " \
     "(dedup) flag"
 

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh
index d74844890a9242b3c322b8472642fd0cba82be44..54d56c9a69a45938e2dbbf98f370f072ac9061b1 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_021_pos.ksh
@@ -35,6 +35,11 @@
 
 verify_runnable "both"
 
+# See issue: https://github.com/zfsonlinux/zfs/issues/6446
+if is_linux; then
+       log_unsupported "Test often hangs. Skipping."
+fi
+
 log_assert "Verify resumability of a full and incremental ZFS send/receive " \
     "with the -e (embedded) flag"
 

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh
index 487a01426b07cf00dc1cf40fcf63d08d676c376e..ff51fecafc42c867eb490117c37119913634d95e 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_022_pos.ksh
@@ -54,10 +54,10 @@ log_onexit resume_cleanup $sendfs $streamfs
 
 test_fs_setup $sendfs $recvfs
 log_must zfs bookmark $sendfs@a $sendfs#bm_a
-log_must zfs destroy $sendfs@a
+log_must_busy zfs destroy $sendfs@a
 log_must zfs receive -v $recvfs </$POOL/initial.zsend
 resume_test "zfs send -i \#bm_a $sendfs@b" $streamfs $recvfs
-log_must zfs destroy -r -f $sendfs
+log_must_busy zfs destroy -r -f $sendfs
 log_must zfs receive -v $sendfs </$POOL/initial.zsend
 log_must zfs receive -v $sendfs </$POOL/incremental.zsend
 file_check $sendfs $recvfs

diff --git a/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh b/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh
index e7453a0e1ed2c12351b58365cd031bd81e6ea1ad..bcea6b7286f758adb4e58c536b9541000de622de 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/rsend_024_pos.ksh
@@ -50,7 +50,7 @@ streamfs=$POOL/stream
 log_onexit resume_cleanup $sendfs $streamfs
 
 test_fs_setup $sendfs $recvfs
-log_must zfs unmount $sendfs
+log_must zfs unmount -f $sendfs
 resume_test "zfs send $sendfs" $streamfs $recvfs
 file_check $sendfs $recvfs
 

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh
index 70e200bad121a6e49d0bb494d663cf21fdab5ea7..666e11f702f863d7ed702c2b0959d30cf39c61c1 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_lz4_disabled.ksh
@@ -52,8 +52,8 @@ for compress in off gzip; do
                poolexists $POOL3 && destroy_pool $POOL3
                log_must zpool create $pool_opt $POOL3 $DISK3
 
-               datasetexists $send_ds && log_must zfs destroy -r $send_ds
-               datasetexists $recv_ds && log_must zfs destroy -r $recv_ds
+               datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
+               datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds
 
                log_must zfs create -o compress=$compress $send_ds
                typeset dir=$(get_prop mountpoint $send_ds)
@@ -63,7 +63,7 @@ for compress in off gzip; do
                log_must eval "zfs send -c $send_ds@full >$BACKDIR/full-c"
                log_must eval "zfs recv $recv_ds <$BACKDIR/full-c"
 
-               log_must zfs destroy -r $recv_ds
+               log_must_busy zfs destroy -r $recv_ds
 
                log_must eval "zfs send $send_ds@full >$BACKDIR/full"
                log_must eval "zfs recv $recv_ds <$BACKDIR/full"

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh
index 53c7ce8bd3ff8cd8974bd352922936a6a48527a5..614394e5262b3e488d40c25fb7c8e467af07f9a4 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh
@@ -50,8 +50,8 @@ datasetexists $POOL3 && log_must zpool destroy $POOL3
 log_must zpool create -d $POOL3 $DISK3
 
 for compress in $compress_types; do
-       datasetexists $send_ds && log_must zfs destroy -r $send_ds
-       datasetexists $recv_ds && log_must zfs destroy -r $recv_ds
+       datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
+       datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds
 
        log_must zfs create -o compress=$compress $send_ds
        typeset dir=$(get_prop mountpoint $send_ds)

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh
index e810f712afa08a8248f041b9d058e72fbe73fea4..3677db83f1928c5937c23b42672b79c8b40bebe5 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh
@@ -56,8 +56,8 @@ log_onexit cleanup_pool $POOL2
 write_compressible $BACKDIR ${megs}m
 
 for compress in $compress_types; do
-       datasetexists $send_ds && log_must zfs destroy -r $send_ds
-       datasetexists $send_vol && log_must zfs destroy -r $send_vol
+       datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
+       datasetexists $send_vol && log_must_busy zfs destroy -r $send_vol
        log_must zfs create -o compress=$compress $send_ds
        log_must zfs create -V 1g -o compress=$compress $send_vol
 

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh
index 17bf1f7c09c060f4ef6e87d6d624ba9484dc194c..9b886f81572c25979c531beb1b5c3783534ed77c 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh
@@ -59,7 +59,7 @@ for prop in $(get_rand_compress_any 6); do
                verify_stream_size $BACKDIR/compressed $sendfs
 
                log_must rm $BACKDIR/uncompressed $BACKDIR/compressed
-               log_must zfs destroy -r $sendfs
+               log_must_busy zfs destroy -r $sendfs
        done
 done
 

diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh
index 903cefd57d8eae75506b30c60281f3efe6bc4fbb..d2f8c51d124340d2021a89a483bc9b3d3f2dc813 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh
@@ -29,7 +29,7 @@
 
 function cleanup
 {
-       log_must zfs destroy -r $vol
+       log_must_busy zfs destroy -r $vol
        cleanup_pool $POOL2
 }
 

diff --git a/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh b/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh
index ee7421d083a9e53d82592fbbf0084c73c5371429..84c0a5e3c3d38b4c32d45bbc9d9c90bd0b0210e3 100755 (executable)
--- a/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh
+++ b/tests/zfs-tests/tests/functional/rsend/send-cpL_varied_recsize.ksh
@@ -55,8 +55,8 @@ verify_runnable "both"
 
 function cleanup
 {
-       datasetexists $TESTPOOL/128k && log_must zfs destroy $TESTPOOL/128k
-       datasetexists $TESTPOOL/1m && log_must zfs destroy $TESTPOOL/1m
+       datasetexists $TESTPOOL/128k && log_must_busy zfs destroy $TESTPOOL/128k
+       datasetexists $TESTPOOL/1m && log_must_busy zfs destroy $TESTPOOL/1m
        cleanup_pool $POOL2
        destroy_pool $POOL3
 }
@@ -153,7 +153,7 @@ function check
 
        check_recsize $recv_ds $expected_file_bs $expected_recsize
        $do_size_test && verify_stream_size $stream $recv_ds
-       log_must zfs destroy -r $recv_ds
+       log_must_busy zfs destroy -r $recv_ds
 }
 
 log_assert "Verify compressed send works with datasets of varying recsize."