Add log_must_{retry,busy} helpers

Add helpers which automatically retry the provided command when
the error message matches the provided keyword.  This provides an
easy way to handle the asynchronous nature of some ZFS commands.

For example, the `zfs destroy` command may need to be retried in
the case where the block device is unexpected busy.  This can be
accomplished as follows:

  log_must_busy $ZFS destroy ...

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Issue #5002

diff --git a/tests/test-runner/include/logapi.shlib b/tests/test-runner/include/logapi.shlib
index 6fd4ab369256443f2c7e02b58bc694fcea3b9369..b970e4eccce76ef642ce242b7d16565b8e92100a 100644 (file)
--- a/tests/test-runner/include/logapi.shlib
+++ b/tests/test-runner/include/logapi.shlib
@@ -69,6 +69,81 @@ function log_must
        (( $? != 0 )) && log_fail
 }
 
+# Execute a positive test but retry the command on failure if the output
+# matches an expected pattern.  Otherwise behave like log_must and exit
+# $STF_FAIL is test fails.
+#
+# $1 - retry keyword
+# $2 - retry attempts
+# $3-$@ - command to execute
+#
+function log_must_retry
+{
+       typeset out=""
+       typeset logfile="/tmp/log.$$"
+       typeset status=1
+       typeset expect=$1
+       typeset retry=$2
+       typeset delay=1
+       shift 2
+
+       while [[ -e $logfile ]]; do
+               logfile="$logfile.$$"
+       done
+
+       while (( $retry > 0 )); do
+               "$@" 2>$logfile
+               status=$?
+               out="$CAT $logfile"
+
+               if (( $status == 0 )); then
+                       $out | $EGREP -i "internal error|assertion failed" \
+                               > /dev/null 2>&1
+                       # internal error or assertion failed
+                       if [[ $? -eq 0 ]]; then
+                               print -u2 $($out)
+                               _printerror "$@" "internal error or" \
+                                       " assertion failure exited $status"
+                               status=1
+                       else
+                               [[ -n $LOGAPI_DEBUG ]] && print $($out)
+                               _printsuccess "$@"
+                       fi
+                       break
+               else
+                       $out | $GREP -i "$expect" > /dev/null 2>&1
+                       if (( $? == 0 )); then
+                               print -u2 $($out)
+                               _printerror "$@" "Retry in $delay seconds"
+                               $SLEEP $delay
+
+                               (( retry=retry - 1 ))
+                               (( delay=delay * 2 ))
+                       else
+                               break;
+                       fi
+               fi
+       done
+
+       if (( $status != 0 )) ; then
+               print -u2 $($out)
+               _printerror "$@" "exited $status"
+       fi
+
+       _recursive_output $logfile "false"
+       return $status
+}
+
+# Execute a positive test and exit $STF_FAIL is test fails after being
+# retried up to 5 times when the command returns the keyword "busy".
+#
+# $@ - command to execute
+function log_must_busy
+{
+       log_must_retry "busy" 5 "$@"
+       (( $? != 0 )) && log_fail
+}
+
 # Execute a negative test and exit $STF_FAIL if test passes
 #
 # $@ - command to execute