6 # This file and its contents are supplied under the terms of the
7 # Common Development and Distribution License ("CDDL"), version 1.0.
8 # You may only use this file in accordance with the terms of version
11 # A full copy of the text of the CDDL should have accompanied this
12 # source. A copy of the CDDL is also available via the Internet at
13 # http://www.illumos.org/license/CDDL.
19 # Copyright (c) 2015 by Delphix. All rights reserved.
20 # Copyright (C) 2016 Lawrence Livermore National Security, LLC.
23 BASE_DIR=$(dirname "$0")
24 SCRIPT_COMMON=common.sh
25 if [ -f "${BASE_DIR}/${SCRIPT_COMMON}" ]; then
26 . "${BASE_DIR}/${SCRIPT_COMMON}"
28 echo "Missing helper script ${SCRIPT_COMMON}" && exit 1
31 # shellcheck disable=SC2034
35 DEFAULTWORKDIR=/var/tmp
36 DEFAULTCOREDIR=/var/tmp/zloop
40 echo -e "\n$0 [-t <timeout>] [ -s <vdev size> ] [-c <dump directory>]" \
41 "[ -- [extra ztest parameters]]\n" \
43 " This script runs ztest repeatedly with randomized arguments.\n" \
44 " If a crash is encountered, the ztest logs, any associated\n" \
45 " vdev files, and core file (if one exists) are moved to the\n" \
46 " output directory ($DEFAULTCOREDIR by default). Any options\n" \
47 " after the -- end-of-options marker will be passed to ztest.\n" \
50 " -t Total time to loop for, in seconds. If not provided,\n" \
51 " zloop runs forever.\n" \
52 " -s Size of vdev devices.\n" \
53 " -f Specify working directory for ztest vdev files.\n" \
54 " -c Specify a core dump directory to use.\n" \
55 " -m Max number of core dumps to allow before exiting.\n" \
56 " -l Create 'ztest.core.N' symlink to core directory.\n" \
57 " -h Print this help message.\n" \
63 # shellcheck disable=SC2068
65 # shellcheck disable=SC2181
66 if [[ $? -ne 0 ]]; then
67 # shellcheck disable=SC2145
68 echo "Command failed: $@"
74 origcorepattern="$(cat /proc/sys/kernel/core_pattern)"
75 coreglob="$(grep -E -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*"
77 if [[ $coreglob = "*" ]]; then
78 echo "Setting core file pattern..."
79 echo "core" > /proc/sys/kernel/core_pattern
80 coreglob="$(grep -E -o '^([^|%[:space:]]*)' \
81 /proc/sys/kernel/core_pattern)*"
86 # shellcheck disable=SC2012 disable=2086
87 printf "%s" "$(ls -tr1 $coreglob 2> /dev/null | head -1)"
93 core_id=$($GDB --batch -c "$1" | grep "Core was generated by" | \
95 # shellcheck disable=SC2076
96 if [[ "$core_id" =~ "zdb " ]]; then
105 if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then
106 df -h "$workdir" >>ztest.out
107 coreid=$(date "+zloop-%y%m%d-%H%M%S")
108 foundcrashes=$((foundcrashes + 1))
111 zdbcmd="$ZDB -U "$workdir/zpool.cache" -dddMmDDG ztest"
112 zdbdebug=$($zdbcmd 2>&1)
113 echo -e "$zdbcmd\n" >>ztest.zdb
114 echo "$zdbdebug" >>ztest.zdb
116 dest=$coredir/$coreid
117 or_die mkdir -p "$dest"
118 or_die mkdir -p "$dest/vdev"
120 if [[ $symlink -ne 0 ]]; then
121 or_die ln -sf "$dest" ztest.core.$foundcrashes
124 echo "*** ztest crash found - moving logs to $dest"
126 or_die mv ztest.history "$dest/"
127 or_die mv ztest.zdb "$dest/"
128 or_die mv ztest.out "$dest/"
129 or_die mv "$workdir/ztest*" "$dest/vdev/"
131 if [[ -e "$workdir/zpool.cache" ]]; then
132 or_die mv "$workdir/zpool.cache" "$dest/vdev/"
136 if [[ -f "$core" ]]; then
137 coreprog=$(core_prog "$core")
138 coredebug=$($GDB --batch --quiet \
139 -ex "set print thread-events off" \
140 -ex "printf \"*\n* Backtrace \n*\n\"" \
142 -ex "printf \"*\n* Libraries \n*\n\"" \
143 -ex "info sharedlib" \
144 -ex "printf \"*\n* Threads (full) \n*\n\"" \
146 -ex "printf \"*\n* Backtraces \n*\n\"" \
147 -ex "thread apply all bt" \
148 -ex "printf \"*\n* Backtraces (full) \n*\n\"" \
149 -ex "thread apply all bt full" \
150 -ex "quit" "$coreprog" "$core" 2>&1 | \
153 # Dump core + logs to stored directory
154 echo "$coredebug" >>"$dest/ztest.gdb"
155 or_die mv "$core" "$dest/"
157 # Record info in cores logfile
158 echo "*** core @ $coredir/$coreid/$core:" | \
162 if [[ $coremax -gt 0 ]] &&
163 [[ $foundcrashes -ge $coremax ]]; then
164 echo "exiting... max $coremax allowed cores"
173 # expected format: zloop [-t timeout] [-c coredir] [-- extra ztest args]
174 coredir=$DEFAULTCOREDIR
175 basedir=$DEFAULTWORKDIR
181 while getopts ":ht:m:s:c:f:l" opt; do
183 t ) [[ $OPTARG -gt 0 ]] && timeout=$OPTARG ;;
184 m ) [[ $OPTARG -gt 0 ]] && coremax=$OPTARG ;;
185 s ) [[ $OPTARG ]] && size=$OPTARG ;;
186 c ) [[ $OPTARG ]] && coredir=$OPTARG ;;
187 f ) [[ $OPTARG ]] && basedir=$(readlink -f "$OPTARG") ;;
192 * ) echo "Invalid argument: -$OPTARG";
197 # pass remaining arguments on to ztest
198 shift $((OPTIND - 1))
202 export ASAN_OPTIONS=abort_on_error=1:disable_coredump=0
204 if [[ -f "$(core_file)" ]]; then
205 echo -n "There's a core dump here you might want to look at first... "
211 if [[ ! -d $coredir ]]; then
212 echo "core dump directory ($coredir) does not exist, creating it."
213 or_die mkdir -p "$coredir"
216 if [[ ! -w $coredir ]]; then
217 echo "core dump directory ($coredir) is not writable."
221 or_die rm -f ztest.history
222 or_die rm -f ztest.zdb
223 or_die rm -f ztest.cores
225 ztrc=0 # ztest return value
226 foundcrashes=0 # number of crashes found so far
227 starttime=$(date +%s)
230 # if no timeout was specified, loop forever.
231 while [[ $timeout -eq 0 ]] || [[ $curtime -le $((starttime + timeout)) ]]; do
234 # start each run with an empty directory
235 workdir="$basedir/$rundir"
236 or_die rm -rf "$workdir"
237 or_die mkdir "$workdir"
239 # switch between common arrangements & fully randomized
240 if [[ $((RANDOM % 2)) -eq 0 ]]; then
246 mirrors=$(((RANDOM % 3) * 1))
247 parity=$(((RANDOM % 3) + 1))
248 raidz=$((((RANDOM % 9) + parity + 1) * (RANDOM % 2)))
249 vdevs=$(((RANDOM % 3) + 3))
251 align=$(((RANDOM % 2) * 3 + 9))
252 runtime=$((RANDOM % 100))
253 passtime=$((RANDOM % (runtime / 3 + 1) + 10))
255 zopt="$zopt -m $mirrors"
256 zopt="$zopt -r $raidz"
257 zopt="$zopt -R $parity"
258 zopt="$zopt -v $vdevs"
259 zopt="$zopt -a $align"
260 zopt="$zopt -T $runtime"
261 zopt="$zopt -P $passtime"
262 zopt="$zopt -s $size"
263 zopt="$zopt -f $workdir"
265 # shellcheck disable=SC2124
266 cmd="$ZTEST $zopt $@"
267 desc="$(date '+%m/%d %T') $cmd"
268 echo "$desc" | tee -a ztest.history
269 echo "$desc" >>ztest.out
270 $cmd >>ztest.out 2>&1
272 grep -E '===|WARNING' ztest.out >>ztest.history
279 echo "zloop finished, $foundcrashes crashes found"
281 #restore core pattern
282 echo "$origcorepattern" > /proc/sys/kernel/core_pattern
286 if [[ $foundcrashes -gt 0 ]]; then