From: Magnus Jacobsson Date: Thu, 30 Jul 2020 20:31:04 +0000 (+0200) Subject: Port rtest.sh ksh code to Python in rtest.py X-Git-Tag: 2.46.0~20^2^2~145^2~12 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=a1b9ffc0c37a435607ac8b1c123c8c542ef1cec9;p=graphviz Port rtest.sh ksh code to Python in rtest.py Resolves https://gitlab.com/graphviz/graphviz/-/issues/1779 The reason this is a separate commit from the previoust commit, which contains the renaming of the file, is to trick git to show the changes as a one-file diff regardless of the similarity index, rather than one deleted and one added file. This makes it easier to compare the ksh code to the Python code. --- diff --git a/rtest/rtest.py b/rtest/rtest.py index 7570f4301..81d06bdc1 100755 --- a/rtest/rtest.py +++ b/rtest/rtest.py @@ -1,4 +1,4 @@ -#!/bin/ksh +#!/usr/bin/env python3 # # Graphviz regression test driver # @@ -7,448 +7,357 @@ # TODO: # Report differences with shared version and with new output. -# bsh, linux-ksh filter -bar=1;echo foo | read bar<<- \! -2 -! -if ((bar==1)) -then - echo "Graphviz test suite requires ksh93" - exit 1 -fi -# csh, ksh88 filter -#(( 1.5 == 1 )) && { echo "Graphviz test suite requires ksh93"; exit 1; } - -TESTFILE=tests.txt # Test specifications -GRAPHDIR=graphs # Directory of input graphs and data -OUTDIR=ndata # Directory for test output -OUTHTML=nhtml # Directory for html test report -REFDIR=${REFDIR-""} # Directory for expected test output -GENERATE= # If set, generate test data -VERBOSE= # If set, give verbose output -NOOP= # If set, just print list of tests -DOT=${DOT-$(whence dot)} # should be $(top_builddir)/cmd/dot/dot_static -DIFFIMG=${DIFFIMG-$(whence diffimg)} - -TESTNAME= # name of test -GRAPH= # graph specification -IDX= -typeset -i i j SUBTESTCNT -typeset -i CRASH_CNT=0 DIFF_CNT=0 TOT_CNT=0 -typeset -i LINECNT=0 -typeset -A TESTTYPES -typeset -a ALG -typeset -a FMT -typeset -a FLAGS -TMPINFILE=tmp$$.gv -TMPFILE1=tmpnew$$ -TMPFILE2=tmpref$$ +import os +import shutil +import tempfile +import subprocess +import sys +import platform +import argparse +import atexit +import pathlib + +TESTFILE = 'tests.txt' # Test specifications +GRAPHDIR = 'graphs' # Directory of input graphs and data +OUTDIR = 'ndata' # Directory for test output +OUTHTML = 'nhtml' # Directory for html test report +REFDIR = os.environ.get('REFDIR', '') # Directory for expected test output +GENERATE = False # If set, generate test data +VERBOSE = False # If set, give verbose output +NOOP = False # If set, just print list of tests +DOT = os.environ.get('DOT', shutil.which('dot')) +DIFFIMG = os.environ.get('DIFFIMG', shutil.which('diffimg')) + +TESTNAME = '' # name of test +GRAPH = '' # graph specification +IDX = '' +CRASH_CNT = 0 +DIFF_CNT = 0 +TOT_CNT = 0 +TESTTYPES = {} +TMPINFILE = 'tmp{}.gv'.format(os.getpid()) +TMPFILE1 = 'tmpnew{}'.format(os.getpid()) +TMPFILE2 = 'tmpref{}'.format(os.getpid()) # Read single line, storing it in LINE and update count. -# Return 0 on success. -function readLine -{ - if read -u 3 LINE - then - (( LINECNT+=1 )) - return 0 - else - return 1 - fi -} +# Returns the line on success, else returns None +def readLine(): + LINE = f3.readline() + if LINE != '': + return LINE.strip() + else: + return None # Skip blank lines and comments (lines starting with #) # Use first real line as the test name -function skipLines -{ - while readLine - do - if [[ -n $LINE && ${LINE:0:1} != \# ]] - then - return 0 - fi - done - return 1 -} +def skipLines(): + while True: + LINE = readLine() + if LINE is None: + return None + if LINE and not LINE.startswith('#'): + return LINE # Subtests have the form: layout format optional_flags # Store the 3 parts in the arrays ALG, FMT, FLAGS. # Stop at a blank line -function readSubtests -{ - (( SUBTESTCNT=0 )) - while readLine - do - if [[ -z "$LINE" ]] - then - return - fi - if [[ ${LINE:0:1} != \# ]] - then - echo $LINE | read ALG0 FMT0 FLAGS0 - ALG[$SUBTESTCNT]=$ALG0 - FMT[$SUBTESTCNT]=$FMT0 - FLAGS[$SUBTESTCNT]=$FLAGS0 - (( SUBTESTCNT+=1 )) - fi - done -} - -function readTest -{ +def readSubtests(): + SUBTESTS = [] + ALG = [] + FMT = [] + FLAGS = [] + while True: + LINE = readLine() + if LINE == '': + return SUBTESTS + if not LINE.startswith('#'): + items = LINE.split(' ') + ALG0, FMT0 = items[0: 2] + FLAGS0 = items[2:] + SUBTESTS.append( + { + 'ALG': ALG0, + 'FMT': FMT0, + 'FLAGS': FLAGS0, + } + ) + +def readTest(): # read test name - if skipLines - then - TESTNAME=$LINE - else - return 1 - fi + LINE = skipLines() + if LINE is not None: + TESTNAME = LINE + else: + return None # read input graph - if skipLines - then - GRAPH=$LINE - else - return 1 - fi - - readSubtests - return 0 -} - -# newfile = $1 -# oldfile = $2 -# assume subscript indicates file type -function strip { - case $1 in - *.ps ) - awk -f strps.awk $1 > $TFILE1 - awk -f strps.awk $2 > $TFILE2 - ;; - *.svg ) - sed '/^$/d' > $TFILE1 - sed '/^$/d' > $TFILE2 - ;; - * ) - cp $1 $TFILE1 - cp $2 $TFILE2 - ;; - esac - -} + LINE = skipLines() + if LINE is not None: + GRAPH = LINE + else: + return None + + SUBTESTS = readSubtests() + return { + 'TESTNAME': TESTNAME, + 'GRAPH': GRAPH, + 'SUBTESTS': SUBTESTS, + } # Compare old and new output and report if different. # Args: testname index fmt -function doDiff -{ - FILE1=$OUTDIR/$OUTFILE - FILE2=$REFDIR/$OUTFILE - F=${3%%:*} - case $F in - ps | ps2 ) - awk -f strps.awk $FILE1 > $TMPFILE1 - awk -f strps.awk $FILE2 > $TMPFILE2 - diff -q $TMPFILE2 $TMPFILE1 > /dev/null - if [[ $? != 0 ]] - then - print -u 2 "Test $1:$2 : == Failed == $OUTFILE" - (( DIFF_CNT+=1 )) - else - if [[ -n "$VERBOSE" ]] - then - print -u 2 "Test $1:$2 : == OK == $OUTFILE" - fi - fi - ;; - svg ) - sed '/^$/d' > $TMPFILE1 - sed '/^$/d' > $TMPFILE2 - diff -q $TMPFILE2 $TMPFILE1 > /dev/null - if [[ $? != 0 ]] - then - print -u 2 "Test $1:$2 : == Failed == $OUTFILE" - (( DIFF_CNT+=1 )) - else - if [[ -n "$VERBOSE" ]] - then - print -u 2 "Test $1:$2 : == OK == $OUTFILE" - fi - fi - ;; - png ) - $DIFFIMG $FILE2 $FILE1 >$OUTHTML/dif_$OUTFILE - if [[ $? != 0 ]] - then - echo "

" >>$OUTHTML/index.html - cp $FILE2 $OUTHTML/old_$OUTFILE - echo "" >>$OUTHTML/index.html - cp $FILE1 $OUTHTML/new_$OUTFILE - echo "" >>$OUTHTML/index.html - echo "" >>$OUTHTML/index.html - print -u 2 "Test $1:$2 : == Failed == $OUTFILE" - (( DIFF_CNT+=1 )) - else - if [[ -n "$VERBOSE" ]] - then - print -u 2 "Test $1:$2 : == OK == $OUTFILE" - fi - rm $OUTHTML/dif_$OUTFILE - fi - ;; - * ) - diff -q $FILE2 $FILE1 > /dev/null - if [[ $? != 0 ]] - then - print -u 2 "Test $1:$2 : == Failed == $OUTFILE" - (( DIFF_CNT+=1 )) - else - if [[ -n "$VERBOSE" ]] - then - print -u 2 "Test $1:$2 : == OK == $OUTFILE" - fi - fi - ;; - esac -} +def doDiff(OUTFILE, OUTDIR, REFDIR, testname, subtest_index, fmt): + global OUTHTML + global DIFF_CNT + global VERBOSE + FILE1 = os.path.join(OUTDIR, OUTFILE) + FILE2 = os.path.join(REFDIR, OUTFILE) + F = fmt.split(':')[0] + if F in ['ps', 'ps2']: + with open(TMPFILE1, mode='w') as fd1, \ + open(TMPFILE2, mode='w') as fd2: + subprocess.check_call( + ['awk', '-f', 'strps.awk', FILE1], + stdout=fd1, + ) + subprocess.check_call( + ['awk', '-f', 'strps.awk', FILE2], + stdout=fd2, + ) + returncode = subprocess.call( + ['diff', '-q', TMPFILE1, TMPFILE2], + stdout=subprocess.DEVNULL, + ) + elif F == 'svg': + with open(TMPFILE1, mode='w') as fd1, \ + open(TMPFILE2, mode='w') as fd2: + subprocess.check_call( + ['sed', '/^$/d', FILE1], + stdout=fd1, + ) + subprocess.check_call( + ['sed', '/^$/d', FILE2], + stdout=fd2, + ) + returncode = subprocess.call( + ['diff', '-q', '--strip-trailing-cr', TMPFILE1, TMPFILE2], + stdout=subprocess.DEVNULL, + ) + elif F == 'png': + returncode = subprocess.call( + [DIFFIMG, FILE1, FILE2, os.path.join(OUTHTML, 'dif_' + OUTFILE)], + ) + if returncode != 0: + with open(os.path.join(OUTHTML, 'index.html'), mode='a') as fd: + print('

', file=fd) + shutil.copyfile(FILE2, os.path.join(OUTHTML, 'old_' + OUTFILE)) + print('', file=fd) + shutil.copyfile(FILE1, os.path.join(OUTHTML, 'new_' + OUTFILE)) + print('', file=fd) + print('', file=fd) + else: + os.unlink(os.path.join(OUTHTML, 'dif_' + OUTFILE)) + else: + returncode = subprocess.call( + ['diff', '--strip-trailing-cr', FILE2, FILE1], + stdout=subprocess.DEVNULL, + ) + if returncode != 0: + print('Test {0}:{1} : == Failed == {2}'.format(testname, subtest_index, OUTFILE), file=sys.stderr) + DIFF_CNT += 1 + else: + if VERBOSE: + print('Test {0}:{1} : == OK == {2}'.format(testname, subtest_index, OUTFILE), file=sys.stderr) # Generate output file name given 3 parameters. # testname layout format # If format ends in :*, remove this, change the colons to underscores, # and append to basename # If the last two parameters have been used before, add numeric suffix. -function genOutname -{ - if [[ $3 == *:* ]] - then - F=${3%%:*} - XFMT=${3#$F} - XFMT=${XFMT/:/_} - else - F=$3 - XFMT="" - fi - - IDX="$2$XFMT$F" - j=${TESTTYPES[$IDX]} - if (( j == 0 )) - then - TESTTYPES[$IDX]=1 - J="" - else - TESTTYPES[$IDX]=$(( j+1 )) - J=$j - fi - OUTFILE="$1_$2$XFMT$J.$F" -} - -# clear out all entries of associated array -function aunset #name -{ - typeset i - nameref v=$1 - for i in ${!v[@]} - do unset v[$i] - done -} - -function doTest -{ - if (( SUBTESTCNT == 0 )) - then +def genOutname(name, alg, fmt): + global TESTTYPES + fmt_split = fmt.split(':') + if len(fmt_split) >= 2: + F = fmt_split[0] + XFMT = '_' + '_'.join(fmt_split[1:]) + else: + F=fmt + XFMT='' + + IDX = alg + XFMT + F + j = TESTTYPES.get(IDX, 0) + if j == 0: + TESTTYPES[IDX] = 1 + J = '' + else: + TESTTYPES[IDX]= j + 1 + J = str(j) + OUTFILE = name + '_' + alg + XFMT + J + '.' + F + return OUTFILE + +def doTest(TEST): + global GENERATE + global TOT_CNT + global CRASH_CNT + global DIFF_CNT + global TESTTYPES + TESTNAME = TEST['TESTNAME'] + SUBTESTS = TEST['SUBTESTS'] + if len(SUBTESTS) == 0: + return + GRAPH = TEST['GRAPH'] + if GRAPH == '=': + INFILE = os.path.join(GRAPHDIR, TESTNAME + '.gv') + elif GRAPH.startswith('graph') or GRAPH.startswith('digraph'): + with open(TMPINFILE, mode='w') as fd: + fd.write(GRAPH) + INFILE = TMPINFILE + elif os.path.splitext(GRAPH)[1] == '.gv': + INFILE = os.path.join(GRAPHDIR, GRAPH) + else: + print('Unknown graph spec, test {} - ignoring'.format(TESTNAME), + file=sys.stderr) return - fi - case $GRAPH in - = ) - INFILE=$GRAPHDIR/$TESTNAME.gv - ;; - graph* | digraph* ) - INFILE=$TMPINFILE - echo "$GRAPH" > $INFILE - ;; - *.gv ) - INFILE=$GRAPHDIR/$GRAPH - ;; - * ) - echo "Unknown graph spec, test $TESTNAME - ignoring" - return - ;; - esac - for ((i=0;i errout - RVAL=$? - if [[ -s errout ]] - then - cat errout - fi - - if [[ $RVAL != 0 || ! -s $OUTPATH ]] - then - (( CRASH_CNT+=1 )) - print -u 2 "Test $TESTNAME:$i : == Layout failed ==" - print -u 2 " $testcmd" - elif [[ $GENERATE == 1 ]] - then + result = subprocess.Popen( + testcmd, + universal_newlines=True, + stderr = subprocess.PIPE, + ) + _, errout = result.communicate() + RVAL = result.returncode + + if errout: + print(errout) + + if RVAL != 0 or not os.path.exists(OUTPATH): + CRASH_CNT += 1 + print('Test {0}:{1} : == Layout failed =='.format(TESTNAME, i), file=sys.stderr) + print(' ' + ' '.join(testcmd), file=sys.stderr) + elif GENERATE == 1: continue - elif [[ -r $REFDIR/$OUTFILE ]] - then - doDiff $TESTNAME $i ${FMT[$i]} - else - print -u 2 "Test $TESTNAME:$i : == No file $REFDIR/$OUTFILE for comparison ==" - fi - done + elif os.path.exists(os.path.join(REFDIR, OUTFILE)): + doDiff(OUTFILE, OUTDIR, REFDIR, TESTNAME, i, SUBTEST['FMT']) + else: + print('Test {0}:{1} : == No file {2}/{3} for comparison =='.format(TESTNAME, i, REFDIR, OUTFILE), file=sys.stderr) # clear TESTTYPES - aunset TESTTYPES -# for W in ${!TESTTYPES[@]} -# do -# TESTTYPES[$W]=0 -# done -} - -trap 'rm -f $TMPFILE1 $TMPFILE2 $TMPINFILE errout; exit' 0 1 2 3 15 + TESTTYPES = {} -Usage='rtest [-gvn] [TESTFILE]\n - -g : generate test data\n - -v : verbose\n - -n : print test' +def cleanup(): + pathlib.Path(TMPFILE1).unlink(missing_ok=True) + pathlib.Path(TMPFILE2).unlink(missing_ok=True) + pathlib.Path(TMPINFILE).unlink(missing_ok=True) +atexit.register(cleanup) # Set REFDIR -if [[ ! "$REFDIR" ]] -then - SYSTYPE=$(uname -s) - case "$SYSTYPE" in - Linux*) - REFDIR=linux.x86 - ;; - Darwin*) - REFDIR=macosx - ;; - *) - print "Unrecognized system \"$SYSTYPE\"" - REFDIR=nshare - ;; - esac -fi - -while getopts :gnv c -do - case $c in - n ) - VERBOSE=1 - NOOP=1 - ;; - v ) - VERBOSE=1 - ;; - g ) - GENERATE=1 - if [[ ! -d "$REFDIR" ]] - then - mkdir $REFDIR - fi - OUTDIR=$REFDIR - ;; - :) - echo $OPTARG requires a value - exit 2 - ;; - \? ) - if [[ "$OPTARG" == '?' ]] - then - print $Usage - exit 0 - else - echo "rtest: unknown flag $OPTARG - ignored" - fi - ;; - esac -done -shift $((OPTIND-1)) - -if [[ $# > 0 ]] -then - if [[ -r $1 ]] - then - TESTFILE=$1 - else - print -u 2 "Test file $1 does not exist" - exit 1 - fi -fi +REFDIR = os.environ.get('REFDIR') +if not REFDIR: + if platform.system() == 'Linux': + REFDIR = 'linux.x86' + elif platform.system() == 'Darwin': + REFDIR = 'macosx' + else: + print('Unrecognized system "{0}"'.format(platform.system()), file=sys.stderr) + REFDIR = 'nshare' + +parser = argparse.ArgumentParser(description='Run regression tests.') +parser.add_argument('-g', + dest='generate', + action='store_true', + help='generate test data' +) +parser.add_argument('-v', + dest='verbose', + action='store_true', + help='verbose' +) +parser.add_argument('-n', + dest='noop', + action='store_true', + help='noop' +) +parser.add_argument('testfile', + nargs='?', + help='test files' +) +args = parser.parse_args() +VERBOSE = args.verbose or args.noop +NOOP = args.noop +GENERATE = args.generate +if GENERATE: + if not os.path.isdir(REFDIR): + os.mkdir(OUTDIR) + OUTDIR = REFDIR + +if args.testfile: + if os.path.exists(args.testfile): + TESTFILE = args.testfile + else: + print('Test file {0} does not exist'.format(args.testfile), file=sys.stderr) + sys.exit(1) # Check environment and initialize -if [[ $NOOP != 1 ]] -then -if [[ ! -d "$REFDIR" ]] -then - print -u 2 "Test data directory $REFDIR does not exist" - exit 1 -fi - -if [[ ! -d "$OUTDIR" ]] -then - mkdir $OUTDIR -fi - -if [[ ! -d "$OUTHTML" ]] -then - mkdir $OUTHTML -fi -rm -f $OUTHTML/* - -if [[ ! "$DOT" ]] -then - print -u 2 "Could not find a value for DOT" - exit -fi -if [[ ! -x $DOT ]] -then - print -u 2 "$DOT program is not executable" - exit 1 -fi - -if [[ $GENERATE != 1 && ! -x $DIFFIMG ]] -then - print -u 2 "$DIFFIMG program is not executable" - exit 1 -fi -fi - - -exec 3< $TESTFILE -while readTest -do - doTest -done -if [[ $NOOP == 1 ]] -then -print -u 2 "No. tests: $TOT_CNT" -elif [[ $GENERATE == 1 ]] -then -print -u 2 "No. tests: $TOT_CNT Layout failures: $CRASH_CNT" -else -print -u 2 "No. tests: $TOT_CNT Layout failures: $CRASH_CNT Changes: $DIFF_CNT" -fi -(( EXIT_STATUS=CRASH_CNT+DIFF_CNT )) -exit $EXIT_STATUS +if not NOOP: + if not os.path.isdir(REFDIR): + print('Test data directory {0} does not exist'.format(REFDIR), + file=sys.stderr) + sys.exit(1) + +if not os.path.isdir(OUTDIR): + os.mkdir(OUTDIR) + +if not os.path.isdir(OUTHTML): + os.mkdir(OUTHTML) +for filename in os.listdir(OUTHTML): + os.unlink(os.path.join(OUTHTML, filename)) + +if not DOT: + print('Could not find a value for DOT', file=sys.stderr) + sys.exit(1) +if not os.path.isfile(DOT) or not os.access(DOT, os.X_OK): + print('{0} program is not executable'.format(DOT)) + sys.exit(1) + +if not GENERATE: + if DIFFIMG: + if not os.path.isfile(DIFFIMG) or not os.access(DIFFIMG, os.X_OK): + print('{0} program is not executable'.format(DIFFIMG)) + sys.exit(1) + else: + print('Could not find a value for DIFFIMG', file=sys.stderr) + sys.exit(1) + + +f3 = open(TESTFILE) +while True: + TEST = readTest() + if TEST is None: + break + doTest(TEST) +if NOOP == 1: + print('No. tests: ' + str(TOT_CNT), file=sys.stderr) +elif GENERATE == 1: + print('No. tests: ' + str(TOT_CNT) + ' Layout failures: ' + str(CRASH_CNT), file=sys.stderr) +else: + print('No. tests: ' + + str(TOT_CNT) + ' Layout failures: ' + + str(CRASH_CNT) + ' Changes: ' + + str(DIFF_CNT), file=sys.stderr) +EXIT_STATUS = CRASH_CNT + DIFF_CNT +sys.exit(EXIT_STATUS)