From: DRC Date: Tue, 27 Jun 2017 19:22:39 +0000 (-0500) Subject: TJBench: Improve consistency of results X-Git-Tag: 1.5.2~4 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=1db1ce45da2e78d87ff05119b674c71d630926aa;p=libjpeg-turbo TJBench: Improve consistency of results Given that libjpeg-turbo can often process hundreds of megapixels/second on modern hardware, the default of one warmup iteration was essentially meaningless. Furthermore, the -warmup option was a bit clunky, since it required some foreknowledge of how fast the benchmarks were going to execute. This commit introduces a 1-second warmup interval for each benchmark by default, and the -warmup option has been retasked to control the length of that interval. --- diff --git a/ChangeLog.md b/ChangeLog.md index 07a6cf5..d694787 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -59,6 +59,11 @@ more easily defend against one of the progressive JPEG exploits (LJT-01-004) identified in [this report](http://www.libjpeg-turbo.org/pmwiki/uploads/About/TwoIssueswiththeJPEGStandard.pdf). +10. TJBench will now run each benchmark for 1 second prior to starting the +timer, in order to improve the consistency of the results. Furthermore, the +`-warmup` option is now used to specify the amount of warmup time rather than +the number of warmup iterations. + 1.5.1 ===== diff --git a/java/TJBench.java b/java/TJBench.java index 618cc97..415d8e1 100644 --- a/java/TJBench.java +++ b/java/TJBench.java @@ -34,7 +34,7 @@ import org.libjpegturbo.turbojpeg.*; class TJBench { - static int flags = 0, quiet = 0, pf = TJ.PF_BGR, yuvpad = 1, warmup = 1; + static int flags = 0, quiet = 0, pf = TJ.PF_BGR, yuvpad = 1; static boolean compOnly, decompOnly, doTile, doYUV, write = true; static final String[] pixFormatStr = { @@ -55,7 +55,7 @@ class TJBench { static TJScalingFactor sf; static int xformOp = TJTransform.OP_NONE, xformOpt = 0; - static double benchTime = 5.0; + static double benchTime = 5.0, warmup = 1.0; static final double getTime() { @@ -162,7 +162,7 @@ class TJBench { } /* Benchmark */ - iter -= warmup; + iter = -1; elapsed = elapsedDecode = 0.0; while (true) { int tile = 0; @@ -184,11 +184,14 @@ class TJBench { tjd.decompress(dstBuf, x, y, width, pitch, height, pf, flags); } } - iter++; - if (iter >= 1) { - elapsed += getTime() - start; + elapsed += getTime() - start; + if (iter >= 0) { + iter++; if (elapsed >= benchTime) break; + } else if (elapsed >= warmup) { + iter = 0; + elapsed = elapsedDecode = 0.0; } } if(doYUV) @@ -321,7 +324,7 @@ class TJBench { } /* Benchmark */ - iter = -warmup; + iter = -1; elapsed = elapsedEncode = 0.0; while (true) { int tile = 0; @@ -346,11 +349,14 @@ class TJBench { totalJpegSize += jpegSize[tile]; } } - iter++; - if (iter >= 1) { - elapsed += getTime() - start; + elapsed += getTime() - start; + if (iter >= 0) { + iter++; if (elapsed >= benchTime) break; + } else if (elapsed >= warmup) { + iter = 0; + elapsed = elapsedEncode = 0.0; } } if (doYUV) @@ -541,17 +547,20 @@ class TJBench { } } - iter = -warmup; + iter = -1; elapsed = 0.; while (true) { start = getTime(); tjt.transform(jpegBuf, t, flags); jpegSize = tjt.getTransformedSizes(); - iter++; - if (iter >= 1) { - elapsed += getTime() - start; + elapsed += getTime() - start; + if (iter >= 0) { + iter++; if (elapsed >= benchTime) break; + } else if (elapsed >= warmup) { + iter = 0; + elapsed = 0.0; } } t = null; @@ -659,8 +668,9 @@ class TJBench { System.out.println("-grayscale = Perform lossless grayscale conversion prior to decompression"); System.out.println(" test (can be combined with the other transforms above)"); System.out.println("-benchtime = Run each benchmark for at least seconds (default = 5.0)"); - System.out.println("-warmup = Execute each benchmark times to prime the cache before"); - System.out.println(" taking performance measurements (default = 1)"); + System.out.println("-warmup = Run each benchmark for seconds (default = 1.0) prior to"); + System.out.println(" starting the timer, in order to prime the caches and thus improve the"); + System.out.println(" consistency of the results."); System.out.println("-componly = Stop after running compression tests. Do not test decompression."); System.out.println("-nowrite = Do not write reference or output images (improves consistency"); System.out.println(" of performance measurements.)\n"); diff --git a/tjbench.c b/tjbench.c index 12b4efe..21b297c 100644 --- a/tjbench.c +++ b/tjbench.c @@ -46,7 +46,7 @@ #define _throwbmp(m) _throw(m, bmpgeterr()) int flags=TJFLAG_NOREALLOC, componly=0, decomponly=0, doyuv=0, quiet=0, - dotile=0, pf=TJPF_BGR, yuvpad=1, warmup=1, dowrite=1; + dotile=0, pf=TJPF_BGR, yuvpad=1, dowrite=1; char *ext="ppm"; const char *pixFormatStr[TJ_NUMPF]= { @@ -64,7 +64,7 @@ const char *subName[TJ_NUMSAMP]={"444", "422", "420", "GRAY", "440", "411"}; tjscalingfactor *scalingfactors=NULL, sf={1, 1}; int nsf=0; int xformop=TJXOP_NONE, xformopt=0; int (*customFilter)(short *, tjregion, tjregion, int, int, tjtransform *); -double benchtime=5.0; +double benchtime=5.0, warmup=1.0; char *formatName(int subsamp, int cs, char *buf) @@ -146,7 +146,7 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf, } /* Benchmark */ - iter=-warmup; + iter=-1; elapsed=elapsedDecode=0.; while(1) { @@ -176,12 +176,17 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf, _throwtj("executing tjDecompress2()"); } } - iter++; - if(iter>=1) + elapsed+=gettime()-start; + if(iter>=0) { - elapsed+=gettime()-start; + iter++; if(elapsed>=benchtime) break; } + else if(elapsed>=warmup) + { + iter=0; + elapsed=elapsedDecode=0.; + } } if(doyuv) elapsed-=elapsedDecode; @@ -340,7 +345,7 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual, } /* Benchmark */ - iter=-warmup; + iter=-1; elapsed=elapsedEncode=0.; while(1) { @@ -374,12 +379,17 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual, totaljpegsize+=jpegsize[tile]; } } - iter++; - if(iter>=1) + elapsed+=gettime()-start; + if(iter>=0) { - elapsed+=gettime()-start; + iter++; if(elapsed>=benchtime) break; } + else if(elapsed>=warmup) + { + iter=0; + elapsed=elapsedEncode=0.; + } } if(doyuv) elapsed-=elapsedEncode; @@ -623,7 +633,7 @@ int decompTest(char *filename) } } - iter=-warmup; + iter=-1; elapsed=0.; while(1) { @@ -631,12 +641,17 @@ int decompTest(char *filename) if(tjTransform(handle, srcbuf, srcsize, _ntilesw*_ntilesh, jpegbuf, jpegsize, t, flags)==-1) _throwtj("executing tjTransform()"); - iter++; - if(iter>=1) + elapsed+=gettime()-start; + if(iter>=0) { - elapsed+=gettime()-start; + iter++; if(elapsed>=benchtime) break; } + else if(elapsed>=warmup) + { + iter=0; + elapsed=0.; + } } free(t); t=NULL; @@ -765,8 +780,9 @@ void usage(char *progname) printf("-grayscale = Perform lossless grayscale conversion prior to decompression\n"); printf(" test (can be combined with the other transforms above)\n"); printf("-benchtime = Run each benchmark for at least seconds (default = 5.0)\n"); - printf("-warmup = Execute each benchmark times to prime the cache before\n"); - printf(" taking performance measurements (default = 1)\n"); + printf("-warmup = Run each benchmark for seconds (default = 1.0) prior to\n"); + printf(" starting the timer, in order to prime the caches and thus improve the\n"); + printf(" consistency of the results.\n"); printf("-componly = Stop after running compression tests. Do not test decompression.\n"); printf("-nowrite = Do not write reference or output images (improves consistency of\n"); printf(" performance measurements.)\n\n"); @@ -880,13 +896,10 @@ int main(int argc, char *argv[]) } if(!strcasecmp(argv[i], "-warmup") && i=0) - { - warmup=temp; - printf("Warmup runs = %d\n\n", warmup); - } + double temp=atof(argv[++i]); + if(temp>=0.0) warmup=temp; else usage(argv[0]); + printf("Warmup time = %.1f seconds\n\n", warmup); } if(!strcmp(argv[i], "-?")) usage(argv[0]); if(!strcasecmp(argv[i], "-alloc")) flags&=(~TJFLAG_NOREALLOC);