TJBench: Improve consistency of results

Given that libjpeg-turbo can often process hundreds of megapixels/second
on modern hardware, the default of one warmup iteration was essentially
meaningless.  Furthermore, the -warmup option was a bit clunky, since
it required some foreknowledge of how fast the benchmarks were going to
execute.

This commit introduces a 1-second warmup interval for each benchmark by
default, and the -warmup option has been retasked to control the length
of that interval.

diff --git a/ChangeLog.md b/ChangeLog.md
index 07a6cf526d345ddfdf008823e938de2531024db6..d694787946f311635619d870df36ad27da0d4468 100644 (file)
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -59,6 +59,11 @@ more easily defend against one of the progressive JPEG exploits (LJT-01-004)
 identified in
 [this report](http://www.libjpeg-turbo.org/pmwiki/uploads/About/TwoIssueswiththeJPEGStandard.pdf).
 
+10. TJBench will now run each benchmark for 1 second prior to starting the
+timer, in order to improve the consistency of the results.  Furthermore, the
+`-warmup` option is now used to specify the amount of warmup time rather than
+the number of warmup iterations.
+
 
 1.5.1
 =====

diff --git a/java/TJBench.java b/java/TJBench.java
index 618cc977dfddcb065418f5be1aa80c16fc526ee7..415d8e1bd0f56c3fa9f4f11edd1abdc15790f6b8 100644 (file)
--- a/java/TJBench.java
+++ b/java/TJBench.java
@@ -34,7 +34,7 @@ import org.libjpegturbo.turbojpeg.*;
 
 class TJBench {
 
-  static int flags = 0, quiet = 0, pf = TJ.PF_BGR, yuvpad = 1, warmup = 1;
+  static int flags = 0, quiet = 0, pf = TJ.PF_BGR, yuvpad = 1;
   static boolean compOnly, decompOnly, doTile, doYUV, write = true;
 
   static final String[] pixFormatStr = {
@@ -55,7 +55,7 @@ class TJBench {
 
   static TJScalingFactor sf;
   static int xformOp = TJTransform.OP_NONE, xformOpt = 0;
-  static double benchTime = 5.0;
+  static double benchTime = 5.0, warmup = 1.0;
 
 
   static final double getTime() {
@@ -162,7 +162,7 @@ class TJBench {
     }
 
     /* Benchmark */
-    iter -= warmup;
+    iter = -1;
     elapsed = elapsedDecode = 0.0;
     while (true) {
       int tile = 0;
@@ -184,11 +184,14 @@ class TJBench {
             tjd.decompress(dstBuf, x, y, width, pitch, height, pf, flags);
         }
       }
-      iter++;
-      if (iter >= 1) {
-        elapsed += getTime() - start;
+      elapsed += getTime() - start;
+      if (iter >= 0) {
+        iter++;
         if (elapsed >= benchTime)
           break;
+      } else if (elapsed >= warmup) {
+        iter = 0;
+        elapsed = elapsedDecode = 0.0;
       }
     }
     if(doYUV)
@@ -321,7 +324,7 @@ class TJBench {
       }
 
       /* Benchmark */
-      iter = -warmup;
+      iter = -1;
       elapsed = elapsedEncode = 0.0;
       while (true) {
         int tile = 0;
@@ -346,11 +349,14 @@ class TJBench {
             totalJpegSize += jpegSize[tile];
           }
         }
-        iter++;
-        if (iter >= 1) {
-          elapsed += getTime() - start;
+        elapsed += getTime() - start;
+        if (iter >= 0) {
+          iter++;
           if (elapsed >= benchTime)
             break;
+        } else if (elapsed >= warmup) {
+          iter = 0;
+          elapsed = elapsedEncode = 0.0;
         }
       }
       if (doYUV)
@@ -541,17 +547,20 @@ class TJBench {
           }
         }
 
-        iter = -warmup;
+        iter = -1;
         elapsed = 0.;
         while (true) {
           start = getTime();
           tjt.transform(jpegBuf, t, flags);
           jpegSize = tjt.getTransformedSizes();
-          iter++;
-          if (iter >= 1) {
-            elapsed += getTime() - start;
+          elapsed += getTime() - start;
+          if (iter >= 0) {
+            iter++;
             if (elapsed >= benchTime)
               break;
+          } else if (elapsed >= warmup) {
+            iter = 0;
+            elapsed = 0.0;
           }
         }
         t = null;
@@ -659,8 +668,9 @@ class TJBench {
     System.out.println("-grayscale = Perform lossless grayscale conversion prior to decompression");
     System.out.println("     test (can be combined with the other transforms above)");
     System.out.println("-benchtime <t> = Run each benchmark for at least <t> seconds (default = 5.0)");
-    System.out.println("-warmup <w> = Execute each benchmark <w> times to prime the cache before");
-    System.out.println("     taking performance measurements (default = 1)");
+    System.out.println("-warmup <t> = Run each benchmark for <t> seconds (default = 1.0) prior to");
+    System.out.println("     starting the timer, in order to prime the caches and thus improve the");
+    System.out.println("     consistency of the results.");
     System.out.println("-componly = Stop after running compression tests.  Do not test decompression.");
     System.out.println("-nowrite = Do not write reference or output images (improves consistency");
     System.out.println("     of performance measurements.)\n");

diff --git a/tjbench.c b/tjbench.c
index 12b4efe558b787a0a46ec6e2e050dec06c68e34b..21b297c62897576b5c4b44557f6620caf5314bb9 100644 (file)
--- a/tjbench.c
+++ b/tjbench.c
@@ -46,7 +46,7 @@
 #define _throwbmp(m) _throw(m, bmpgeterr())
 
 int flags=TJFLAG_NOREALLOC, componly=0, decomponly=0, doyuv=0, quiet=0,
-       dotile=0, pf=TJPF_BGR, yuvpad=1, warmup=1, dowrite=1;
+       dotile=0, pf=TJPF_BGR, yuvpad=1, dowrite=1;
 char *ext="ppm";
 const char *pixFormatStr[TJ_NUMPF]=
 {
@@ -64,7 +64,7 @@ const char *subName[TJ_NUMSAMP]={"444", "422", "420", "GRAY", "440", "411"};
 tjscalingfactor *scalingfactors=NULL, sf={1, 1};  int nsf=0;
 int xformop=TJXOP_NONE, xformopt=0;
 int (*customFilter)(short *, tjregion, tjregion, int, int, tjtransform *);
-double benchtime=5.0;
+double benchtime=5.0, warmup=1.0;
 
 
 char *formatName(int subsamp, int cs, char *buf)
@@ -146,7 +146,7 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf,
        }
 
        /* Benchmark */
-       iter=-warmup;
+       iter=-1;
        elapsed=elapsedDecode=0.;
        while(1)
        {
@@ -176,12 +176,17 @@ int decomp(unsigned char *srcbuf, unsigned char **jpegbuf,
                                                _throwtj("executing tjDecompress2()");
                        }
                }
-               iter++;
-               if(iter>=1)
+               elapsed+=gettime()-start;
+               if(iter>=0)
                {
-                       elapsed+=gettime()-start;
+                       iter++;
                        if(elapsed>=benchtime) break;
                }
+               else if(elapsed>=warmup)
+               {
+                       iter=0;
+                       elapsed=elapsedDecode=0.;
+               }
        }
        if(doyuv) elapsed-=elapsedDecode;
 
@@ -340,7 +345,7 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual,
                }
 
                /* Benchmark */
-               iter=-warmup;
+               iter=-1;
                elapsed=elapsedEncode=0.;
                while(1)
                {
@@ -374,12 +379,17 @@ int fullTest(unsigned char *srcbuf, int w, int h, int subsamp, int jpegqual,
                                        totaljpegsize+=jpegsize[tile];
                                }
                        }
-                       iter++;
-                       if(iter>=1)
+                       elapsed+=gettime()-start;
+                       if(iter>=0)
                        {
-                               elapsed+=gettime()-start;
+                               iter++;
                                if(elapsed>=benchtime) break;
                        }
+                       else if(elapsed>=warmup)
+                       {
+                               iter=0;
+                               elapsed=elapsedEncode=0.;
+                       }
                }
                if(doyuv) elapsed-=elapsedEncode;
 
@@ -623,7 +633,7 @@ int decompTest(char *filename)
                                }
                        }
 
-                       iter=-warmup;
+                       iter=-1;
                        elapsed=0.;
                        while(1)
                        {
@@ -631,12 +641,17 @@ int decompTest(char *filename)
                                if(tjTransform(handle, srcbuf, srcsize, _ntilesw*_ntilesh, jpegbuf,
                                        jpegsize, t, flags)==-1)
                                        _throwtj("executing tjTransform()");
-                               iter++;
-                               if(iter>=1)
+                               elapsed+=gettime()-start;
+                               if(iter>=0)
                                {
-                                       elapsed+=gettime()-start;
+                                       iter++;
                                        if(elapsed>=benchtime) break;
                                }
+                               else if(elapsed>=warmup)
+                               {
+                                       iter=0;
+                                       elapsed=0.;
+                               }
                        }
 
                        free(t);  t=NULL;
@@ -765,8 +780,9 @@ void usage(char *progname)
        printf("-grayscale = Perform lossless grayscale conversion prior to decompression\n");
        printf("     test (can be combined with the other transforms above)\n");
        printf("-benchtime <t> = Run each benchmark for at least <t> seconds (default = 5.0)\n");
-       printf("-warmup <w> = Execute each benchmark <w> times to prime the cache before\n");
-       printf("     taking performance measurements (default = 1)\n");
+       printf("-warmup <t> = Run each benchmark for <t> seconds (default = 1.0) prior to\n");
+       printf("     starting the timer, in order to prime the caches and thus improve the\n");
+       printf("     consistency of the results.\n");
        printf("-componly = Stop after running compression tests.  Do not test decompression.\n");
        printf("-nowrite = Do not write reference or output images (improves consistency of\n");
        printf("     performance measurements.)\n\n");
@@ -880,13 +896,10 @@ int main(int argc, char *argv[])
                        }
                        if(!strcasecmp(argv[i], "-warmup") && i<argc-1)
                        {
-                               int temp=atoi(argv[++i]);
-                               if(temp>=0)
-                               {
-                                       warmup=temp;
-                                       printf("Warmup runs = %d\n\n", warmup);
-                               }
+                               double temp=atof(argv[++i]);
+                               if(temp>=0.0) warmup=temp;
                                else usage(argv[0]);
+                               printf("Warmup time = %.1f seconds\n\n", warmup);
                        }
                        if(!strcmp(argv[i], "-?")) usage(argv[0]);
                        if(!strcasecmp(argv[i], "-alloc")) flags&=(~TJFLAG_NOREALLOC);