TurboJPEG: Properly handle gigapixel images

Prevent several integer overflow issues and subsequent segfaults that
occurred when attempting to compress or decompress gigapixel images with
the TurboJPEG API:

- Modify tjBufSize(), tjBufSizeYUV2(), and tjPlaneSizeYUV() to avoid
  integer overflow when computing the return values and to return an
  error if such an overflow is unavoidable.
- Modify tjunittest to validate the above.
- Modify tjCompress2(), tjEncodeYUVPlanes(), tjDecompress2(), and
  tjDecodeYUVPlanes() to avoid integer overflow when computing the row
  pointers in the 64-bit TurboJPEG C API.
- Modify TJBench (both C and Java versions) to avoid overflowing the
  size argument to malloc()/new and to fail gracefully if such an
  overflow is unavoidable.

In general, this allows gigapixel images to be accommodated by the
64-bit TurboJPEG C API when using automatic JPEG buffer (re)allocation.
Such images cannot currently be accommodated without automatic JPEG
buffer (re)allocation, due to the fact that tjAlloc() accepts a 32-bit
integer argument (oops.)  Such images cannot be accommodated in the
TurboJPEG Java API due to the fact that Java always uses a signed 32-bit
integer as an array index.

Fixes #361

diff --git a/ChangeLog.md b/ChangeLog.md
index 0b015c1fa14f72e648c683400ea3aeeda58fd529..b646c939cfe87e4b11aca9d66cf4c572f3417e3f 100644 (file)
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -21,6 +21,10 @@ result will be similar regardless of whether a 4:2:2 JPEG image is rotated or
 transposed prior to decompression (in the frequency domain) or after
 decompression (in the spatial domain.)
 
+4. Fixed an integer overflow and subsequent segfault that occurred when
+attempting to compress or decompress images with more than 1 billion pixels
+using the TurboJPEG API.
+
 
 2.0.2
 =====

diff --git a/java/TJBench.java b/java/TJBench.java
index 7829e53cd419f3ae3b2f9578eb298d48ab9ea500..cc3178eaf8d48346e9b120165b984559c531bb18 100644 (file)
--- a/java/TJBench.java
+++ b/java/TJBench.java
@@ -121,6 +121,8 @@ final class TJBench {
     int rindex = TJ.getRedOffset(pixelFormat);
     int gindex = TJ.getGreenOffset(pixelFormat);
     int bindex = TJ.getBlueOffset(pixelFormat);
+    if ((long)w[0] * (long)h[0] * (long)ps > (long)Integer.MAX_VALUE)
+      throw new Exception("Image is too large");
     byte[] dstBuf = new byte[w[0] * h[0] * ps];
     int pixels = w[0] * h[0], dstPtr = 0, rgbPtr = 0;
 
@@ -175,8 +177,11 @@ final class TJBench {
 
     tjd = new TJDecompressor();
 
-    if (dstBuf == null)
+    if (dstBuf == null) {
+      if ((long)pitch * (long)scaledh > (long)Integer.MAX_VALUE)
+        throw new Exception("Image is too large");
       dstBuf = new byte[pitch * scaledh];
+    }
 
     /* Set the destination buffer to gray so we know whether the decompressor
        attempted to write to it */
@@ -331,6 +336,8 @@ final class TJBench {
     String pfStr = PIXFORMATSTR[pf];
     YUVImage yuvImage = null;
 
+    if ((long)pitch * (long)h > (long)Integer.MAX_VALUE)
+      throw new Exception("Image is too large");
     tmpBuf = new byte[pitch * h];
 
     if (quiet == 0)
@@ -491,6 +498,8 @@ final class TJBench {
     int tw, th, ttilew, ttileh, tntilesw, tntilesh, tsubsamp;
 
     FileInputStream fis = new FileInputStream(fileName);
+    if (fis.getChannel().size() > (long)Integer.MAX_VALUE)
+      throw new Exception("Image is too large");
     int srcSize = (int)fis.getChannel().size();
     srcBuf = new byte[srcSize];
     fis.read(srcBuf, 0, srcSize);

diff --git a/tjbench.c b/tjbench.c
index b94637bceff392ff61b54e9c18271bd78354737f..120fccbd9ad5a51c53c08b51cc91a68b1a0747a8 100644 (file)
--- a/tjbench.c
+++ b/tjbench.c
@@ -32,6 +32,7 @@
 #include <ctype.h>
 #include <math.h>
 #include <errno.h>
+#include <limits.h>
 #include <cdjpeg.h>
 #include "./tjutil.h"
 #include "./turbojpeg.h"
@@ -161,7 +162,10 @@ static int decomp(unsigned char *srcBuf, unsigned char **jpegBuf,
     THROW_TJ("executing tjInitDecompress()");
 
   if (dstBuf == NULL) {
-    if ((dstBuf = (unsigned char *)malloc(pitch * scaledh)) == NULL)
+    if ((unsigned long long)pitch * (unsigned long long)scaledh >
+        (unsigned long long)((size_t)-1))
+      THROW("allocating destination buffer", "Image is too large");
+    if ((dstBuf = (unsigned char *)malloc((size_t)pitch * scaledh)) == NULL)
       THROW_UNIX("allocating destination buffer");
     dstBufAlloc = 1;
   }
@@ -172,8 +176,10 @@ static int decomp(unsigned char *srcBuf, unsigned char **jpegBuf,
   if (doYUV) {
     int width = doTile ? tilew : scaledw;
     int height = doTile ? tileh : scaledh;
-    int yuvSize = tjBufSizeYUV2(width, yuvPad, height, subsamp);
+    unsigned long yuvSize = tjBufSizeYUV2(width, yuvPad, height, subsamp);
 
+    if (yuvSize == (unsigned long)-1)
+      THROW_TJ("allocating YUV buffer");
     if ((yuvBuf = (unsigned char *)malloc(yuvSize)) == NULL)
       THROW_UNIX("allocating YUV buffer");
     memset(yuvBuf, 127, yuvSize);
@@ -267,13 +273,13 @@ static int decomp(unsigned char *srcBuf, unsigned char **jpegBuf,
   if (srcBuf && sf.num == 1 && sf.denom == 1) {
     if (!quiet) printf("Compression error written to %s.\n", tempStr);
     if (subsamp == TJ_GRAYSCALE) {
-      int index, index2;
+      unsigned long index, index2;
 
       for (row = 0, index = 0; row < h; row++, index += pitch) {
         for (col = 0, index2 = index; col < w; col++, index2 += ps) {
-          int rindex = index2 + tjRedOffset[pf];
-          int gindex = index2 + tjGreenOffset[pf];
-          int bindex = index2 + tjBlueOffset[pf];
+          unsigned long rindex = index2 + tjRedOffset[pf];
+          unsigned long gindex = index2 + tjGreenOffset[pf];
+          unsigned long bindex = index2 + tjBlueOffset[pf];
           int y = (int)((double)srcBuf[rindex] * 0.299 +
                         (double)srcBuf[gindex] * 0.587 +
                         (double)srcBuf[bindex] * 0.114 + 0.5);
@@ -314,13 +320,16 @@ static int fullTest(unsigned char *srcBuf, int w, int h, int subsamp,
     *srcPtr2;
   double start, elapsed, elapsedEncode;
   int totalJpegSize = 0, row, col, i, tilew = w, tileh = h, retval = 0;
-  int iter, yuvSize = 0;
-  unsigned long *jpegSize = NULL;
+  int iter;
+  unsigned long *jpegSize = NULL, yuvSize = 0;
   int ps = tjPixelSize[pf];
   int ntilesw = 1, ntilesh = 1, pitch = w * ps;
   const char *pfStr = pixFormatStr[pf];
 
-  if ((tmpBuf = (unsigned char *)malloc(pitch * h)) == NULL)
+  if ((unsigned long long)pitch * (unsigned long long)h >
+      (unsigned long long)((size_t)-1))
+    THROW("allocating temporary image buffer", "Image is too large");
+  if ((tmpBuf = (unsigned char *)malloc((size_t)pitch * h)) == NULL)
     THROW_UNIX("allocating temporary image buffer");
 
   if (!quiet)
@@ -346,6 +355,8 @@ static int fullTest(unsigned char *srcBuf, int w, int h, int subsamp,
 
     if ((flags & TJFLAG_NOREALLOC) != 0)
       for (i = 0; i < ntilesw * ntilesh; i++) {
+        if (tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX)
+          THROW("getting buffer size", "Image is too large");
         if ((jpegBuf[i] = (unsigned char *)
                           tjAlloc(tjBufSize(tilew, tileh, subsamp))) == NULL)
           THROW_UNIX("allocating JPEG tiles");
@@ -363,6 +374,8 @@ static int fullTest(unsigned char *srcBuf, int w, int h, int subsamp,
 
     if (doYUV) {
       yuvSize = tjBufSizeYUV2(tilew, yuvPad, tileh, subsamp);
+      if (yuvSize == (unsigned long)-1)
+        THROW_TJ("allocating YUV buffer");
       if ((yuvBuf = (unsigned char *)malloc(yuvSize)) == NULL)
         THROW_UNIX("allocating YUV buffer");
       memset(yuvBuf, 127, yuvSize);
@@ -437,7 +450,7 @@ static int fullTest(unsigned char *srcBuf, int w, int h, int subsamp,
       if (doYUV) {
         printf("Encode YUV    --> Frame rate:         %f fps\n",
                (double)iter / elapsedEncode);
-        printf("                  Output image size:  %d bytes\n", yuvSize);
+        printf("                  Output image size:  %lu bytes\n", yuvSize);
         printf("                  Compression ratio:  %f:1\n",
                (double)(w * h * ps) / (double)yuvSize);
         printf("                  Throughput:         %f Megapixels/sec\n",
@@ -578,8 +591,11 @@ static int decompTest(char *fileName)
       THROW_UNIX("allocating JPEG size array");
     memset(jpegSize, 0, sizeof(unsigned long) * ntilesw * ntilesh);
 
-    if ((flags & TJFLAG_NOREALLOC) != 0 || !doTile)
+    if ((flags & TJFLAG_NOREALLOC) != 0 &&
+        (doTile || xformOp != TJXOP_NONE || xformOpt != 0 || customFilter))
       for (i = 0; i < ntilesw * ntilesh; i++) {
+        if (tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX)
+          THROW("getting buffer size", "Image is too large");
         if ((jpegBuf[i] = (unsigned char *)
                           tjAlloc(tjBufSize(tilew, tileh, subsamp))) == NULL)
           THROW_UNIX("allocating JPEG tiles");
@@ -685,7 +701,7 @@ static int decompTest(char *fileName)
       }
     } else {
       if (quiet == 1) printf("N/A     N/A     ");
-      tjFree(jpegBuf[0]);
+      if(jpegBuf[0]) tjFree(jpegBuf[0]);
       jpegBuf[0] = NULL;
       decompsrc = 1;
     }
@@ -700,7 +716,8 @@ static int decompTest(char *fileName)
     } else if (quiet == 1) printf("N/A\n");
 
     for (i = 0; i < ntilesw * ntilesh; i++) {
-      tjFree(jpegBuf[i]);  jpegBuf[i] = NULL;
+      if(jpegBuf[i]) tjFree(jpegBuf[i]);
+      jpegBuf[i] = NULL;
     }
     free(jpegBuf);  jpegBuf = NULL;
     if (jpegSize) { free(jpegSize);  jpegSize = NULL; }

diff --git a/tjunittest.c b/tjunittest.c
index a5baf5b0dd01fb9d2074a01623475d8e01855e60..da7c6ffabbfbf3616beafb0592c8114b9c0cfb54 100644 (file)
--- a/tjunittest.c
+++ b/tjunittest.c
@@ -554,6 +554,42 @@ bailout:
 }
 
 
+#if SIZEOF_SIZE_T == 8
+#define CHECKSIZE(function) { \
+  if ((unsigned long long)size < (unsigned long long)0xFFFFFFFF) \
+    THROW(#function " overflow"); \
+}
+#else
+#define CHECKSIZE(function) { \
+  if (size != (unsigned long)(-1) || \
+      !strcmp(tjGetErrorStr2(NULL), "No error")) \
+    THROW(#function " overflow"); \
+}
+#endif
+
+static void overflowTest(void)
+{
+  /* Ensure that the various buffer size functions don't overflow */
+  unsigned long size;
+
+  size = tjBufSize(26755, 26755, TJSAMP_444);
+  CHECKSIZE(tjBufSize());
+  size = TJBUFSIZE(26755, 26755);
+  CHECKSIZE(TJBUFSIZE());
+  size = tjBufSizeYUV2(37838, 1, 37838, TJSAMP_444);
+  CHECKSIZE(tjBufSizeYUV2());
+  size = TJBUFSIZEYUV(37838, 37838, TJSAMP_444);
+  CHECKSIZE(TJBUFSIZEYUV());
+  size = tjBufSizeYUV(37838, 37838, TJSAMP_444);
+  CHECKSIZE(tjBufSizeYUV());
+  size = tjPlaneSizeYUV(0, 65536, 0, 65536, TJSAMP_444);
+  CHECKSIZE(tjPlaneSizeYUV());
+
+bailout:
+  return;
+}
+
+
 static void bufSizeTest(void)
 {
   int w, h, i, subsamp;
@@ -865,6 +901,7 @@ int main(int argc, char *argv[])
   }
   if (alloc) printf("Testing automatic buffer allocation\n");
   if (doYUV) num4bf = 4;
+  overflowTest();
   doTest(35, 39, _3byteFormats, 2, TJSAMP_444, "test");
   doTest(39, 41, _4byteFormats, num4bf, TJSAMP_444, "test");
   doTest(41, 35, _3byteFormats, 2, TJSAMP_422, "test");

diff --git a/turbojpeg.c b/turbojpeg.c
index d4919722c75af3f90345e7e3a4442e5e9fc3b400..b6a1f204845ca0e628b6f5f94aa5b32cc76f95fe 100644 (file)
--- a/turbojpeg.c
+++ b/turbojpeg.c
@@ -491,7 +491,7 @@ DLLEXPORT tjhandle tjInitCompress(void)
 
 DLLEXPORT unsigned long tjBufSize(int width, int height, int jpegSubsamp)
 {
-  unsigned long retval = 0;
+  unsigned long long retval = 0;
   int mcuw, mcuh, chromasf;
 
   if (width < 1 || height < 1 || jpegSubsamp < 0 || jpegSubsamp >= NUMSUBOPT)
@@ -503,15 +503,17 @@ DLLEXPORT unsigned long tjBufSize(int width, int height, int jpegSubsamp)
   mcuw = tjMCUWidth[jpegSubsamp];
   mcuh = tjMCUHeight[jpegSubsamp];
   chromasf = jpegSubsamp == TJSAMP_GRAY ? 0 : 4 * 64 / (mcuw * mcuh);
-  retval = PAD(width, mcuw) * PAD(height, mcuh) * (2 + chromasf) + 2048;
+  retval = PAD(width, mcuw) * PAD(height, mcuh) * (2ULL + chromasf) + 2048ULL;
+  if (retval > (unsigned long long)((unsigned long)-1))
+    THROWG("tjBufSize(): Image is too large");
 
 bailout:
-  return retval;
+  return (unsigned long)retval;
 }
 
 DLLEXPORT unsigned long TJBUFSIZE(int width, int height)
 {
-  unsigned long retval = 0;
+  unsigned long long retval = 0;
 
   if (width < 1 || height < 1)
     THROWG("TJBUFSIZE(): Invalid argument");
@@ -519,17 +521,20 @@ DLLEXPORT unsigned long TJBUFSIZE(int width, int height)
   /* This allows for rare corner cases in which a JPEG image can actually be
      larger than the uncompressed input (we wouldn't mention it if it hadn't
      happened before.) */
-  retval = PAD(width, 16) * PAD(height, 16) * 6 + 2048;
+  retval = PAD(width, 16) * PAD(height, 16) * 6ULL + 2048ULL;
+  if (retval > (unsigned long long)((unsigned long)-1))
+    THROWG("TJBUFSIZE(): Image is too large");
 
 bailout:
-  return retval;
+  return (unsigned long)retval;
 }
 
 
 DLLEXPORT unsigned long tjBufSizeYUV2(int width, int pad, int height,
                                       int subsamp)
 {
-  int retval = 0, nc, i;
+  unsigned long long retval = 0;
+  int nc, i;
 
   if (subsamp < 0 || subsamp >= NUMSUBOPT)
     THROWG("tjBufSizeYUV2(): Invalid argument");
@@ -541,11 +546,13 @@ DLLEXPORT unsigned long tjBufSizeYUV2(int width, int pad, int height,
     int ph = tjPlaneHeight(i, height, subsamp);
 
     if (pw < 0 || ph < 0) return -1;
-    else retval += stride * ph;
+    else retval += (unsigned long long)stride * ph;
   }
+  if (retval > (unsigned long long)((unsigned long)-1))
+    THROWG("tjBufSizeYUV2(): Image is too large");
 
 bailout:
-  return retval;
+  return (unsigned long)retval;
 }
 
 DLLEXPORT unsigned long tjBufSizeYUV(int width, int height, int subsamp)
@@ -604,7 +611,7 @@ bailout:
 DLLEXPORT unsigned long tjPlaneSizeYUV(int componentID, int width, int stride,
                                        int height, int subsamp)
 {
-  unsigned long retval = 0;
+  unsigned long long retval = 0;
   int pw, ph;
 
   if (width < 1 || height < 1 || subsamp < 0 || subsamp >= NUMSUBOPT)
@@ -617,10 +624,12 @@ DLLEXPORT unsigned long tjPlaneSizeYUV(int componentID, int width, int stride,
   if (stride == 0) stride = pw;
   else stride = abs(stride);
 
-  retval = stride * (ph - 1) + pw;
+  retval = (unsigned long long)stride * (ph - 1) + pw;
+  if (retval > (unsigned long long)((unsigned long)-1))
+    THROWG("tjPlaneSizeYUV(): Image is too large");
 
 bailout:
-  return retval;
+  return (unsigned long)retval;
 }
 
 
@@ -672,9 +681,9 @@ DLLEXPORT int tjCompress2(tjhandle handle, const unsigned char *srcBuf,
   jpeg_start_compress(cinfo, TRUE);
   for (i = 0; i < height; i++) {
     if (flags & TJFLAG_BOTTOMUP)
-      row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * pitch];
+      row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * (size_t)pitch];
     else
-      row_pointer[i] = (JSAMPROW)&srcBuf[i * pitch];
+      row_pointer[i] = (JSAMPROW)&srcBuf[i * (size_t)pitch];
   }
   while (cinfo->next_scanline < cinfo->image_height)
     jpeg_write_scanlines(cinfo, &row_pointer[cinfo->next_scanline],
@@ -783,9 +792,9 @@ DLLEXPORT int tjEncodeYUVPlanes(tjhandle handle, const unsigned char *srcBuf,
     THROW("tjEncodeYUVPlanes(): Memory allocation failure");
   for (i = 0; i < height; i++) {
     if (flags & TJFLAG_BOTTOMUP)
-      row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * pitch];
+      row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * (size_t)pitch];
     else
-      row_pointer[i] = (JSAMPROW)&srcBuf[i * pitch];
+      row_pointer[i] = (JSAMPROW)&srcBuf[i * (size_t)pitch];
   }
   if (height < ph0)
     for (i = height; i < ph0; i++) row_pointer[i] = row_pointer[height - 1];
@@ -1293,9 +1302,9 @@ DLLEXPORT int tjDecompress2(tjhandle handle, const unsigned char *jpegBuf,
   }
   for (i = 0; i < (int)dinfo->output_height; i++) {
     if (flags & TJFLAG_BOTTOMUP)
-      row_pointer[i] = &dstBuf[(dinfo->output_height - i - 1) * pitch];
+      row_pointer[i] = &dstBuf[(dinfo->output_height - i - 1) * (size_t)pitch];
     else
-      row_pointer[i] = &dstBuf[i * pitch];
+      row_pointer[i] = &dstBuf[i * (size_t)pitch];
   }
   while (dinfo->output_scanline < dinfo->output_height)
     jpeg_read_scanlines(dinfo, &row_pointer[dinfo->output_scanline],
@@ -1450,9 +1459,9 @@ DLLEXPORT int tjDecodeYUVPlanes(tjhandle handle,
     THROW("tjDecodeYUVPlanes(): Memory allocation failure");
   for (i = 0; i < height; i++) {
     if (flags & TJFLAG_BOTTOMUP)
-      row_pointer[i] = &dstBuf[(height - i - 1) * pitch];
+      row_pointer[i] = &dstBuf[(height - i - 1) * (size_t)pitch];
     else
-      row_pointer[i] = &dstBuf[i * pitch];
+      row_pointer[i] = &dstBuf[i * (size_t)pitch];
   }
   if (height < ph0)
     for (i = height; i < ph0; i++) row_pointer[i] = row_pointer[height - 1];