From: DRC Date: Thu, 11 Jul 2019 20:30:04 +0000 (-0500) Subject: TurboJPEG: Properly handle gigapixel images X-Git-Tag: 2.0.3~11 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=2a9e3bd7430cfda1bc812d139e0609c6aca0b884;p=libjpeg-turbo TurboJPEG: Properly handle gigapixel images Prevent several integer overflow issues and subsequent segfaults that occurred when attempting to compress or decompress gigapixel images with the TurboJPEG API: - Modify tjBufSize(), tjBufSizeYUV2(), and tjPlaneSizeYUV() to avoid integer overflow when computing the return values and to return an error if such an overflow is unavoidable. - Modify tjunittest to validate the above. - Modify tjCompress2(), tjEncodeYUVPlanes(), tjDecompress2(), and tjDecodeYUVPlanes() to avoid integer overflow when computing the row pointers in the 64-bit TurboJPEG C API. - Modify TJBench (both C and Java versions) to avoid overflowing the size argument to malloc()/new and to fail gracefully if such an overflow is unavoidable. In general, this allows gigapixel images to be accommodated by the 64-bit TurboJPEG C API when using automatic JPEG buffer (re)allocation. Such images cannot currently be accommodated without automatic JPEG buffer (re)allocation, due to the fact that tjAlloc() accepts a 32-bit integer argument (oops.) Such images cannot be accommodated in the TurboJPEG Java API due to the fact that Java always uses a signed 32-bit integer as an array index. Fixes #361 --- diff --git a/ChangeLog.md b/ChangeLog.md index 0b015c1..b646c93 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -21,6 +21,10 @@ result will be similar regardless of whether a 4:2:2 JPEG image is rotated or transposed prior to decompression (in the frequency domain) or after decompression (in the spatial domain.) +4. Fixed an integer overflow and subsequent segfault that occurred when +attempting to compress or decompress images with more than 1 billion pixels +using the TurboJPEG API. + 2.0.2 ===== diff --git a/java/TJBench.java b/java/TJBench.java index 7829e53..cc3178e 100644 --- a/java/TJBench.java +++ b/java/TJBench.java @@ -121,6 +121,8 @@ final class TJBench { int rindex = TJ.getRedOffset(pixelFormat); int gindex = TJ.getGreenOffset(pixelFormat); int bindex = TJ.getBlueOffset(pixelFormat); + if ((long)w[0] * (long)h[0] * (long)ps > (long)Integer.MAX_VALUE) + throw new Exception("Image is too large"); byte[] dstBuf = new byte[w[0] * h[0] * ps]; int pixels = w[0] * h[0], dstPtr = 0, rgbPtr = 0; @@ -175,8 +177,11 @@ final class TJBench { tjd = new TJDecompressor(); - if (dstBuf == null) + if (dstBuf == null) { + if ((long)pitch * (long)scaledh > (long)Integer.MAX_VALUE) + throw new Exception("Image is too large"); dstBuf = new byte[pitch * scaledh]; + } /* Set the destination buffer to gray so we know whether the decompressor attempted to write to it */ @@ -331,6 +336,8 @@ final class TJBench { String pfStr = PIXFORMATSTR[pf]; YUVImage yuvImage = null; + if ((long)pitch * (long)h > (long)Integer.MAX_VALUE) + throw new Exception("Image is too large"); tmpBuf = new byte[pitch * h]; if (quiet == 0) @@ -491,6 +498,8 @@ final class TJBench { int tw, th, ttilew, ttileh, tntilesw, tntilesh, tsubsamp; FileInputStream fis = new FileInputStream(fileName); + if (fis.getChannel().size() > (long)Integer.MAX_VALUE) + throw new Exception("Image is too large"); int srcSize = (int)fis.getChannel().size(); srcBuf = new byte[srcSize]; fis.read(srcBuf, 0, srcSize); diff --git a/tjbench.c b/tjbench.c index b94637b..120fccb 100644 --- a/tjbench.c +++ b/tjbench.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include "./tjutil.h" #include "./turbojpeg.h" @@ -161,7 +162,10 @@ static int decomp(unsigned char *srcBuf, unsigned char **jpegBuf, THROW_TJ("executing tjInitDecompress()"); if (dstBuf == NULL) { - if ((dstBuf = (unsigned char *)malloc(pitch * scaledh)) == NULL) + if ((unsigned long long)pitch * (unsigned long long)scaledh > + (unsigned long long)((size_t)-1)) + THROW("allocating destination buffer", "Image is too large"); + if ((dstBuf = (unsigned char *)malloc((size_t)pitch * scaledh)) == NULL) THROW_UNIX("allocating destination buffer"); dstBufAlloc = 1; } @@ -172,8 +176,10 @@ static int decomp(unsigned char *srcBuf, unsigned char **jpegBuf, if (doYUV) { int width = doTile ? tilew : scaledw; int height = doTile ? tileh : scaledh; - int yuvSize = tjBufSizeYUV2(width, yuvPad, height, subsamp); + unsigned long yuvSize = tjBufSizeYUV2(width, yuvPad, height, subsamp); + if (yuvSize == (unsigned long)-1) + THROW_TJ("allocating YUV buffer"); if ((yuvBuf = (unsigned char *)malloc(yuvSize)) == NULL) THROW_UNIX("allocating YUV buffer"); memset(yuvBuf, 127, yuvSize); @@ -267,13 +273,13 @@ static int decomp(unsigned char *srcBuf, unsigned char **jpegBuf, if (srcBuf && sf.num == 1 && sf.denom == 1) { if (!quiet) printf("Compression error written to %s.\n", tempStr); if (subsamp == TJ_GRAYSCALE) { - int index, index2; + unsigned long index, index2; for (row = 0, index = 0; row < h; row++, index += pitch) { for (col = 0, index2 = index; col < w; col++, index2 += ps) { - int rindex = index2 + tjRedOffset[pf]; - int gindex = index2 + tjGreenOffset[pf]; - int bindex = index2 + tjBlueOffset[pf]; + unsigned long rindex = index2 + tjRedOffset[pf]; + unsigned long gindex = index2 + tjGreenOffset[pf]; + unsigned long bindex = index2 + tjBlueOffset[pf]; int y = (int)((double)srcBuf[rindex] * 0.299 + (double)srcBuf[gindex] * 0.587 + (double)srcBuf[bindex] * 0.114 + 0.5); @@ -314,13 +320,16 @@ static int fullTest(unsigned char *srcBuf, int w, int h, int subsamp, *srcPtr2; double start, elapsed, elapsedEncode; int totalJpegSize = 0, row, col, i, tilew = w, tileh = h, retval = 0; - int iter, yuvSize = 0; - unsigned long *jpegSize = NULL; + int iter; + unsigned long *jpegSize = NULL, yuvSize = 0; int ps = tjPixelSize[pf]; int ntilesw = 1, ntilesh = 1, pitch = w * ps; const char *pfStr = pixFormatStr[pf]; - if ((tmpBuf = (unsigned char *)malloc(pitch * h)) == NULL) + if ((unsigned long long)pitch * (unsigned long long)h > + (unsigned long long)((size_t)-1)) + THROW("allocating temporary image buffer", "Image is too large"); + if ((tmpBuf = (unsigned char *)malloc((size_t)pitch * h)) == NULL) THROW_UNIX("allocating temporary image buffer"); if (!quiet) @@ -346,6 +355,8 @@ static int fullTest(unsigned char *srcBuf, int w, int h, int subsamp, if ((flags & TJFLAG_NOREALLOC) != 0) for (i = 0; i < ntilesw * ntilesh; i++) { + if (tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX) + THROW("getting buffer size", "Image is too large"); if ((jpegBuf[i] = (unsigned char *) tjAlloc(tjBufSize(tilew, tileh, subsamp))) == NULL) THROW_UNIX("allocating JPEG tiles"); @@ -363,6 +374,8 @@ static int fullTest(unsigned char *srcBuf, int w, int h, int subsamp, if (doYUV) { yuvSize = tjBufSizeYUV2(tilew, yuvPad, tileh, subsamp); + if (yuvSize == (unsigned long)-1) + THROW_TJ("allocating YUV buffer"); if ((yuvBuf = (unsigned char *)malloc(yuvSize)) == NULL) THROW_UNIX("allocating YUV buffer"); memset(yuvBuf, 127, yuvSize); @@ -437,7 +450,7 @@ static int fullTest(unsigned char *srcBuf, int w, int h, int subsamp, if (doYUV) { printf("Encode YUV --> Frame rate: %f fps\n", (double)iter / elapsedEncode); - printf(" Output image size: %d bytes\n", yuvSize); + printf(" Output image size: %lu bytes\n", yuvSize); printf(" Compression ratio: %f:1\n", (double)(w * h * ps) / (double)yuvSize); printf(" Throughput: %f Megapixels/sec\n", @@ -578,8 +591,11 @@ static int decompTest(char *fileName) THROW_UNIX("allocating JPEG size array"); memset(jpegSize, 0, sizeof(unsigned long) * ntilesw * ntilesh); - if ((flags & TJFLAG_NOREALLOC) != 0 || !doTile) + if ((flags & TJFLAG_NOREALLOC) != 0 && + (doTile || xformOp != TJXOP_NONE || xformOpt != 0 || customFilter)) for (i = 0; i < ntilesw * ntilesh; i++) { + if (tjBufSize(tilew, tileh, subsamp) > (unsigned long)INT_MAX) + THROW("getting buffer size", "Image is too large"); if ((jpegBuf[i] = (unsigned char *) tjAlloc(tjBufSize(tilew, tileh, subsamp))) == NULL) THROW_UNIX("allocating JPEG tiles"); @@ -685,7 +701,7 @@ static int decompTest(char *fileName) } } else { if (quiet == 1) printf("N/A N/A "); - tjFree(jpegBuf[0]); + if(jpegBuf[0]) tjFree(jpegBuf[0]); jpegBuf[0] = NULL; decompsrc = 1; } @@ -700,7 +716,8 @@ static int decompTest(char *fileName) } else if (quiet == 1) printf("N/A\n"); for (i = 0; i < ntilesw * ntilesh; i++) { - tjFree(jpegBuf[i]); jpegBuf[i] = NULL; + if(jpegBuf[i]) tjFree(jpegBuf[i]); + jpegBuf[i] = NULL; } free(jpegBuf); jpegBuf = NULL; if (jpegSize) { free(jpegSize); jpegSize = NULL; } diff --git a/tjunittest.c b/tjunittest.c index a5baf5b..da7c6ff 100644 --- a/tjunittest.c +++ b/tjunittest.c @@ -554,6 +554,42 @@ bailout: } +#if SIZEOF_SIZE_T == 8 +#define CHECKSIZE(function) { \ + if ((unsigned long long)size < (unsigned long long)0xFFFFFFFF) \ + THROW(#function " overflow"); \ +} +#else +#define CHECKSIZE(function) { \ + if (size != (unsigned long)(-1) || \ + !strcmp(tjGetErrorStr2(NULL), "No error")) \ + THROW(#function " overflow"); \ +} +#endif + +static void overflowTest(void) +{ + /* Ensure that the various buffer size functions don't overflow */ + unsigned long size; + + size = tjBufSize(26755, 26755, TJSAMP_444); + CHECKSIZE(tjBufSize()); + size = TJBUFSIZE(26755, 26755); + CHECKSIZE(TJBUFSIZE()); + size = tjBufSizeYUV2(37838, 1, 37838, TJSAMP_444); + CHECKSIZE(tjBufSizeYUV2()); + size = TJBUFSIZEYUV(37838, 37838, TJSAMP_444); + CHECKSIZE(TJBUFSIZEYUV()); + size = tjBufSizeYUV(37838, 37838, TJSAMP_444); + CHECKSIZE(tjBufSizeYUV()); + size = tjPlaneSizeYUV(0, 65536, 0, 65536, TJSAMP_444); + CHECKSIZE(tjPlaneSizeYUV()); + +bailout: + return; +} + + static void bufSizeTest(void) { int w, h, i, subsamp; @@ -865,6 +901,7 @@ int main(int argc, char *argv[]) } if (alloc) printf("Testing automatic buffer allocation\n"); if (doYUV) num4bf = 4; + overflowTest(); doTest(35, 39, _3byteFormats, 2, TJSAMP_444, "test"); doTest(39, 41, _4byteFormats, num4bf, TJSAMP_444, "test"); doTest(41, 35, _3byteFormats, 2, TJSAMP_422, "test"); diff --git a/turbojpeg.c b/turbojpeg.c index d491972..b6a1f20 100644 --- a/turbojpeg.c +++ b/turbojpeg.c @@ -491,7 +491,7 @@ DLLEXPORT tjhandle tjInitCompress(void) DLLEXPORT unsigned long tjBufSize(int width, int height, int jpegSubsamp) { - unsigned long retval = 0; + unsigned long long retval = 0; int mcuw, mcuh, chromasf; if (width < 1 || height < 1 || jpegSubsamp < 0 || jpegSubsamp >= NUMSUBOPT) @@ -503,15 +503,17 @@ DLLEXPORT unsigned long tjBufSize(int width, int height, int jpegSubsamp) mcuw = tjMCUWidth[jpegSubsamp]; mcuh = tjMCUHeight[jpegSubsamp]; chromasf = jpegSubsamp == TJSAMP_GRAY ? 0 : 4 * 64 / (mcuw * mcuh); - retval = PAD(width, mcuw) * PAD(height, mcuh) * (2 + chromasf) + 2048; + retval = PAD(width, mcuw) * PAD(height, mcuh) * (2ULL + chromasf) + 2048ULL; + if (retval > (unsigned long long)((unsigned long)-1)) + THROWG("tjBufSize(): Image is too large"); bailout: - return retval; + return (unsigned long)retval; } DLLEXPORT unsigned long TJBUFSIZE(int width, int height) { - unsigned long retval = 0; + unsigned long long retval = 0; if (width < 1 || height < 1) THROWG("TJBUFSIZE(): Invalid argument"); @@ -519,17 +521,20 @@ DLLEXPORT unsigned long TJBUFSIZE(int width, int height) /* This allows for rare corner cases in which a JPEG image can actually be larger than the uncompressed input (we wouldn't mention it if it hadn't happened before.) */ - retval = PAD(width, 16) * PAD(height, 16) * 6 + 2048; + retval = PAD(width, 16) * PAD(height, 16) * 6ULL + 2048ULL; + if (retval > (unsigned long long)((unsigned long)-1)) + THROWG("TJBUFSIZE(): Image is too large"); bailout: - return retval; + return (unsigned long)retval; } DLLEXPORT unsigned long tjBufSizeYUV2(int width, int pad, int height, int subsamp) { - int retval = 0, nc, i; + unsigned long long retval = 0; + int nc, i; if (subsamp < 0 || subsamp >= NUMSUBOPT) THROWG("tjBufSizeYUV2(): Invalid argument"); @@ -541,11 +546,13 @@ DLLEXPORT unsigned long tjBufSizeYUV2(int width, int pad, int height, int ph = tjPlaneHeight(i, height, subsamp); if (pw < 0 || ph < 0) return -1; - else retval += stride * ph; + else retval += (unsigned long long)stride * ph; } + if (retval > (unsigned long long)((unsigned long)-1)) + THROWG("tjBufSizeYUV2(): Image is too large"); bailout: - return retval; + return (unsigned long)retval; } DLLEXPORT unsigned long tjBufSizeYUV(int width, int height, int subsamp) @@ -604,7 +611,7 @@ bailout: DLLEXPORT unsigned long tjPlaneSizeYUV(int componentID, int width, int stride, int height, int subsamp) { - unsigned long retval = 0; + unsigned long long retval = 0; int pw, ph; if (width < 1 || height < 1 || subsamp < 0 || subsamp >= NUMSUBOPT) @@ -617,10 +624,12 @@ DLLEXPORT unsigned long tjPlaneSizeYUV(int componentID, int width, int stride, if (stride == 0) stride = pw; else stride = abs(stride); - retval = stride * (ph - 1) + pw; + retval = (unsigned long long)stride * (ph - 1) + pw; + if (retval > (unsigned long long)((unsigned long)-1)) + THROWG("tjPlaneSizeYUV(): Image is too large"); bailout: - return retval; + return (unsigned long)retval; } @@ -672,9 +681,9 @@ DLLEXPORT int tjCompress2(tjhandle handle, const unsigned char *srcBuf, jpeg_start_compress(cinfo, TRUE); for (i = 0; i < height; i++) { if (flags & TJFLAG_BOTTOMUP) - row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * pitch]; + row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * (size_t)pitch]; else - row_pointer[i] = (JSAMPROW)&srcBuf[i * pitch]; + row_pointer[i] = (JSAMPROW)&srcBuf[i * (size_t)pitch]; } while (cinfo->next_scanline < cinfo->image_height) jpeg_write_scanlines(cinfo, &row_pointer[cinfo->next_scanline], @@ -783,9 +792,9 @@ DLLEXPORT int tjEncodeYUVPlanes(tjhandle handle, const unsigned char *srcBuf, THROW("tjEncodeYUVPlanes(): Memory allocation failure"); for (i = 0; i < height; i++) { if (flags & TJFLAG_BOTTOMUP) - row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * pitch]; + row_pointer[i] = (JSAMPROW)&srcBuf[(height - i - 1) * (size_t)pitch]; else - row_pointer[i] = (JSAMPROW)&srcBuf[i * pitch]; + row_pointer[i] = (JSAMPROW)&srcBuf[i * (size_t)pitch]; } if (height < ph0) for (i = height; i < ph0; i++) row_pointer[i] = row_pointer[height - 1]; @@ -1293,9 +1302,9 @@ DLLEXPORT int tjDecompress2(tjhandle handle, const unsigned char *jpegBuf, } for (i = 0; i < (int)dinfo->output_height; i++) { if (flags & TJFLAG_BOTTOMUP) - row_pointer[i] = &dstBuf[(dinfo->output_height - i - 1) * pitch]; + row_pointer[i] = &dstBuf[(dinfo->output_height - i - 1) * (size_t)pitch]; else - row_pointer[i] = &dstBuf[i * pitch]; + row_pointer[i] = &dstBuf[i * (size_t)pitch]; } while (dinfo->output_scanline < dinfo->output_height) jpeg_read_scanlines(dinfo, &row_pointer[dinfo->output_scanline], @@ -1450,9 +1459,9 @@ DLLEXPORT int tjDecodeYUVPlanes(tjhandle handle, THROW("tjDecodeYUVPlanes(): Memory allocation failure"); for (i = 0; i < height; i++) { if (flags & TJFLAG_BOTTOMUP) - row_pointer[i] = &dstBuf[(height - i - 1) * pitch]; + row_pointer[i] = &dstBuf[(height - i - 1) * (size_t)pitch]; else - row_pointer[i] = &dstBuf[i * pitch]; + row_pointer[i] = &dstBuf[i * (size_t)pitch]; } if (height < ph0) for (i = height; i < ph0; i++) row_pointer[i] = row_pointer[height - 1];