TurboJPEG: Fix potential memory leaks

Referring to https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=746,
it seems that the values of local buffer pointers in TurboJPEG API
functions aren't always preserved if longjmp() returns control to a
point prior to the allocation of the local buffers.  This is known to
be an issue with GCC 4.x and clang with -O1 and higher optimization
levels but not with GCC 5.x and later.  It is unknown why GCC 5.x and
6.x do not suffer from the issue, but possibly the local buffer pointers
are not allocated on the stack when using those more recent compilers.

In any case, this commit modifies the TurboJPEG API library code such
that the jump buffer is always updated after any local buffer pointers
are allocated but before any subsequent libjpeg API functions are
called.

diff --git a/ChangeLog.md b/ChangeLog.md
index 47ebb70b68ea8b88fd723c73996a2e66e18f588f..620c2f8e465326d6155c17683bbf566976697d4c 100644 (file)
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -38,6 +38,12 @@ API/ABI emulation, since the behavior is entirely internal.  Note that
 `-copy all` must be passed to jpegtran in order to transfer the EXIF tags from
 the source image to the destination image.
 
+8. Fixed several memory leaks in the TurboJPEG API library that could occur
+if the library was built with certain compilers and optimization levels
+(known to occur with GCC 4.x and clang with `-O1` and higher but not with
+GCC 5.x or 6.x) and one of the underlying libjpeg API functions threw an error
+after a TurboJPEG API function allocated a local buffer.
+
 
 1.5.1
 =====

diff --git a/turbojpeg.c b/turbojpeg.c
index 6533b4117f8087d21d509eacac9e714613b5489a..c9bc6197b86b64ef79b5b401d79a2ce9d41692ca 100644 (file)
--- a/turbojpeg.c
+++ b/turbojpeg.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C)2009-2016 D. R. Commander.  All Rights Reserved.
+ * Copyright (C)2009-2017 D. R. Commander.  All Rights Reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -772,13 +772,6 @@ DLLEXPORT int DLLCALL tjCompress2(tjhandle handle, const unsigned char *srcBuf,
                || jpegSubsamp<0 || jpegSubsamp>=NUMSUBOPT || jpegQual<0 || jpegQual>100)
                _throw("tjCompress2(): Invalid argument");
 
-       if(setjmp(this->jerr.setjmp_buffer))
-       {
-               /* If we get here, the JPEG code has signaled an error. */
-               retval=-1;
-               goto bailout;
-       }
-
        if(pitch==0) pitch=width*tjPixelSize[pixelFormat];
 
        #ifndef JCS_EXTENSIONS
@@ -791,6 +784,16 @@ DLLEXPORT int DLLCALL tjCompress2(tjhandle handle, const unsigned char *srcBuf,
        }
        #endif
 
+       if((row_pointer=(JSAMPROW *)malloc(sizeof(JSAMPROW)*height))==NULL)
+               _throw("tjCompress2(): Memory allocation failure");
+
+       if(setjmp(this->jerr.setjmp_buffer))
+       {
+               /* If we get here, the JPEG code has signaled an error. */
+               retval=-1;
+               goto bailout;
+       }
+
        cinfo->image_width=width;
        cinfo->image_height=height;
 
@@ -807,8 +810,6 @@ DLLEXPORT int DLLCALL tjCompress2(tjhandle handle, const unsigned char *srcBuf,
                return -1;
 
        jpeg_start_compress(cinfo, TRUE);
-       if((row_pointer=(JSAMPROW *)malloc(sizeof(JSAMPROW)*height))==NULL)
-               _throw("tjCompress2(): Memory allocation failure");
        for(i=0; i<height; i++)
        {
                if(flags&TJFLAG_BOTTOMUP)
@@ -888,13 +889,6 @@ DLLEXPORT int DLLCALL tjEncodeYUVPlanes(tjhandle handle,
        if(subsamp!=TJSAMP_GRAY && (!dstPlanes[1] || !dstPlanes[2]))
                _throw("tjEncodeYUVPlanes(): Invalid argument");
 
-       if(setjmp(this->jerr.setjmp_buffer))
-       {
-               /* If we get here, the JPEG code has signaled an error. */
-               retval=-1;
-               goto bailout;
-       }
-
        if(pixelFormat==TJPF_CMYK)
                _throw("tjEncodeYUVPlanes(): Cannot generate YUV images from CMYK pixels");
 
@@ -910,6 +904,13 @@ DLLEXPORT int DLLCALL tjEncodeYUVPlanes(tjhandle handle,
        }
        #endif
 
+       if(setjmp(this->jerr.setjmp_buffer))
+       {
+               /* If we get here, the JPEG code has signaled an error. */
+               retval=-1;
+               goto bailout;
+       }
+
        cinfo->image_width=width;
        cinfo->image_height=height;
 
@@ -986,6 +987,13 @@ DLLEXPORT int DLLCALL tjEncodeYUVPlanes(tjhandle handle,
                }
        }
 
+       if(setjmp(this->jerr.setjmp_buffer))
+       {
+               /* If we get here, the JPEG code has signaled an error. */
+               retval=-1;
+               goto bailout;
+       }
+
        for(row=0; row<ph0; row+=cinfo->max_v_samp_factor)
        {
                (*cinfo->cconvert->color_convert)(cinfo, &row_pointer[row], tmpbuf, 0,
@@ -1160,6 +1168,13 @@ DLLEXPORT int DLLCALL tjCompressFromYUVPlanes(tjhandle handle,
                }
        }
 
+       if(setjmp(this->jerr.setjmp_buffer))
+       {
+               /* If we get here, the JPEG code has signaled an error. */
+               retval=-1;
+               goto bailout;
+       }
+
        for(row=0; row<(int)cinfo->image_height;
                row+=cinfo->max_v_samp_factor*DCTSIZE)
        {
@@ -1438,6 +1453,12 @@ DLLEXPORT int DLLCALL tjDecompress2(tjhandle handle,
        if((row_pointer=(JSAMPROW *)malloc(sizeof(JSAMPROW)
                *dinfo->output_height))==NULL)
                _throw("tjDecompress2(): Memory allocation failure");
+       if(setjmp(this->jerr.setjmp_buffer))
+       {
+               /* If we get here, the JPEG code has signaled an error. */
+               retval=-1;
+               goto bailout;
+       }
        for(i=0; i<(int)dinfo->output_height; i++)
        {
                if(flags&TJFLAG_BOTTOMUP)
@@ -1660,6 +1681,13 @@ DLLEXPORT int DLLCALL tjDecodeYUVPlanes(tjhandle handle,
                }
        }
 
+       if(setjmp(this->jerr.setjmp_buffer))
+       {
+               /* If we get here, the JPEG code has signaled an error. */
+               retval=-1;
+               goto bailout;
+       }
+
        for(row=0; row<ph0; row+=dinfo->max_v_samp_factor)
        {
                JDIMENSION inrow=0, outrow=0;
@@ -1840,6 +1868,13 @@ DLLEXPORT int DLLCALL tjDecompressToYUVPlanes(tjhandle handle,
                }
        }
 
+       if(setjmp(this->jerr.setjmp_buffer))
+       {
+               /* If we get here, the JPEG code has signaled an error. */
+               retval=-1;
+               goto bailout;
+       }
+
        if(flags&TJFLAG_FASTUPSAMPLE) dinfo->do_fancy_upsampling=FALSE;
        if(flags&TJFLAG_FASTDCT) dinfo->dct_method=JDCT_FASTEST;
        dinfo->raw_data_out=TRUE;
@@ -2017,6 +2052,11 @@ DLLEXPORT int DLLCALL tjTransform(tjhandle handle,
        else if(flags&TJFLAG_FORCESSE) putenv("JSIMD_FORCESSE=1");
        else if(flags&TJFLAG_FORCESSE2) putenv("JSIMD_FORCESSE2=1");
 
+       if((xinfo=(jpeg_transform_info *)malloc(sizeof(jpeg_transform_info)*n))
+               ==NULL)
+               _throw("tjTransform(): Memory allocation failure");
+       MEMZERO(xinfo, sizeof(jpeg_transform_info)*n);
+
        if(setjmp(this->jerr.setjmp_buffer))
        {
                /* If we get here, the JPEG code has signaled an error. */
@@ -2026,11 +2066,6 @@ DLLEXPORT int DLLCALL tjTransform(tjhandle handle,
 
        jpeg_mem_src_tj(dinfo, jpegBuf, jpegSize);
 
-       if((xinfo=(jpeg_transform_info *)malloc(sizeof(jpeg_transform_info)*n))
-               ==NULL)
-               _throw("tjTransform(): Memory allocation failure");
-       MEMZERO(xinfo, sizeof(jpeg_transform_info)*n);
-
        for(i=0; i<n; i++)
        {
                xinfo[i].transform=xformtypes[t[i].op];