Fix incorrect data output and buffer overruns in the new tjDecompressToYUV2() functio...

git-svn-id: svn+ssh://svn.code.sf.net/p/libjpeg-turbo/code/trunk@982 632fc199-4ca6-4c93-a231-07263d6284db

diff --git a/java/TJUnitTest.java b/java/TJUnitTest.java
index 2cce93bb68f17a251af2ff7b7562584942116f60..c894b13cd70c4ad1cc2e34fa7e64c480972e1e27 100644 (file)
--- a/java/TJUnitTest.java
+++ b/java/TJUnitTest.java
@@ -757,14 +757,15 @@ public class TJUnitTest {
                                  String baseName, int subsamp,
                                  int flags) throws Exception {
     int i;
-    if (subsamp == TJ.SAMP_444 || subsamp == TJ.SAMP_GRAY) {
-      TJScalingFactor[] sf = TJ.getScalingFactors();
-      for (i = 0; i < sf.length; i++)
+    TJScalingFactor[] sf = TJ.getScalingFactors();
+    for (i = 0; i < sf.length; i++) {
+      int num = sf[i].getNum();
+      int denom = sf[i].getDenom();
+      if (subsamp == TJ.SAMP_444 || subsamp == TJ.SAMP_GRAY ||
+          (num == 1 && (denom == 4 || denom == 2 || denom == 1)))
         decompTest(tjd, jpegBuf, jpegSize, w, h, pf, baseName, subsamp,
                    flags, sf[i]);
-    } else
-      decompTest(tjd, jpegBuf, jpegSize, w, h, pf, baseName, subsamp,
-                 flags, new TJScalingFactor(1, 1));
+    }
   }
 
   private static void doTest(int w, int h, int[] formats, int subsamp,

diff --git a/tjunittest.c b/tjunittest.c
index 85ba32e8f7e59158b6b4017b45d81a07c39be604..c14c7d3248cc7444faa6592318a82d9ad98c784c 100644 (file)
--- a/tjunittest.c
+++ b/tjunittest.c
@@ -366,7 +366,7 @@ void compTest(tjhandle handle, unsigned char **dstBuf,
                if(!alloc)
                {
                        flags|=TJFLAG_NOREALLOC;
-                       *dstSize=(yuv==YUVENCODE? tjBufSizeYUV(w, h, subsamp)
+                       *dstSize=(yuv==YUVENCODE? tjBufSizeYUV2(w, pad, h, subsamp)
                                : tjBufSize(w, h, subsamp));
                }
                _tj(tjCompress2(handle, srcBuf, w, 0, h, pf, dstBuf, dstSize, subsamp,
@@ -466,18 +466,16 @@ void decompTest(tjhandle handle, unsigned char *jpegBuf,
        int flags)
 {
        int i, n=0;
-       tjscalingfactor *sf=tjGetScalingFactors(&n), sf1={1, 1};
+       tjscalingfactor *sf=tjGetScalingFactors(&n);
        if(!sf || !n) _throwtj();
 
-       if(subsamp==TJSAMP_444 || subsamp==TJSAMP_GRAY)
+       for(i=0; i<n; i++)
        {
-               for(i=0; i<n; i++)
+               if(subsamp==TJSAMP_444 || subsamp==TJSAMP_GRAY ||
+                       (sf[i].num==1 && (sf[i].denom==4 || sf[i].denom==2 || sf[i].denom==1)))
                        _decompTest(handle, jpegBuf, jpegSize, w, h, pf, basename, subsamp,
                                flags, sf[i]);
        }
-       else
-               _decompTest(handle, jpegBuf, jpegSize, w, h, pf, basename, subsamp, flags,
-                       sf1);
 
        bailout:
        return;

diff --git a/turbojpeg.c b/turbojpeg.c
index e5836491981632216392b3f5e5b38c5bf4868d58..e53cf667fd8b3954ea0d1666d152da2053fdf3e4 100644 (file)
--- a/turbojpeg.c
+++ b/turbojpeg.c
@@ -39,6 +39,7 @@
 #include "./turbojpeg.h"
 #include "./tjutil.h"
 #include "transupp.h"
+#include "./jpegcomp.h"
 
 extern void jpeg_mem_dest_tj(j_compress_ptr, unsigned char **,
        unsigned long *, boolean);
@@ -1058,10 +1059,11 @@ DLLEXPORT int DLLCALL tjDecompressToYUV2(tjhandle handle,
        int width, int pad, int height, int flags)
 {
        int i, sfi, row, retval=0;  JSAMPROW *outbuf[MAX_COMPONENTS];
-       int jpegwidth, jpegheight, scaledw, scaledh;
+       int jpegwidth, jpegheight, jpegSubsamp, scaledw, scaledh;
        int cw[MAX_COMPONENTS], ch[MAX_COMPONENTS], iw[MAX_COMPONENTS],
                tmpbufsize=0, usetmpbuf=0, th[MAX_COMPONENTS];
        JSAMPLE *_tmpbuf=NULL, *ptr=dstBuf;  JSAMPROW *tmpbuf[MAX_COMPONENTS];
+       int dctsize;
 
        getinstance(handle);
        if((this->init&DECOMPRESS)==0)
@@ -1089,6 +1091,9 @@ DLLEXPORT int DLLCALL tjDecompressToYUV2(tjhandle handle,
 
        jpeg_mem_src_tj(dinfo, jpegBuf, jpegSize);
        jpeg_read_header(dinfo, TRUE);
+       jpegSubsamp=getSubsamp(dinfo);
+       if(jpegSubsamp<0)
+               _throw("tjTransform(): Could not determine subsampling type for JPEG image");
 
        jpegwidth=dinfo->image_width;  jpegheight=dinfo->image_height;
        if(width==0) width=jpegwidth;
@@ -1108,18 +1113,20 @@ DLLEXPORT int DLLCALL tjDecompressToYUV2(tjhandle handle,
        sfi=i;
        jpeg_calc_output_dimensions(dinfo);
 
+       dctsize=DCTSIZE*sf[sfi].num/sf[sfi].denom;
+
        for(i=0; i<dinfo->num_components; i++)
        {
                jpeg_component_info *compptr=&dinfo->comp_info[i];
                int ih;
-               iw[i]=compptr->width_in_blocks*DCTSIZE*sf[sfi].num/sf[sfi].denom;
-               ih=compptr->height_in_blocks*DCTSIZE*sf[sfi].num/sf[sfi].denom;
+               iw[i]=compptr->width_in_blocks*dctsize;
+               ih=compptr->height_in_blocks*dctsize;
                cw[i]=PAD(dinfo->output_width, dinfo->max_h_samp_factor)
                        *compptr->h_samp_factor/dinfo->max_h_samp_factor;
                ch[i]=PAD(dinfo->output_height, dinfo->max_v_samp_factor)
                        *compptr->v_samp_factor/dinfo->max_v_samp_factor;
                if(iw[i]!=cw[i] || ih!=ch[i]) usetmpbuf=1;
-               th[i]=compptr->v_samp_factor*DCTSIZE*sf[sfi].num/sf[sfi].denom;
+               th[i]=compptr->v_samp_factor*dctsize;
                tmpbufsize+=iw[i]*th[i];
                if((outbuf[i]=(JSAMPROW *)malloc(sizeof(JSAMPROW)*ch[i]))==NULL)
                        _throw("tjDecompressToYUV2(): Memory allocation failure");
@@ -1152,19 +1159,37 @@ DLLEXPORT int DLLCALL tjDecompressToYUV2(tjhandle handle,
 
        jpeg_start_decompress(dinfo);
        for(row=0; row<(int)dinfo->output_height;
-               row+=dinfo->max_v_samp_factor*DCTSIZE*sf[sfi].num/sf[sfi].denom)
+               row+=dinfo->max_v_samp_factor*dinfo->_min_DCT_scaled_size)
        {
                JSAMPARRAY yuvptr[MAX_COMPONENTS];
                int crow[MAX_COMPONENTS];
                for(i=0; i<dinfo->num_components; i++)
                {
                        jpeg_component_info *compptr=&dinfo->comp_info[i];
+                       if(jpegSubsamp==TJ_420)
+                       {
+                               /* When 4:2:0 subsampling is used with IDCT scaling, libjpeg will try
+                                  to be clever and use the IDCT to perform upsampling on the U and V
+                                  planes.  For instance, if the output image is to be scaled by 1/2
+                                  relative to the JPEG image, then the scaling factor and upsampling
+                                  effectively cancel each other, so a normal 8x8 IDCT can be used.
+                                  However, this is not desirable when using the decompress-to-YUV
+                                  functionality in TurboJPEG, since we want to output the U and V
+                                  planes in their subsampled form.  Thus, we have to override some
+                                  internal libjpeg parameters to force it to use the "scaled" IDCT
+                                  functions on the U and V planes. */
+                               compptr->_DCT_scaled_size=dctsize;
+                               compptr->MCU_sample_width=tjMCUWidth[jpegSubsamp]*
+                                       sf[sfi].num/sf[sfi].denom*
+                                       compptr->v_samp_factor/dinfo->max_v_samp_factor;
+                               dinfo->idct->inverse_DCT[i] = dinfo->idct->inverse_DCT[0];
+                       }
                        crow[i]=row*compptr->v_samp_factor/dinfo->max_v_samp_factor;
                        if(usetmpbuf) yuvptr[i]=tmpbuf[i];
                        else yuvptr[i]=&outbuf[i][crow[i]];
                }
                jpeg_read_raw_data(dinfo, yuvptr,
-                       dinfo->max_v_samp_factor*DCTSIZE*sf[sfi].num/sf[sfi].denom);
+                       dinfo->max_v_samp_factor*dinfo->_min_DCT_scaled_size);
                if(usetmpbuf)
                {
                        int j;