Add overflow checks for opj_aligned_malloc (#841)

See
https://pdfium.googlesource.com/pdfium/+/b20ab6c7acb3be1393461eb650ca8fa4660c937e/third_party/libopenjpeg20/0020-opj_aligned_malloc.patch

diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c
index 2d793bb6a15abf8f1484f8b1f72a3cdbd7b8c977..98ead59e2e9ce3723f3c76e7224fbefc2a950405 100644 (file)
--- a/src/lib/openjp2/dwt.c
+++ b/src/lib/openjp2/dwt.c
@@ -395,7 +395,7 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
 
        OPJ_INT32 rw;                   /* width of the resolution level computed   */
        OPJ_INT32 rh;                   /* height of the resolution level computed  */
-       OPJ_UINT32 l_data_size;
+       size_t l_data_size;
 
        opj_tcd_resolution_t * l_cur_res = 0;
        opj_tcd_resolution_t * l_last_res = 0;
@@ -407,8 +407,14 @@ static INLINE OPJ_BOOL opj_dwt_encode_procedure(opj_tcd_tilecomp_t * tilec,void
        l_cur_res = tilec->resolutions + l;
        l_last_res = l_cur_res - 1;
 
-       l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions) * (OPJ_UINT32)sizeof(OPJ_INT32);
-       bj = (OPJ_INT32*)opj_malloc((size_t)l_data_size);
+       l_data_size = opj_dwt_max_resolution( tilec->resolutions,tilec->numresolutions);
+       /* overflow check */
+       if (l_data_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       l_data_size *= sizeof(OPJ_INT32);
+       bj = (OPJ_INT32*)opj_malloc(l_data_size);
        /* l_data_size is equal to 0 when numresolutions == 1 but bj is not used */
        /* in that case, so do not error out */
        if (l_data_size != 0 && ! bj) {
@@ -638,7 +644,13 @@ static OPJ_BOOL opj_dwt_decode_tile(opj_thread_pool_t* tp, opj_tcd_tilecomp_t* t
                return OPJ_TRUE;
        }
        num_threads = opj_thread_pool_get_thread_count(tp);
-       h_mem_size = opj_dwt_max_resolution(tr, numres) * sizeof(OPJ_INT32);
+       h_mem_size = opj_dwt_max_resolution(tr, numres);
+       /* overflow check */
+       if (h_mem_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       h_mem_size *= sizeof(OPJ_INT32);
        h.mem = (OPJ_INT32*)opj_aligned_malloc(h_mem_size);
        if (! h.mem){
                /* FIXME event manager error callback */
@@ -1003,7 +1015,21 @@ OPJ_BOOL opj_dwt_decode_real(opj_tcd_tilecomp_t* OPJ_RESTRICT tilec, OPJ_UINT32
 
        OPJ_UINT32 w = (OPJ_UINT32)(tilec->x1 - tilec->x0);
 
-       h.wavelet = (opj_v4_t*) opj_aligned_malloc((opj_dwt_max_resolution(res, numres)+5) * sizeof(opj_v4_t));
+       size_t l_data_size;
+       
+       l_data_size = opj_dwt_max_resolution(res, numres);
+       /* overflow check */
+       if (l_data_size > (SIZE_MAX - 5U)) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       l_data_size += 5U;
+       /* overflow check */
+       if (l_data_size > (SIZE_MAX / sizeof(opj_v4_t))) {
+               /* FIXME event manager error callback */
+               return OPJ_FALSE;
+       }
+       h.wavelet = (opj_v4_t*) opj_aligned_malloc(l_data_size * sizeof(opj_v4_t));
        if (!h.wavelet) {
                /* FIXME event manager error callback */
                return OPJ_FALSE;

diff --git a/src/lib/openjp2/pi.c b/src/lib/openjp2/pi.c
index 809b33d731dca0b4df4868949d29a5a474cc2a3a..41a2f046fde9d284ed813d06d14c4a44df9fa345 100644 (file)
--- a/src/lib/openjp2/pi.c
+++ b/src/lib/openjp2/pi.c
@@ -1238,14 +1238,14 @@ opj_pi_iterator_t *opj_pi_create_decode(opj_image_t *p_image,
 
        /* memory allocation for include */
        /* prevent an integer overflow issue */
+       /* 0 < l_tcp->numlayers < 65536 c.f. opj_j2k_read_cod in j2k.c */
        l_current_pi->include = 00;
        if (l_step_l <= (SIZE_MAX / (l_tcp->numlayers + 1U)))
        {
                l_current_pi->include = (OPJ_INT16*) opj_calloc((size_t)(l_tcp->numlayers + 1U) * l_step_l, sizeof(OPJ_INT16));
        }
 
-       if
-               (!l_current_pi->include)
+       if (!l_current_pi->include)
        {
                opj_free(l_tmp_data);
                opj_free(l_tmp_ptr);

diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c
index 53451677d63c491186483f3d8d688fa5b0145974..453d29089ffd8469216e8ff183ce72b81e8aabaf 100644 (file)
--- a/src/lib/openjp2/t1.c
+++ b/src/lib/openjp2/t1.c
@@ -1406,56 +1406,135 @@ static OPJ_BOOL opj_t1_allocate_buffers(
                OPJ_UINT32 w,
                OPJ_UINT32 h)
 {
-       OPJ_UINT32 datasize=w * h;
-       OPJ_UINT32 flagssize;
-
        /* encoder uses tile buffer, so no need to allocate */
        if (!t1->encoder) {
-               if(datasize > t1->datasize){
+               size_t datasize;
+               
+#if (SIZE_MAX / 0xFFFFFFFFU) < 0xFFFFFFFFU /* UINT32_MAX */
+               /* Overflow check */
+               if ((w > 0U) && ((size_t)h > (SIZE_MAX / (size_t)w))) {
+                       /* FIXME event manager error callback */
+                       return OPJ_FALSE;
+               }
+#endif
+               datasize = (size_t)w * h;
+
+               /* Overflow check */
+               if (datasize > (SIZE_MAX / sizeof(OPJ_INT32))) {
+                       /* FIXME event manager error callback */
+                       return OPJ_FALSE;
+               }
+
+               if(datasize > (size_t)t1->datasize){
                        opj_aligned_free(t1->data);
                        t1->data = (OPJ_INT32*) opj_aligned_malloc(datasize * sizeof(OPJ_INT32));
                        if(!t1->data){
                                /* FIXME event manager error callback */
                                return OPJ_FALSE;
                        }
-                       t1->datasize=datasize;
+#if SIZE_MAX > 0xFFFFFFFFU /* UINT32_MAX */
+                       /* TODO remove this if t1->datasize type changes to size_t */
+                       /* Overflow check */
+                       if (datasize > (size_t)0xFFFFFFFFU /* UINT32_MAX */) {
+                               /* FIXME event manager error callback */
+                               return OPJ_FALSE;
+                       }
+#endif
+                       t1->datasize = (OPJ_UINT32)datasize;
                }
                /* memset first arg is declared to never be null by gcc */
                if (t1->data != NULL) {
-                       memset(t1->data,0,datasize * sizeof(OPJ_INT32));
+                       memset(t1->data, 0, datasize * sizeof(OPJ_INT32));
                }
        }
-       t1->flags_stride=w+2;
-       flagssize=t1->flags_stride * (h+2);
 
-       if(flagssize > t1->flagssize){
-               opj_aligned_free(t1->flags);
-               t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t));
-               if(!t1->flags){
+       {
+               size_t flagssize;
+
+               /* Overflow check */
+               if (w > (0xFFFFFFFFU /* UINT32_MAX */ - 2U)) {
+                       /* FIXME event manager error callback */
+                       return OPJ_FALSE;
+               }
+               t1->flags_stride = w + 2U; /* can't be 0U */
+
+#if (SIZE_MAX - 3U) < 0xFFFFFFFFU /* UINT32_MAX */
+               /* Overflow check */
+               if (h > (0xFFFFFFFFU /* UINT32_MAX */ - 3U)) {
+                       /* FIXME event manager error callback */
+                       return OPJ_FALSE;
+               }
+#endif
+               flagssize = (size_t)h + 3U;
+               
+               /* Overflow check */
+               if (flagssize > (SIZE_MAX / (size_t)t1->flags_stride)) {
                        /* FIXME event manager error callback */
                        return OPJ_FALSE;
                }
-               t1->flagssize=flagssize;
+               flagssize *= (size_t)t1->flags_stride;
+
+               if(flagssize > (size_t)t1->flagssize){
+                       /* Overflow check */
+                       if (flagssize > (SIZE_MAX / sizeof(opj_flag_t))) {
+                               /* FIXME event manager error callback */
+                               return OPJ_FALSE;
+                       }
+                       opj_aligned_free(t1->flags);
+                       t1->flags = (opj_flag_t*) opj_aligned_malloc(flagssize * sizeof(opj_flag_t));
+                       if(!t1->flags){
+                               /* FIXME event manager error callback */
+                               return OPJ_FALSE;
+                       }
+#if SIZE_MAX > 0xFFFFFFFFU /* UINT32_MAX */
+                       /* TODO remove this if t1->flagssize type changes to size_t */
+                       /* Overflow check */
+                       if (flagssize > (size_t)0xFFFFFFFFU /* UINT32_MAX */) {
+                               /* FIXME event manager error callback */
+                               return OPJ_FALSE;
+                       }
+#endif
+                       t1->flagssize = (OPJ_UINT32)flagssize;
+               }
+               memset(t1->flags, 0, flagssize * sizeof(opj_flag_t));
        }
-       memset(t1->flags,0,flagssize * sizeof(opj_flag_t));
-       
        if (!t1->encoder) {
-               OPJ_UINT32 colflags_size=t1->flags_stride * ((h+3) / 4 + 2);
-
-               if(colflags_size > t1->colflags_size){
+               size_t colflags_size = ((((size_t)h + 3U) / 4U) + 2U); /* Can't overflow, h checked against UINT32_MAX - 3U */
+               
+               /* Overflow check */
+               if (colflags_size > (SIZE_MAX / (size_t)t1->flags_stride)) {
+                       /* FIXME event manager error callback */
+                       return OPJ_FALSE;
+               }
+               colflags_size *= (size_t)t1->flags_stride;
+               
+               if(colflags_size > (size_t)t1->colflags_size){
+                       /* Overflow check */
+                       if ((size_t)colflags_size > (SIZE_MAX / sizeof(opj_colflag_t))) {
+                               /* FIXME event manager error callback */
+                               return OPJ_FALSE;
+                       }
                        opj_aligned_free(t1->colflags);
                        t1->colflags = (opj_colflag_t*) opj_aligned_malloc(colflags_size * sizeof(opj_colflag_t));
                        if(!t1->colflags){
                                /* FIXME event manager error callback */
                                return OPJ_FALSE;
                        }
-                       t1->colflags_size=colflags_size;
+#if SIZE_MAX > 0xFFFFFFFFU /* UINT32_MAX */
+                       /* TODO remove this if t1->colflags_size type changes to size_t */
+                       /* Overflow check */
+                       if (colflags_size > (size_t)0xFFFFFFFFU /* UINT32_MAX */) {
+                               /* FIXME event manager error callback */
+                               return OPJ_FALSE;
+                       }
+#endif
+                       t1->colflags_size = (OPJ_UINT32)colflags_size;
                }
-               memset(t1->colflags,0,colflags_size * sizeof(opj_colflag_t));
+               memset(t1->colflags, 0, colflags_size * sizeof(opj_colflag_t));
        }
 
-       t1->w=w;
-       t1->h=h;
+       t1->w = w;
+       t1->h = h;
 
        return OPJ_TRUE;
 }