Subtile decoding: fix overflows in subband coordinate computation that cause later...

diff --git a/src/lib/openjp2/dwt.c b/src/lib/openjp2/dwt.c
index d33fbeb37ecc35497ca2c3726b8c837a94a93f46..de3ff7270fade0e3258b1916a51ab3868c3976cd 100644 (file)
--- a/src/lib/openjp2/dwt.c
+++ b/src/lib/openjp2/dwt.c
@@ -1642,20 +1642,24 @@ static void opj_dwt_get_band_coordinates(opj_tcd_tilecomp_t* tilec,
     OPJ_UINT32 x0b = bandno & 1;
     OPJ_UINT32 y0b = bandno >> 1;
     if (tbx0) {
-        *tbx0 = (nb == 0) ? tcx0 : opj_uint_ceildiv(tcx0 - (1U <<
-                (nb - 1)) * x0b, 1U << nb);
+        *tbx0 = (nb == 0) ? tcx0 :
+                (tcx0 <= (1U << (nb - 1)) * x0b) ? 0 :
+                opj_uint_ceildivpow2(tcx0 - (1U << (nb - 1)) * x0b, nb);
     }
     if (tby0) {
-        *tby0 = (nb == 0) ? tcy0 : opj_uint_ceildiv(tcy0 - (1U <<
-                (nb - 1)) * y0b, 1U << nb);
+        *tby0 = (nb == 0) ? tcy0 :
+                (tcy0 <= (1U << (nb - 1)) * y0b) ? 0 :
+                opj_uint_ceildivpow2(tcy0 - (1U << (nb - 1)) * y0b, nb);
     }
     if (tbx1) {
-        *tbx1 = (nb == 0) ? tcx1 : opj_uint_ceildiv(tcx1 - (1U <<
-                (nb - 1)) * x0b, 1U << nb);
+        *tbx1 = (nb == 0) ? tcx1 :
+                (tcx1 <= (1U << (nb - 1)) * x0b) ? 0 :
+                opj_uint_ceildivpow2(tcx1 - (1U << (nb - 1)) * x0b, nb);
     }
     if (tby1) {
-        *tby1 = (nb == 0) ? tcy1 : opj_uint_ceildiv(tcy1 - (1U <<
-                (nb - 1)) * y0b, 1U << nb);
+        *tby1 = (nb == 0) ? tcy1 :
+                (tcy1 <= (1U << (nb - 1)) * y0b) ? 0 :
+                opj_uint_ceildivpow2(tcy1 - (1U << (nb - 1)) * y0b, nb);
     }
 }
 
@@ -1711,14 +1715,12 @@ static OPJ_BOOL opj_dwt_decode_partial_tile(opj_tcd_t *tcd,
     }
     h_mem_size = opj_dwt_max_resolution(tr, numres);
     /* overflow check */
-    if (h_mem_size > (SIZE_MAX / PARALLEL_COLS_53 / sizeof(OPJ_INT32))) {
+    if (h_mem_size > (SIZE_MAX / sizeof(OPJ_INT32))) {
         /* FIXME event manager error callback */
         return OPJ_FALSE;
     }
-    /* We need PARALLEL_COLS_53 times the height of the array, */
-    /* since for the vertical pass */
-    /* we process PARALLEL_COLS_53 columns at a time */
-    h_mem_size *= PARALLEL_COLS_53 * sizeof(OPJ_INT32);
+
+    h_mem_size *= sizeof(OPJ_INT32);
     h.mem = (OPJ_INT32*)opj_aligned_32_malloc(h_mem_size);
     if (! h.mem) {
         /* FIXME event manager error callback */

diff --git a/src/lib/openjp2/tcd.c b/src/lib/openjp2/tcd.c
index 7a5e2b44565be8cf93b622d5a5e968854afc3f1d..1c56c1b373e1ada2fc28d7a30f9941f294711315 100644 (file)
--- a/src/lib/openjp2/tcd.c
+++ b/src/lib/openjp2/tcd.c
@@ -2414,14 +2414,18 @@ OPJ_BOOL opj_tcd_is_subband_area_of_interest(opj_tcd_t *tcd,
     /* equation B-15 of the standard */
     OPJ_UINT32 x0b = bandno & 1;
     OPJ_UINT32 y0b = bandno >> 1;
-    OPJ_UINT32 tbx0 = (nb == 0) ? tcx0 : opj_uint_ceildiv(tcx0 - (1U <<
-                      (nb - 1)) * x0b, 1U << nb);
-    OPJ_UINT32 tby0 = (nb == 0) ? tcy0 : opj_uint_ceildiv(tcy0 - (1U <<
-                      (nb - 1)) * y0b, 1U << nb);
-    OPJ_UINT32 tbx1 = (nb == 0) ? tcx1 : opj_uint_ceildiv(tcx1 - (1U <<
-                      (nb - 1)) * x0b, 1U << nb);
-    OPJ_UINT32 tby1 = (nb == 0) ? tcy1 : opj_uint_ceildiv(tcy1 - (1U <<
-                      (nb - 1)) * y0b, 1U << nb);
+    OPJ_UINT32 tbx0 = (nb == 0) ? tcx0 :
+                      (tcx0 <= (1U << (nb - 1)) * x0b) ? 0 :
+                      opj_uint_ceildivpow2(tcx0 - (1U << (nb - 1)) * x0b, nb);
+    OPJ_UINT32 tby0 = (nb == 0) ? tcy0 :
+                      (tcy0 <= (1U << (nb - 1)) * y0b) ? 0 :
+                      opj_uint_ceildivpow2(tcy0 - (1U << (nb - 1)) * y0b, nb);
+    OPJ_UINT32 tbx1 = (nb == 0) ? tcx1 :
+                      (tcx1 <= (1U << (nb - 1)) * x0b) ? 0 :
+                      opj_uint_ceildivpow2(tcx1 - (1U << (nb - 1)) * x0b, nb);
+    OPJ_UINT32 tby1 = (nb == 0) ? tcy1 :
+                      (tcy1 <= (1U << (nb - 1)) * y0b) ? 0 :
+                      opj_uint_ceildivpow2(tcy1 - (1U << (nb - 1)) * y0b, nb);
     OPJ_BOOL intersects;
 
     if (tbx0 < filter_margin) {