[trunk] fixed a possible overflow in opj_t1_encode_cblks call to opj_int_fix_mul...

diff --git a/src/lib/openjp2/opj_intmath.h b/src/lib/openjp2/opj_intmath.h
index f35e40ab007dfe03ad05a0a3e938a4506f640332..641fcaa22631668e4cfa6c1dca3eaca52bf7dc08 100644 (file)
--- a/src/lib/openjp2/opj_intmath.h
+++ b/src/lib/openjp2/opj_intmath.h
@@ -108,7 +108,7 @@ Divide an integer and round upwards
 @return Returns a divided by b
 */
 static INLINE OPJ_INT32 opj_int_ceildiv(OPJ_INT32 a, OPJ_INT32 b) {
-  assert(b);
+       assert(b);
        return (a + b - 1) / b;
 }
 
@@ -117,6 +117,7 @@ Divide an integer and round upwards
 @return Returns a divided by b
 */
 static INLINE OPJ_UINT32  opj_uint_ceildiv(OPJ_UINT32  a, OPJ_UINT32  b) {
+       assert(b);
        return (a + b - 1) / b;
 }
 
@@ -165,9 +166,19 @@ Multiply two fixed-precision rational numbers.
 @return Returns a * b
 */
 static INLINE OPJ_INT32 opj_int_fix_mul(OPJ_INT32 a, OPJ_INT32 b) {
-    OPJ_INT64 temp = (OPJ_INT64) a * (OPJ_INT64) b ;
-    temp += 4096;
-    return (OPJ_INT32) (temp >> 13) ;
+       OPJ_INT64 temp = (OPJ_INT64) a * (OPJ_INT64) b ;
+       temp += 4096;
+       assert((temp >> 13) <= (OPJ_INT64)0x7FFFFFFF);
+       assert((temp >> 13) >= (-(OPJ_INT64)0x7FFFFFFF - (OPJ_INT64)1));
+       return (OPJ_INT32) (temp >> 13);
+}
+
+static INLINE OPJ_INT32 opj_int_fix_mul_t1(OPJ_INT32 a, OPJ_INT32 b) {
+       OPJ_INT64 temp = (OPJ_INT64) a * (OPJ_INT64) b ;
+       temp += 4096;
+       assert((temp >> (13 + 11 - T1_NMSEDEC_FRACBITS)) <= (OPJ_INT64)0x7FFFFFFF);
+       assert((temp >> (13 + 11 - T1_NMSEDEC_FRACBITS)) >= (-(OPJ_INT64)0x7FFFFFFF - (OPJ_INT64)1));
+       return (OPJ_INT32) (temp >> (13 + 11 - T1_NMSEDEC_FRACBITS)) ;
 }
 
 /* ----------------------------------------------------------------------- */

diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c
index e61cd3c94a6f18dd4899403fa824d7e971b72c13..8cb230f48edaeb9c0de3ae8c3b07c2896cc56862 100644 (file)
--- a/src/lib/openjp2/t1.c
+++ b/src/lib/openjp2/t1.c
@@ -1534,9 +1534,9 @@ OPJ_BOOL opj_t1_encode_cblks(   opj_t1_t *t1,
                                                                for (i = 0; i < cblk_w; ++i) {
                                                                        OPJ_INT32 tmp = tiledp[tileIndex];
                                                                        tiledp[tileIndex] =
-                                                                               opj_int_fix_mul(
+                                                                               opj_int_fix_mul_t1(
                                                                                tmp,
-                                                                               bandconst) >> (11 - T1_NMSEDEC_FRACBITS);
+                                                                               bandconst);
                                                                        tileIndex++;
                                                                }
                                                                tileIndex += tileLineAdvance;