From: Matthieu Darbois <mayeut@users.noreply.github.com>
Date: Sat, 20 Dec 2014 13:03:11 +0000 (+0000)
Subject: [trunk] fixed a possible overflow in opj_t1_encode_cblks call to opj_int_fix_mul... 
X-Git-Tag: v2.1.1~22^2~159
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=e966fef3eba04cf008852f89dcf71cd5a2dfcb40;p=openjpeg

[trunk] fixed a possible overflow in opj_t1_encode_cblks call to opj_int_fix_mul (fixes issue 141)
---

diff --git a/src/lib/openjp2/opj_intmath.h b/src/lib/openjp2/opj_intmath.h
index f35e40ab..641fcaa2 100644
--- a/src/lib/openjp2/opj_intmath.h
+++ b/src/lib/openjp2/opj_intmath.h
@@ -108,7 +108,7 @@ Divide an integer and round upwards
 @return Returns a divided by b
 */
 static INLINE OPJ_INT32 opj_int_ceildiv(OPJ_INT32 a, OPJ_INT32 b) {
-  assert(b);
+	assert(b);
 	return (a + b - 1) / b;
 }
 
@@ -117,6 +117,7 @@ Divide an integer and round upwards
 @return Returns a divided by b
 */
 static INLINE OPJ_UINT32  opj_uint_ceildiv(OPJ_UINT32  a, OPJ_UINT32  b) {
+	assert(b);
 	return (a + b - 1) / b;
 }
 
@@ -165,9 +166,19 @@ Multiply two fixed-precision rational numbers.
 @return Returns a * b
 */
 static INLINE OPJ_INT32 opj_int_fix_mul(OPJ_INT32 a, OPJ_INT32 b) {
-    OPJ_INT64 temp = (OPJ_INT64) a * (OPJ_INT64) b ;
-    temp += 4096;
-    return (OPJ_INT32) (temp >> 13) ;
+	OPJ_INT64 temp = (OPJ_INT64) a * (OPJ_INT64) b ;
+	temp += 4096;
+	assert((temp >> 13) <= (OPJ_INT64)0x7FFFFFFF);
+	assert((temp >> 13) >= (-(OPJ_INT64)0x7FFFFFFF - (OPJ_INT64)1));
+	return (OPJ_INT32) (temp >> 13);
+}
+
+static INLINE OPJ_INT32 opj_int_fix_mul_t1(OPJ_INT32 a, OPJ_INT32 b) {
+	OPJ_INT64 temp = (OPJ_INT64) a * (OPJ_INT64) b ;
+	temp += 4096;
+	assert((temp >> (13 + 11 - T1_NMSEDEC_FRACBITS)) <= (OPJ_INT64)0x7FFFFFFF);
+	assert((temp >> (13 + 11 - T1_NMSEDEC_FRACBITS)) >= (-(OPJ_INT64)0x7FFFFFFF - (OPJ_INT64)1));
+	return (OPJ_INT32) (temp >> (13 + 11 - T1_NMSEDEC_FRACBITS)) ;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/src/lib/openjp2/t1.c b/src/lib/openjp2/t1.c
index e61cd3c9..8cb230f4 100644
--- a/src/lib/openjp2/t1.c
+++ b/src/lib/openjp2/t1.c
@@ -1534,9 +1534,9 @@ OPJ_BOOL opj_t1_encode_cblks(   opj_t1_t *t1,
 								for (i = 0; i < cblk_w; ++i) {
 									OPJ_INT32 tmp = tiledp[tileIndex];
 									tiledp[tileIndex] =
-										opj_int_fix_mul(
+										opj_int_fix_mul_t1(
 										tmp,
-										bandconst) >> (11 - T1_NMSEDEC_FRACBITS);
+										bandconst);
 									tileIndex++;
 								}
 								tileIndex += tileLineAdvance;