[trunk] fixed a buffer overflow in opj_tcd_init_decode_tile

Update issue 431

diff --git a/src/lib/openjp2/pi.c b/src/lib/openjp2/pi.c
index 65ff6263e0db17fc39ae53a9fab2d4ca47c968c5..66e5b05c89549d4a3f3034407def55e44d33687c 100644 (file)
--- a/src/lib/openjp2/pi.c
+++ b/src/lib/openjp2/pi.c
@@ -708,10 +708,10 @@ void opj_get_all_encoding_parameters(   const opj_image_t *p_image,
        q = tileno / p_cp->tw;
 
        /* here calculation of tx0, tx1, ty0, ty1, maxprec, l_dx and l_dy */
-       *p_tx0 = opj_int_max((OPJ_INT32)(p_cp->tx0 + p * p_cp->tdx), (OPJ_INT32)p_image->x0);
-       *p_tx1 = opj_int_min((OPJ_INT32)(p_cp->tx0 + (p + 1) * p_cp->tdx), (OPJ_INT32)p_image->x1);
-       *p_ty0 = opj_int_max((OPJ_INT32)(p_cp->ty0 + q * p_cp->tdy), (OPJ_INT32)p_image->y0);
-       *p_ty1 = opj_int_min((OPJ_INT32)(p_cp->ty0 + (q + 1) * p_cp->tdy), (OPJ_INT32)p_image->y1);
+       *p_tx0 = (OPJ_INT32)opj_uint_max(p_cp->tx0 + p * p_cp->tdx, p_image->x0);
+       *p_tx1 = (OPJ_INT32)opj_uint_min(p_cp->tx0 + (p + 1) * p_cp->tdx, p_image->x1);
+       *p_ty0 = (OPJ_INT32)opj_uint_max(p_cp->ty0 + q * p_cp->tdy, p_image->y0);
+       *p_ty1 = (OPJ_INT32)opj_uint_min(p_cp->ty0 + (q + 1) * p_cp->tdy, p_image->y1);
 
        /* max precision and resolution is 0 (can only grow)*/
        *p_max_prec = 0;

diff --git a/src/lib/openjp2/tcd.c b/src/lib/openjp2/tcd.c
index eade5a910ac4fd447825caf13c2eef91d7aa21d2..a041bc1e91734075b9d7e510db1ea6abfb9586a6 100644 (file)
--- a/src/lib/openjp2/tcd.c
+++ b/src/lib/openjp2/tcd.c
@@ -701,10 +701,11 @@ static INLINE OPJ_BOOL opj_tcd_init_tile(opj_tcd_t *p_tcd, OPJ_UINT32 p_tile_no,
        /*fprintf(stderr, "Tile coordinate = %d,%d\n", p, q);*/
        
        /* 4 borders of the tile rescale on the image if necessary */
-       l_tile->x0 = opj_int_max((OPJ_INT32)(l_cp->tx0 + p * l_cp->tdx), (OPJ_INT32)l_image->x0);
-       l_tile->y0 = opj_int_max((OPJ_INT32)(l_cp->ty0 + q * l_cp->tdy), (OPJ_INT32)l_image->y0);
-       l_tile->x1 = opj_int_min((OPJ_INT32)(l_cp->tx0 + (p + 1) * l_cp->tdx), (OPJ_INT32)l_image->x1);
-       l_tile->y1 = opj_int_min((OPJ_INT32)(l_cp->ty0 + (q + 1) * l_cp->tdy), (OPJ_INT32)l_image->y1);
+       l_tile->x0 = (OPJ_INT32)opj_uint_max(l_cp->tx0 + p * l_cp->tdx, l_image->x0);
+       l_tile->y0 = (OPJ_INT32)opj_uint_max(l_cp->ty0 + q * l_cp->tdy, l_image->y0);
+       l_tile->x1 = (OPJ_INT32)opj_uint_min(l_cp->tx0 + (p + 1) * l_cp->tdx, l_image->x1);
+       l_tile->y1 = (OPJ_INT32)opj_uint_min(l_cp->ty0 + (q + 1) * l_cp->tdy, l_image->y1);
+
        /* testcase 1888.pdf.asan.35.988 */
        if (l_tccp->numresolutions == 0) {
                fprintf(stderr, "tiles require at least one resolution\n");