From 238b6be24bcd6cbb626d88d08fceb12622f7ad89 Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Wed, 11 Mar 2015 16:29:13 -0700 Subject: [PATCH] Prevent integer overflow in choose_partitioning Re-arrange the multiplication and right shift operations to avoid integer overflow in choose_partitioning. Change-Id: Ib4005cafb410a67c1960486471d75b6ebe38c4e0 --- vp9/encoder/vp9_encodeframe.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c index 26c5e6a9e..cf8ac0a22 100644 --- a/vp9/encoder/vp9_encodeframe.c +++ b/vp9/encoder/vp9_encodeframe.c @@ -611,12 +611,12 @@ static void choose_partitioning(VP9_COMP *cpi, const BLOCK_SIZE bs = get_plane_block_size(bsize, pd); if (bs == BLOCK_INVALID) - uv_sad = INT_MAX; + uv_sad = UINT_MAX; else uv_sad = cpi->fn_ptr[bs].sdf(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride); - x->color_sensitivity[i - 1] = uv_sad * 4 > y_sad; + x->color_sensitivity[i - 1] = uv_sad > (y_sad >> 2); } d = xd->plane[0].dst.buf; -- 2.50.1