]> granicus.if.org Git - libvpx/commitdiff
Prevent integer overflow in choose_partitioning
authorJingning Han <jingning@google.com>
Wed, 11 Mar 2015 23:29:13 +0000 (16:29 -0700)
committerJingning Han <jingning@google.com>
Wed, 11 Mar 2015 23:31:42 +0000 (16:31 -0700)
Re-arrange the multiplication and right shift operations to avoid
integer overflow in choose_partitioning.

Change-Id: Ib4005cafb410a67c1960486471d75b6ebe38c4e0

vp9/encoder/vp9_encodeframe.c

index 26c5e6a9ed72b4a45507b1a2a083b5cb9dd5c846..cf8ac0a22d71c61ce01ea6172af0a56be38b5aca 100644 (file)
@@ -611,12 +611,12 @@ static void choose_partitioning(VP9_COMP *cpi,
       const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
 
       if (bs == BLOCK_INVALID)
-        uv_sad = INT_MAX;
+        uv_sad = UINT_MAX;
       else
         uv_sad = cpi->fn_ptr[bs].sdf(p->src.buf, p->src.stride,
                                      pd->dst.buf, pd->dst.stride);
 
-      x->color_sensitivity[i - 1] = uv_sad * 4 > y_sad;
+      x->color_sensitivity[i - 1] = uv_sad > (y_sad >> 2);
     }
 
     d = xd->plane[0].dst.buf;