From: Ronald S. Bultje <rbultje@google.com>
Date: Wed, 14 Nov 2012 23:02:43 +0000 (-0800)
Subject: Prevent overflow in variance32x32.
X-Git-Tag: v1.3.0~1217^2~102^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=a0993703702ba1c1af9d4c04debe3f040f18a98d;p=libvpx

Prevent overflow in variance32x32.

Change-Id: I478878c78ef8a770186622d987d318176827ef5f
---

diff --git a/vp9/encoder/variance_c.c b/vp9/encoder/variance_c.c
index 8ac956e95..7703ef0c2 100644
--- a/vp9/encoder/variance_c.c
+++ b/vp9/encoder/variance_c.c
@@ -12,7 +12,7 @@
 #include "variance.h"
 #include "vp9/common/filter.h"
 #include "vp9/common/subpelvar.h"
-
+#include "vpx/vpx_integer.h"
 
 unsigned int vp9_get_mb_ss_c(const short *src_ptr) {
   unsigned int i, sum = 0;
@@ -37,8 +37,7 @@ unsigned int vp9_variance32x32_c(const unsigned char *src_ptr,
 
   variance(src_ptr, source_stride, ref_ptr, recon_stride, 32, 32, &var, &avg);
   *sse = var;
-  // TODO(rbultje): in extreme cases these products will rollover.
-  return (var - (((unsigned int)avg * avg) >> 10));
+  return (var - (((int64_t)avg * avg) >> 10));
 }
 #endif