VP9 HBD: Fix integer overflow problem in variance calc.

BUG=webm:1534
Change-Id: I535ac48e3dd2454cc7088c4f9a1e08ea74107da6

diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 6d1d455fce04f74c4e977d783b4867dcd5300c2f..ed15c177dbd105aa95d28c676bab9c738293472e 100644 (file)
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -378,7 +378,8 @@ static void fill_variance(uint32_t s2, int32_t s, int c, var *v) {
 static void get_variance(var *v) {
   v->variance =
       (int)(256 * (v->sum_square_error -
-                   ((v->sum_error * v->sum_error) >> v->log2_count)) >>
+                   (uint32_t)(((int64_t)v->sum_error * v->sum_error) >>
+                              v->log2_count)) >>
             v->log2_count);
 }