Adjust types in hbd error computation to avoid overflow

Change-Id: I8e08ebc8cbb2d1a1f97c8ef0c9237d8dfe0df208

diff --git a/av1/common/warped_motion.c b/av1/common/warped_motion.c
index 2ff1fcee59d99942cff61b8989432fcd7e8182b0..146a14d8ed6dd0f6e121c0c4e014c5b64029c4c1 100644 (file)
--- a/av1/common/warped_motion.c
+++ b/av1/common/warped_motion.c
@@ -477,7 +477,7 @@ static double highbd_warp_erroradv(WarpedMotionParams *wm, uint8_t *ref8,
   uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
   uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
   int gm_err = 0, no_gm_err = 0;
-  int gm_sumerr = 0, no_gm_sumerr = 0;
+  int64_t gm_sumerr = 0, no_gm_sumerr = 0;
   for (i = p_row; i < p_row + p_height; ++i) {
     for (j = p_col; j < p_col + p_width; ++j) {
       int in[2], out[2];
@@ -492,8 +492,8 @@ static double highbd_warp_erroradv(WarpedMotionParams *wm, uint8_t *ref8,
                                        stride, bd);
       no_gm_err = dst[(j - p_col) + (i - p_row) * p_stride] -
                   ref[(j - p_col) + (i - p_row) * stride];
-      gm_sumerr += gm_err * gm_err;
-      no_gm_sumerr += no_gm_err * no_gm_err;
+      gm_sumerr += (int64_t)gm_err * gm_err;
+      no_gm_sumerr += (int64_t)no_gm_err * no_gm_err;
     }
   }
   return (double)gm_sumerr / no_gm_sumerr;