From ca92da752b80d444b0c9d9db834330cf1cae55d0 Mon Sep 17 00:00:00 2001 From: Sarah Parker Date: Tue, 6 Sep 2016 16:15:07 -0700 Subject: [PATCH] Adjust types in hbd error computation to avoid overflow Change-Id: I8e08ebc8cbb2d1a1f97c8ef0c9237d8dfe0df208 --- av1/common/warped_motion.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/av1/common/warped_motion.c b/av1/common/warped_motion.c index 2ff1fcee5..146a14d8e 100644 --- a/av1/common/warped_motion.c +++ b/av1/common/warped_motion.c @@ -477,7 +477,7 @@ static double highbd_warp_erroradv(WarpedMotionParams *wm, uint8_t *ref8, uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); int gm_err = 0, no_gm_err = 0; - int gm_sumerr = 0, no_gm_sumerr = 0; + int64_t gm_sumerr = 0, no_gm_sumerr = 0; for (i = p_row; i < p_row + p_height; ++i) { for (j = p_col; j < p_col + p_width; ++j) { int in[2], out[2]; @@ -492,8 +492,8 @@ static double highbd_warp_erroradv(WarpedMotionParams *wm, uint8_t *ref8, stride, bd); no_gm_err = dst[(j - p_col) + (i - p_row) * p_stride] - ref[(j - p_col) + (i - p_row) * stride]; - gm_sumerr += gm_err * gm_err; - no_gm_sumerr += no_gm_err * no_gm_err; + gm_sumerr += (int64_t)gm_err * gm_err; + no_gm_sumerr += (int64_t)no_gm_err * no_gm_err; } } return (double)gm_sumerr / no_gm_sumerr; -- 2.50.1