From ad43a73883b2e18e07fc30f82486e83bf1746cdb Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Mon, 1 Feb 2016 09:47:39 -0800 Subject: [PATCH] Fix a signed overflow in vp9 motion cost. Change-Id: I5975e3aede62202d8ee6ced33889350c0a56554a --- vp9/encoder/vp9_mcomp.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vp9/encoder/vp9_mcomp.c b/vp9/encoder/vp9_mcomp.c index 84ef1b43e..4004dd3db 100644 --- a/vp9/encoder/vp9_mcomp.c +++ b/vp9/encoder/vp9_mcomp.c @@ -86,7 +86,9 @@ static int mv_err_cost(const MV *mv, const MV *ref, if (mvcost) { const MV diff = { mv->row - ref->row, mv->col - ref->col }; - return ROUND_POWER_OF_TWO(mv_cost(&diff, mvjcost, mvcost) * + // TODO(aconverse): See if this shift needs to be tied to + // VP9_PROB_COST_SHIFT. + return ROUND_POWER_OF_TWO((unsigned)mv_cost(&diff, mvjcost, mvcost) * error_per_bit, 13); } return 0; @@ -96,8 +98,9 @@ static int mvsad_err_cost(const MACROBLOCK *x, const MV *mv, const MV *ref, int error_per_bit) { const MV diff = { mv->row - ref->row, mv->col - ref->col }; - return ROUND_POWER_OF_TWO(mv_cost(&diff, x->nmvjointsadcost, - x->nmvsadcost) * error_per_bit, 8); + // TODO(aconverse): See if this shift needs to be tied to VP9_PROB_COST_SHIFT. + return ROUND_POWER_OF_TWO((unsigned)mv_cost(&diff, x->nmvjointsadcost, + x->nmvsadcost) * error_per_bit, 8); } void vp9_init_dsmotion_compensation(search_site_config *cfg, int stride) { -- 2.40.0