From: Alex Converse <aconverse@google.com>
Date: Mon, 1 Feb 2016 17:47:39 +0000 (-0800)
Subject: Fix a signed overflow in vp9 motion cost.
X-Git-Tag: v1.6.0~391^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=ad43a73883b2e18e07fc30f82486e83bf1746cdb;p=libvpx

Fix a signed overflow in vp9 motion cost.

Change-Id: I5975e3aede62202d8ee6ced33889350c0a56554a
---

diff --git a/vp9/encoder/vp9_mcomp.c b/vp9/encoder/vp9_mcomp.c
index 84ef1b43e..4004dd3db 100644
--- a/vp9/encoder/vp9_mcomp.c
+++ b/vp9/encoder/vp9_mcomp.c
@@ -86,7 +86,9 @@ static int mv_err_cost(const MV *mv, const MV *ref,
   if (mvcost) {
     const MV diff = { mv->row - ref->row,
                       mv->col - ref->col };
-    return ROUND_POWER_OF_TWO(mv_cost(&diff, mvjcost, mvcost) *
+    // TODO(aconverse): See if this shift needs to be tied to
+    // VP9_PROB_COST_SHIFT.
+    return ROUND_POWER_OF_TWO((unsigned)mv_cost(&diff, mvjcost, mvcost) *
                                   error_per_bit, 13);
   }
   return 0;
@@ -96,8 +98,9 @@ static int mvsad_err_cost(const MACROBLOCK *x, const MV *mv, const MV *ref,
                           int error_per_bit) {
   const MV diff = { mv->row - ref->row,
                     mv->col - ref->col };
-  return ROUND_POWER_OF_TWO(mv_cost(&diff, x->nmvjointsadcost,
-                                    x->nmvsadcost) * error_per_bit, 8);
+  // TODO(aconverse): See if this shift needs to be tied to VP9_PROB_COST_SHIFT.
+  return ROUND_POWER_OF_TWO((unsigned)mv_cost(&diff, x->nmvjointsadcost,
+                                      x->nmvsadcost) * error_per_bit, 8);
 }
 
 void vp9_init_dsmotion_compensation(search_site_config *cfg, int stride) {