From: Jingning Han <jingning@google.com>
Date: Fri, 17 Jun 2016 21:52:05 +0000 (-0700)
Subject: Use 64-bit integer to store distortion in optimize_b
X-Git-Tag: v1.6.0~36^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=f99f78c7aff674580960420d172e77708a0fd5c3;p=libvpx

Use 64-bit integer to store distortion in optimize_b

This fixes the overflow issue.

Bug=webm:1241

Change-Id: Ia168b7fae1ad214a6837aaa785a08bf8506987dd
---

diff --git a/vp9/encoder/vp9_encodemb.c b/vp9/encoder/vp9_encodemb.c
index 936e6254b..e2c2e2379 100644
--- a/vp9/encoder/vp9_encodemb.c
+++ b/vp9/encoder/vp9_encodemb.c
@@ -54,9 +54,10 @@ void vp9_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane) {
   (((1 << (VP9_PROB_COST_SHIFT - 1)) + (R) * (RM)) & \
    ((1 << VP9_PROB_COST_SHIFT) - 1))
 
+// TODO(aconverse): Re-pack this structure.
 typedef struct vp9_token_state {
   int           rate;
-  int           error;
+  int64_t       error;
   int           next;
   int16_t       token;
   tran_low_t    qc;
@@ -112,7 +113,8 @@ static int optimize_b(MACROBLOCK *mb, int plane, int block,
   const int64_t rdmult = (mb->rdmult * plane_rd_mult[ref][type]) >> 1;
   const int64_t rddiv = mb->rddiv;
   int64_t rd_cost0, rd_cost1;
-  int rate0, rate1, error0, error1;
+  int rate0, rate1;
+  int64_t error0, error1;
   int16_t t0, t1;
   EXTRABIT e0;
   int best, band, pt, i, final_eob;