From: Yaowu Xu <yaowu@google.com>
Date: Thu, 16 Jun 2016 00:59:26 +0000 (-0700)
Subject: Fix an ubsan warning: vp9_quantizer.c
X-Git-Tag: v1.6.0~50^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=25110f283;p=libvpx

Fix an ubsan warning: vp9_quantizer.c

BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1219

Change-Id: Ie1bfda4d51c1567f551515f0ed113eec1bc1d98c
---

diff --git a/vp9/encoder/vp9_quantize.c b/vp9/encoder/vp9_quantize.c
index 9766c059c..d68b6845c 100644
--- a/vp9/encoder/vp9_quantize.c
+++ b/vp9/encoder/vp9_quantize.c
@@ -94,7 +94,7 @@ void vp9_highbd_quantize_fp_c(const tran_low_t *coeff_ptr,
       const int coeff_sign = (coeff >> 31);
       const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
       const int64_t tmp = abs_coeff + round_ptr[rc != 0];
-      const uint32_t abs_qcoeff = (uint32_t)((tmp * quant_ptr[rc != 0]) >> 16);
+      const int abs_qcoeff = (int)((tmp * quant_ptr[rc != 0]) >> 16);
       qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
       dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
       if (abs_qcoeff)