From: Yaowu Xu <yaowu@google.com>
Date: Thu, 16 Jun 2016 01:09:11 +0000 (-0700)
Subject: vpx_dsp/quantize.c: fix ubsan warnings
X-Git-Tag: v1.6.0~48
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=de3a8f23c8c10f9fb4f4430cca68d34ee316961d;p=libvpx

vpx_dsp/quantize.c: fix ubsan warnings

BUG=webm:1219

Change-Id: I0c80271c6b78adf40aa7a4cac9e6b431d56958cb
---

diff --git a/vpx_dsp/quantize.c b/vpx_dsp/quantize.c
index 6426cccc7..80fcd66b0 100644
--- a/vpx_dsp/quantize.c
+++ b/vpx_dsp/quantize.c
@@ -53,7 +53,7 @@ void vpx_highbd_quantize_dc(const tran_low_t *coeff_ptr,
     const int coeff_sign = (coeff >> 31);
     const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
     const int64_t tmp = abs_coeff + round_ptr[0];
-    const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> 16);
+    const int abs_qcoeff = (int)((tmp * quant) >> 16);
     qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
     dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr;
     if (abs_qcoeff)
@@ -109,7 +109,7 @@ void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr,
     const int coeff_sign = (coeff >> 31);
     const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
     const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[0], 1);
-    const uint32_t abs_qcoeff = (uint32_t)((tmp * quant) >> 15);
+    const int abs_qcoeff = (int)((tmp * quant) >> 15);
     qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
     dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant_ptr / 2;
     if (abs_qcoeff)