From 25110f2837e0ce58cacf74080d87654c6a046a12 Mon Sep 17 00:00:00 2001 From: Yaowu Xu Date: Wed, 15 Jun 2016 17:59:26 -0700 Subject: [PATCH] Fix an ubsan warning: vp9_quantizer.c BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1219 Change-Id: Ie1bfda4d51c1567f551515f0ed113eec1bc1d98c --- vp9/encoder/vp9_quantize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vp9/encoder/vp9_quantize.c b/vp9/encoder/vp9_quantize.c index 9766c059c..d68b6845c 100644 --- a/vp9/encoder/vp9_quantize.c +++ b/vp9/encoder/vp9_quantize.c @@ -94,7 +94,7 @@ void vp9_highbd_quantize_fp_c(const tran_low_t *coeff_ptr, const int coeff_sign = (coeff >> 31); const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int64_t tmp = abs_coeff + round_ptr[rc != 0]; - const uint32_t abs_qcoeff = (uint32_t)((tmp * quant_ptr[rc != 0]) >> 16); + const int abs_qcoeff = (int)((tmp * quant_ptr[rc != 0]) >> 16); qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; if (abs_qcoeff) -- 2.40.0