From 37102e55ec3614b3a2509ac57d9faf8f14e17300 Mon Sep 17 00:00:00 2001 From: Hien Ho Date: Tue, 1 Oct 2019 15:50:29 -0700 Subject: [PATCH] vpx_dsp/quantize: fix int sanitizer warnings From unit test: AVX/VP9QuantizeTest; SSSE3/VP9QuantizeTest ... implicit conversion from type 'int' of value -139812 (32-bit, signed) to type 'tran_low_t' (aka 'short') changed the value to -8740 (16-bit, signed) BUG=webm:1615 Change-Id: I730946ac6c7a250dcbcfd8a2712c0f1150ddb4fd --- vpx_dsp/quantize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vpx_dsp/quantize.c b/vpx_dsp/quantize.c index dfc22cd85..61818f692 100644 --- a/vpx_dsp/quantize.c +++ b/vpx_dsp/quantize.c @@ -156,7 +156,7 @@ void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, quant_shift_ptr[rc != 0]) >> 16; // quantization qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; - dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0]; + dqcoeff_ptr[rc] = (tran_low_t)(qcoeff_ptr[rc] * dequant_ptr[rc != 0]); if (tmp) eob = i; } -- 2.49.0