]> granicus.if.org Git - libvpx/commitdiff
test/vp9_quantize_test: fix int sanitizer warning
authorHien Ho <hienho@google.com>
Fri, 16 Aug 2019 22:48:32 +0000 (15:48 -0700)
committerHien Ho <hienho@google.com>
Thu, 22 Aug 2019 23:15:17 +0000 (23:15 +0000)
implicit conversion from type 'int' of value 42126 (32-bit, signed)
to type 'tran_low_t' (aka 'short') changed the value to -23410 (16-bit, signed)

BUG=webm:1615

Change-Id: I339c640fce81e9f2dd73ef9c9bee084b6a5638dc

test/vp9_quantize_test.cc

index d094904f1aec0410a58e950651ff73b2a7818214..a0c883302a7d4335dcc30e6a2a1da78ac37b4c9c 100644 (file)
@@ -214,12 +214,15 @@ inline void quant_fp_nz(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
         tmp = clamp(abs_coeff[y] + _round, INT16_MIN, INT16_MAX);
         tmp = (tmp * quant_ptr[rc != 0]) >> (16 - is_32x32);
         qcoeff_ptr[rc] = (tmp ^ coeff_sign[y]) - coeff_sign[y];
-        dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
+        dqcoeff_ptr[rc] =
+            static_cast<tran_low_t>(qcoeff_ptr[rc] * dequant_ptr[rc != 0]);
 
         if (is_32x32) {
-          dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / 2;
+          dqcoeff_ptr[rc] = static_cast<tran_low_t>(qcoeff_ptr[rc] *
+                                                    dequant_ptr[rc != 0] / 2);
         } else {
-          dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
+          dqcoeff_ptr[rc] =
+              static_cast<tran_low_t>(qcoeff_ptr[rc] * dequant_ptr[rc != 0]);
         }
       } else {
         qcoeff_ptr[rc] = 0;