From 06c6e4cbf6e42db1de8589eadd1566c1ef36aed8 Mon Sep 17 00:00:00 2001 From: James Zern Date: Wed, 8 Jun 2016 17:29:02 -0700 Subject: [PATCH] fdct4x4_test: fix unsigned overflow the difference between src and dst will be signed, the error will be unsigned. quiets -fsanitize=integer: unsigned integer overflow: 4294967295 * 4294967295 Change-Id: I502fd707823c4faaa7f587c9cc0312f057e04904 --- test/fdct4x4_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/fdct4x4_test.cc b/test/fdct4x4_test.cc index 5a58830d5..735cccf8d 100644 --- a/test/fdct4x4_test.cc +++ b/test/fdct4x4_test.cc @@ -141,11 +141,11 @@ class Trans4x4TestBase { for (int j = 0; j < kNumCoeffs; ++j) { #if CONFIG_VP9_HIGHBITDEPTH - const uint32_t diff = + const int diff = bit_depth_ == VPX_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; #else ASSERT_EQ(VPX_BITS_8, bit_depth_); - const uint32_t diff = dst[j] - src[j]; + const int diff = dst[j] - src[j]; #endif const uint32_t error = diff * diff; if (max_error < error) @@ -258,10 +258,10 @@ class Trans4x4TestBase { for (int j = 0; j < kNumCoeffs; ++j) { #if CONFIG_VP9_HIGHBITDEPTH - const uint32_t diff = + const int diff = bit_depth_ == VPX_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; #else - const uint32_t diff = dst[j] - src[j]; + const int diff = dst[j] - src[j]; #endif const uint32_t error = diff * diff; EXPECT_GE(static_cast(limit), error) -- 2.40.0