From: James Zern <jzern@google.com>
Date: Thu, 9 Jun 2016 00:29:02 +0000 (-0700)
Subject: fdct4x4_test: fix unsigned overflow
X-Git-Tag: v1.6.0~72^2~1
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=06c6e4cbf6e42db1de8589eadd1566c1ef36aed8;p=libvpx

fdct4x4_test: fix unsigned overflow

the difference between src and dst will be signed, the error will be
unsigned.
quiets -fsanitize=integer:
unsigned integer overflow: 4294967295 * 4294967295

Change-Id: I502fd707823c4faaa7f587c9cc0312f057e04904
---

diff --git a/test/fdct4x4_test.cc b/test/fdct4x4_test.cc
index 5a58830d5..735cccf8d 100644
--- a/test/fdct4x4_test.cc
+++ b/test/fdct4x4_test.cc
@@ -141,11 +141,11 @@ class Trans4x4TestBase {
 
       for (int j = 0; j < kNumCoeffs; ++j) {
 #if CONFIG_VP9_HIGHBITDEPTH
-        const uint32_t diff =
+        const int diff =
             bit_depth_ == VPX_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
 #else
         ASSERT_EQ(VPX_BITS_8, bit_depth_);
-        const uint32_t diff = dst[j] - src[j];
+        const int diff = dst[j] - src[j];
 #endif
         const uint32_t error = diff * diff;
         if (max_error < error)
@@ -258,10 +258,10 @@ class Trans4x4TestBase {
 
       for (int j = 0; j < kNumCoeffs; ++j) {
 #if CONFIG_VP9_HIGHBITDEPTH
-        const uint32_t diff =
+        const int diff =
             bit_depth_ == VPX_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
 #else
-        const uint32_t diff = dst[j] - src[j];
+        const int diff = dst[j] - src[j];
 #endif
         const uint32_t error = diff * diff;
         EXPECT_GE(static_cast<uint32_t>(limit), error)