From: Yi Luo <luoyi@google.com>
Date: Mon, 3 Oct 2016 23:37:00 +0000 (-0700)
Subject: Fix high bitdepth variance overflow on uint32_t
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=a674ba93fe42dd25db52c2ce2f7e498febd10a1f;p=libvpx

Fix high bitdepth variance overflow on uint32_t

BUG=webm:1305

Change-Id: I4c56631359e298b99e618c07bcbae9f793c5e2ac
---

diff --git a/aom_dsp/x86/highbd_variance_sse4.c b/aom_dsp/x86/highbd_variance_sse4.c
index 75d7038f9..298cbf733 100644
--- a/aom_dsp/x86/highbd_variance_sse4.c
+++ b/aom_dsp/x86/highbd_variance_sse4.c
@@ -68,39 +68,42 @@ static INLINE void variance4x4_64_sse4_1(const uint8_t *a8, int a_stride,
 uint32_t aom_highbd_8_variance4x4_sse4_1(const uint8_t *a, int a_stride,
                                          const uint8_t *b, int b_stride,
                                          uint32_t *sse) {
-  int64_t sum;
+  int64_t sum, diff;
   uint64_t local_sse;
 
   variance4x4_64_sse4_1(a, a_stride, b, b_stride, &local_sse, &sum);
   *sse = (uint32_t)local_sse;
 
-  return *sse - (uint32_t)((sum * sum) >> 4);
+  diff = (int64_t)*sse - ((sum * sum) >> 4);
+  return (diff >= 0) ? (uint32_t)diff : 0;
 }
 
 uint32_t aom_highbd_10_variance4x4_sse4_1(const uint8_t *a, int a_stride,
                                           const uint8_t *b, int b_stride,
                                           uint32_t *sse) {
-  int64_t sum;
+  int64_t sum, diff;
   uint64_t local_sse;
 
   variance4x4_64_sse4_1(a, a_stride, b, b_stride, &local_sse, &sum);
   *sse = (uint32_t)ROUND_POWER_OF_TWO(local_sse, 4);
   sum = ROUND_POWER_OF_TWO(sum, 2);
 
-  return *sse - (uint32_t)((sum * sum) >> 4);
+  diff = (int64_t)*sse - ((sum * sum) >> 4);
+  return (diff >= 0) ? (uint32_t)diff : 0;
 }
 
 uint32_t aom_highbd_12_variance4x4_sse4_1(const uint8_t *a, int a_stride,
                                           const uint8_t *b, int b_stride,
                                           uint32_t *sse) {
-  int64_t sum;
+  int64_t sum, diff;
   uint64_t local_sse;
 
   variance4x4_64_sse4_1(a, a_stride, b, b_stride, &local_sse, &sum);
   *sse = (uint32_t)ROUND_POWER_OF_TWO(local_sse, 8);
   sum = ROUND_POWER_OF_TWO(sum, 4);
 
-  return *sse - (uint32_t)((sum * sum) >> 4);
+  diff = (int64_t)*sse - ((sum * sum) >> 4);
+  return diff >= 0 ? (uint32_t)diff : 0;
 }
 
 // Sub-pixel