Prevent negative variance

Due to rounding, hbd variance may become negative. This commit put in
check and clamp of negative values to 0.

Change-Id: I610d9c8aa2d4eebe7bc5f2c5624a9e3cadad4c94

diff --git a/vpx_dsp/x86/highbd_variance_sse2.c b/vpx_dsp/x86/highbd_variance_sse2.c
index 14d029c9a95dd55de8629b4d22a9dbb65f46cc79..f9ebaf7d3e65365631d07fc82d867e5cb8efd619 100644 (file)
--- a/vpx_dsp/x86/highbd_variance_sse2.c
+++ b/vpx_dsp/x86/highbd_variance_sse2.c
@@ -143,24 +143,28 @@ uint32_t vpx_highbd_10_variance##w##x##h##_sse2( \
     const uint8_t *src8, int src_stride, \
     const uint8_t *ref8, int ref_stride, uint32_t *sse) { \
   int sum; \
+  int64_t var; \
   uint16_t *src = CONVERT_TO_SHORTPTR(src8); \
   uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \
   highbd_10_variance_sse2( \
       src, src_stride, ref, ref_stride, w, h, sse, &sum, \
       vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \
-  return *sse - (((int64_t)sum * sum) >> shift); \
+  var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \
+  return (var >= 0) ? (uint32_t)var : 0; \
 } \
 \
 uint32_t vpx_highbd_12_variance##w##x##h##_sse2( \
     const uint8_t *src8, int src_stride, \
     const uint8_t *ref8, int ref_stride, uint32_t *sse) { \
   int sum; \
+  int64_t var; \
   uint16_t *src = CONVERT_TO_SHORTPTR(src8); \
   uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \
   highbd_12_variance_sse2( \
       src, src_stride, ref, ref_stride, w, h, sse, &sum, \
       vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \
-  return *sse - (((int64_t)sum * sum) >> shift); \
+  var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \
+  return (var >= 0) ? (uint32_t)var : 0; \
 }
 
 VAR_FN(64, 64, 16, 12);