From 63a37d16f332350ae79620543dcb6bf5cd6d19ec Mon Sep 17 00:00:00 2001 From: Yaowu Xu Date: Wed, 29 Jun 2016 11:08:17 -0700 Subject: [PATCH] Prevent negative variance Due to rounding, hbd variance may become negative. This commit put in check and clamp of negative values to 0. Change-Id: I610d9c8aa2d4eebe7bc5f2c5624a9e3cadad4c94 --- vpx_dsp/x86/highbd_variance_sse2.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vpx_dsp/x86/highbd_variance_sse2.c b/vpx_dsp/x86/highbd_variance_sse2.c index 14d029c9a..f9ebaf7d3 100644 --- a/vpx_dsp/x86/highbd_variance_sse2.c +++ b/vpx_dsp/x86/highbd_variance_sse2.c @@ -143,24 +143,28 @@ uint32_t vpx_highbd_10_variance##w##x##h##_sse2( \ const uint8_t *src8, int src_stride, \ const uint8_t *ref8, int ref_stride, uint32_t *sse) { \ int sum; \ + int64_t var; \ uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ highbd_10_variance_sse2( \ src, src_stride, ref, ref_stride, w, h, sse, &sum, \ vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ - return *sse - (((int64_t)sum * sum) >> shift); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \ + return (var >= 0) ? (uint32_t)var : 0; \ } \ \ uint32_t vpx_highbd_12_variance##w##x##h##_sse2( \ const uint8_t *src8, int src_stride, \ const uint8_t *ref8, int ref_stride, uint32_t *sse) { \ int sum; \ + int64_t var; \ uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ highbd_12_variance_sse2( \ src, src_stride, ref, ref_stride, w, h, sse, &sum, \ vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ - return *sse - (((int64_t)sum * sum) >> shift); \ + var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \ + return (var >= 0) ? (uint32_t)var : 0; \ } VAR_FN(64, 64, 16, 12); -- 2.40.0