From 3e2c6e2a63092630656c21ab9727454300bb6cef Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Wed, 20 Mar 2019 13:19:01 -0700 Subject: [PATCH] Properly reset memory in hbd setting This avoids a segmentation failure issue in high bit-depth case. Change-Id: I9fbb3ec24b1735678f110cb084a29b15e3ec1a12 --- vp9/encoder/vp9_encoder.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c index 2a9a36f61..03e8ed674 100644 --- a/vp9/encoder/vp9_encoder.c +++ b/vp9/encoder/vp9_encoder.c @@ -4746,13 +4746,16 @@ static void set_mb_wiener_variance(VP9_COMP *cpi) { #if CONFIG_VP9_HIGHBITDEPTH xd->cur_buf = cpi->Source; - if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) + if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { zero_pred = CONVERT_TO_BYTEPTR(zero_pred16); - else + memset(zero_pred16, 0, sizeof(*zero_pred16) * coeff_count); + } else { zero_pred = zero_pred8; -#endif - + memset(zero_pred8, 0, sizeof(*zero_pred8) * coeff_count); + } +#else memset(zero_pred, 0, sizeof(*zero_pred) * coeff_count); +#endif cpi->norm_wiener_variance = 0; -- 2.49.0