From 7cf13826b7234e933d9aa4dac8ebddeab89ab4ac Mon Sep 17 00:00:00 2001 From: Ranjit Kumar Tulabandu Date: Wed, 23 Nov 2016 18:46:44 +0530 Subject: [PATCH] Bug fix to avoid random crashes during ARNR filtering The function 'vp9_find_best_sub_pixel_tree_pruned_more' is modified to return INT_MAX for handling invalid MV cases from UINT32_MAX. yunqingwang: patch 3: rebased on top of the tree. patch 4: The return type of vp9_find_best_sub_pixel_tree* was changed to uint32_t to fix ubsan warnings. Changing UINT_MAX back to INT_MAX was not quite right. Patch 4 modified vp9_temporal_filter.c to accept uint32_t. (Note: Inconsistency exists in vp9_find_best_sub_pixel_tree*, which will be fixed in a separate CL.) Change-Id: Ib1a79dc2aa41ea6335c21669c76883cdbb7e0535 --- vp9/encoder/vp9_temporal_filter.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vp9/encoder/vp9_temporal_filter.c b/vp9/encoder/vp9_temporal_filter.c index a167eeb15..344658483 100644 --- a/vp9/encoder/vp9_temporal_filter.c +++ b/vp9/encoder/vp9_temporal_filter.c @@ -208,17 +208,17 @@ void vp9_highbd_temporal_filter_apply_c( } #endif // CONFIG_VP9_HIGHBITDEPTH -static int temporal_filter_find_matching_mb_c(VP9_COMP *cpi, - uint8_t *arf_frame_buf, - uint8_t *frame_ptr_buf, - int stride) { +static uint32_t temporal_filter_find_matching_mb_c(VP9_COMP *cpi, + uint8_t *arf_frame_buf, + uint8_t *frame_ptr_buf, + int stride) { MACROBLOCK *const x = &cpi->td.mb; MACROBLOCKD *const xd = &x->e_mbd; MV_SPEED_FEATURES *const mv_sf = &cpi->sf.mv; const SEARCH_METHODS old_search_method = mv_sf->search_method; int step_param; int sadpb = x->sadperbit16; - int bestsme = INT_MAX; + uint32_t bestsme = UINT_MAX; uint32_t distortion; uint32_t sse; int cost_list[5]; @@ -334,8 +334,8 @@ static void temporal_filter_iterate_c(VP9_COMP *cpi, ((mb_cols - 1 - mb_col) * 16) + (17 - 2 * VP9_INTERP_EXTEND); for (frame = 0; frame < frame_count; frame++) { - const int thresh_low = 10000; - const int thresh_high = 20000; + const uint32_t thresh_low = 10000; + const uint32_t thresh_high = 20000; if (frames[frame] == NULL) continue; @@ -346,7 +346,7 @@ static void temporal_filter_iterate_c(VP9_COMP *cpi, filter_weight = 2; } else { // Find best match in this frame by MC - int err = temporal_filter_find_matching_mb_c( + uint32_t err = temporal_filter_find_matching_mb_c( cpi, frames[alt_ref_index]->y_buffer + mb_y_offset, frames[frame]->y_buffer + mb_y_offset, frames[frame]->y_stride); -- 2.40.0