From 90e12eaecba74ae459df0912d63e2d7cf878ad68 Mon Sep 17 00:00:00 2001 From: Yunqing Wang Date: Wed, 24 Aug 2016 17:05:17 -0700 Subject: [PATCH] Fix motion vector out of range bugs 2 bugs were fixed in VP9. https://chromium-review.googlesource.com/#/c/366873/ https://chromium-review.googlesource.com/#/c/368440/ Fixed them in VP10 as well. Change-Id: I2e53fabc6131ff80ba6dcfd4c73eb76c59b4c474 --- vp10/encoder/rdopt.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index 8890908a7..60f86a59d 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -4978,6 +4978,10 @@ static int64_t rd_pick_best_sub8x8_mode( MV mvp_full; int max_mv; int cost_list[5]; + int tmp_col_min = x->mv_col_min; + int tmp_col_max = x->mv_col_max; + int tmp_row_min = x->mv_row_min; + int tmp_row_max = x->mv_row_max; /* Is the best so far sufficiently good that we cant justify doing * and new motion search. */ @@ -5041,6 +5045,11 @@ static int64_t rd_pick_best_sub8x8_mode( cpi->sf.mv.subpel_search_method != SUBPEL_TREE ? cost_list : NULL, &bsi->ref_mv[0]->as_mv, INT_MAX, 1); + x->mv_col_min = tmp_col_min; + x->mv_col_max = tmp_col_max; + x->mv_row_min = tmp_row_min; + x->mv_row_max = tmp_row_max; + if (bestsme < INT_MAX) { int distortion; if (cpi->sf.use_upsampled_references) { @@ -5722,8 +5731,6 @@ static void single_motion_search(VP10_COMP *cpi, MACROBLOCK *x, vp10_setup_pre_planes(xd, ref_idx, scaled_ref_frame, mi_row, mi_col, NULL); } - vp10_set_mv_search_range(x, &ref_mv); - // Work out the size of the first step in the mv step search. // 0 here is maximum length first step. 1 is VPXMAX >> 1 etc. if (cpi->sf.mv.auto_mv_step_size && cm->show_frame) { @@ -5771,6 +5778,8 @@ static void single_motion_search(VP10_COMP *cpi, MACROBLOCK *x, } } + vp10_set_mv_search_range(x, &ref_mv); + mvp_full = pred_mv[x->mv_best_ref_index[ref]]; mvp_full.col >>= 3; @@ -5918,8 +5927,6 @@ static void single_motion_search_obmc(VP10_COMP *cpi, MACROBLOCK *x, vp10_setup_pre_planes(xd, ref_idx, scaled_ref_frame, mi_row, mi_col, NULL); } - vp10_set_mv_search_range(x, &ref_mv); - // Work out the size of the first step in the mv step search. // 0 here is maximum length first step. 1 is VPXMAX >> 1 etc. if (cpi->sf.mv.auto_mv_step_size && cm->show_frame) { @@ -5967,6 +5974,8 @@ static void single_motion_search_obmc(VP10_COMP *cpi, MACROBLOCK *x, } } + vp10_set_mv_search_range(x, &ref_mv); + mvp_full = pred_mv.as_mv; mvp_full.col >>= 3; mvp_full.row >>= 3; -- 2.50.1