From 77f88e97faea434a79881f96f758ca862feaeadb Mon Sep 17 00:00:00 2001 From: John Koleszar Date: Wed, 20 Feb 2013 12:34:31 -0800 Subject: [PATCH] Combined motion compensation with scaled predictors This patch extends the previous support for using references of a different resolution in ZEROMV mode to all inter prediction modes. Subpixel based best-mv scoring is disabled when the reference frame differs in resolution from the current frame. Change-Id: Id4dc3e5e6692de98d9857fd56bfad3ac57e944ac --- vp9/common/vp9_onyxc_int.h | 8 +++- vp9/decoder/vp9_decodemv.c | 73 ++++++++++++++-------------------- vp9/decoder/vp9_decodframe.c | 4 ++ vp9/encoder/vp9_onyx_if.c | 58 +++++++++++++++++++++++++-- vp9/encoder/vp9_onyx_int.h | 1 + vp9/encoder/vp9_rdopt.c | 77 +++++++++++++++++++++++++++--------- 6 files changed, 156 insertions(+), 65 deletions(-) diff --git a/vp9/common/vp9_onyxc_int.h b/vp9/common/vp9_onyxc_int.h index 0709e37ed..c4bb12340 100644 --- a/vp9/common/vp9_onyxc_int.h +++ b/vp9/common/vp9_onyxc_int.h @@ -39,7 +39,11 @@ void vp9_initialize_common(void); #define NUM_REF_FRAMES 3 #define NUM_REF_FRAMES_LG2 2 -#define NUM_YV12_BUFFERS (NUM_REF_FRAMES + 1) + +// 1 scratch frame for the new frame, 3 for scaled references on the encoder +// TODO(jkoleszar): These 3 extra references could probably come from the +// normal reference pool. +#define NUM_YV12_BUFFERS (NUM_REF_FRAMES + 4) #define NUM_FRAME_CONTEXTS_LG2 2 #define NUM_FRAME_CONTEXTS (1 << NUM_FRAME_CONTEXTS_LG2) @@ -128,6 +132,8 @@ typedef struct VP9Common { int Width; int Height; + int last_width; + int last_height; int horiz_scale; int vert_scale; diff --git a/vp9/decoder/vp9_decodemv.c b/vp9/decoder/vp9_decodemv.c index 121d6e339..5893c1132 100644 --- a/vp9/decoder/vp9_decodemv.c +++ b/vp9/decoder/vp9_decodemv.c @@ -698,6 +698,9 @@ static void read_mb_modes_mv(VP9D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, int mb_to_top_edge; int mb_to_bottom_edge; const int mb_size = 1 << mi->mbmi.sb_type; + const int use_prev_in_find_mv_refs = cm->Width == cm->last_width && + cm->Height == cm->last_height && + !cm->error_resilient_mode; mb_to_top_edge = xd->mb_to_top_edge; mb_to_bottom_edge = xd->mb_to_bottom_edge; @@ -751,28 +754,21 @@ static void read_mb_modes_mv(VP9D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, vp9_prob mv_ref_p [VP9_MVREFS - 1]; MV_REFERENCE_FRAME ref_frame = mbmi->ref_frame; + xd->scale_factor[0] = cm->active_ref_scale[mbmi->ref_frame - 1]; { int ref_fb_idx; - int recon_y_stride, recon_yoffset; - int recon_uv_stride, recon_uvoffset; + const int use_prev_in_find_best_ref = + xd->scale_factor[0].x_num == xd->scale_factor[0].x_den && + xd->scale_factor[0].y_num == xd->scale_factor[0].y_den && + !cm->error_resilient_mode && + !cm->frame_parallel_decoding_mode; /* Select the appropriate reference frame for this MB */ ref_fb_idx = cm->active_ref_idx[ref_frame - 1]; - recon_y_stride = cm->yv12_fb[ref_fb_idx].y_stride; - recon_uv_stride = cm->yv12_fb[ref_fb_idx].uv_stride; - - recon_yoffset = scaled_buffer_offset(mb_col * 16, mb_row * 16, - recon_y_stride, - &xd->scale_factor[0]); - recon_uvoffset = scaled_buffer_offset(mb_col * 8, mb_row * 8, - recon_uv_stride, - &xd->scale_factor_uv[0]); - - xd->pre.y_buffer = cm->yv12_fb[ref_fb_idx].y_buffer + recon_yoffset; - xd->pre.u_buffer = cm->yv12_fb[ref_fb_idx].u_buffer + recon_uvoffset; - xd->pre.v_buffer = cm->yv12_fb[ref_fb_idx].v_buffer + recon_uvoffset; + setup_pred_block(&xd->pre, &cm->yv12_fb[ref_fb_idx], + mb_row, mb_col, &xd->scale_factor[0], &xd->scale_factor_uv[0]); #ifdef DEC_DEBUG if (dec_debug) @@ -781,7 +777,7 @@ static void read_mb_modes_mv(VP9D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, #endif // if (cm->current_video_frame == 1 && mb_row == 4 && mb_col == 5) // printf("Dello\n"); - vp9_find_mv_refs(cm, xd, mi, cm->error_resilient_mode ? 0 : prev_mi, + vp9_find_mv_refs(cm, xd, mi, use_prev_in_find_mv_refs ? prev_mi : NULL, ref_frame, mbmi->ref_mvs[ref_frame], cm->ref_frame_sign_bias); @@ -814,10 +810,9 @@ static void read_mb_modes_mv(VP9D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, if (mbmi->mode != ZEROMV) { vp9_find_best_ref_mvs(xd, - pbi->common.error_resilient_mode || - pbi->common.frame_parallel_decoding_mode ? - 0 : xd->pre.y_buffer, - recon_y_stride, + use_prev_in_find_best_ref ? + xd->pre.y_buffer : NULL, + xd->pre.y_stride, mbmi->ref_mvs[ref_frame], &nearest, &nearby); @@ -858,39 +853,31 @@ static void read_mb_modes_mv(VP9D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, mbmi->second_ref_frame = 1; if (mbmi->second_ref_frame > 0) { int second_ref_fb_idx; - int recon_y_stride, recon_yoffset; - int recon_uv_stride, recon_uvoffset; + int use_prev_in_find_best_ref; + + xd->scale_factor[1] = cm->active_ref_scale[mbmi->second_ref_frame - 1]; + use_prev_in_find_best_ref = + xd->scale_factor[1].x_num == xd->scale_factor[1].x_den && + xd->scale_factor[1].y_num == xd->scale_factor[1].y_den && + !cm->error_resilient_mode && + !cm->frame_parallel_decoding_mode; /* Select the appropriate reference frame for this MB */ second_ref_fb_idx = cm->active_ref_idx[mbmi->second_ref_frame - 1]; - recon_y_stride = cm->yv12_fb[second_ref_fb_idx].y_stride; - recon_uv_stride = cm->yv12_fb[second_ref_fb_idx].uv_stride; - - recon_yoffset = scaled_buffer_offset(mb_col * 16, mb_row * 16, - recon_y_stride, - &xd->scale_factor[1]); - recon_uvoffset = scaled_buffer_offset(mb_col * 8, mb_row * 8, - recon_uv_stride, - &xd->scale_factor_uv[1]); - xd->second_pre.y_buffer = - cm->yv12_fb[second_ref_fb_idx].y_buffer + recon_yoffset; - xd->second_pre.u_buffer = - cm->yv12_fb[second_ref_fb_idx].u_buffer + recon_uvoffset; - xd->second_pre.v_buffer = - cm->yv12_fb[second_ref_fb_idx].v_buffer + recon_uvoffset; - - vp9_find_mv_refs(cm, xd, mi, cm->error_resilient_mode ? 0 : prev_mi, + setup_pred_block(&xd->second_pre, &cm->yv12_fb[second_ref_fb_idx], + mb_row, mb_col, &xd->scale_factor[1], &xd->scale_factor_uv[1]); + + vp9_find_mv_refs(cm, xd, mi, use_prev_in_find_mv_refs ? prev_mi : NULL, mbmi->second_ref_frame, mbmi->ref_mvs[mbmi->second_ref_frame], cm->ref_frame_sign_bias); if (mbmi->mode != ZEROMV) { vp9_find_best_ref_mvs(xd, - pbi->common.error_resilient_mode || - pbi->common.frame_parallel_decoding_mode ? - 0 : xd->second_pre.y_buffer, - recon_y_stride, + use_prev_in_find_best_ref ? + xd->second_pre.y_buffer : NULL, + xd->second_pre.y_stride, mbmi->ref_mvs[mbmi->second_ref_frame], &nearest_second, &nearby_second); diff --git a/vp9/decoder/vp9_decodframe.c b/vp9/decoder/vp9_decodframe.c index 96fcd4b2c..57467d1a6 100644 --- a/vp9/decoder/vp9_decodframe.c +++ b/vp9/decoder/vp9_decodframe.c @@ -1767,6 +1767,10 @@ int vp9_decode_frame(VP9D_COMP *pbi, const unsigned char **p_data_end) { } corrupt_tokens |= xd->corrupted; + // keep track of the last coded dimensions + pc->last_width = pc->Width; + pc->last_height = pc->Height; + /* Collect information about decoder corruption. */ /* 1. Check first boolean decoder for errors. */ pc->yv12_fb[pc->new_fb_idx].corrupted = bool_error(&header_bc); diff --git a/vp9/encoder/vp9_onyx_if.c b/vp9/encoder/vp9_onyx_if.c index 42f327674..1fec0cc1d 100644 --- a/vp9/encoder/vp9_onyx_if.c +++ b/vp9/encoder/vp9_onyx_if.c @@ -2599,6 +2599,38 @@ static void select_interintra_mode(VP9_COMP *cpi) { } #endif +static void scale_references(VP9_COMP *cpi) { + VP9_COMMON *cm = &cpi->common; + int i; + + for (i = 0; i < 3; i++) { + YV12_BUFFER_CONFIG *ref = &cm->yv12_fb[cm->active_ref_idx[i]]; + + if (ref->y_width != cm->Width || ref->y_height != cm->Height) { + int new_fb = get_free_fb(cm); + + vp8_yv12_realloc_frame_buffer(&cm->yv12_fb[new_fb], + cm->mb_cols * 16, + cm->mb_rows * 16, + VP9BORDERINPIXELS); + scale_and_extend_frame(ref, &cm->yv12_fb[new_fb]); + cpi->scaled_ref_idx[i] = new_fb; + } else { + cpi->scaled_ref_idx[i] = cm->active_ref_idx[i]; + cm->fb_idx_ref_cnt[cm->active_ref_idx[i]]++; + } + } +} + +static void release_scaled_references(VP9_COMP *cpi) { + VP9_COMMON *cm = &cpi->common; + int i; + + for (i = 0; i < 3; i++) { + cm->fb_idx_ref_cnt[cpi->scaled_ref_idx[i]]--; + } +} + static void encode_frame_to_data_rate(VP9_COMP *cpi, unsigned long *size, unsigned char *dest, @@ -2656,6 +2688,8 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi, cpi->Source = cpi->un_scaled_source; } + scale_references(cpi); + // Clear down mmx registers to allow floating point in what follows vp9_clear_system_state(); @@ -3304,6 +3338,7 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi, update_reference_segmentation_map(cpi); } + release_scaled_references(cpi); update_reference_frames(cpi); vp9_copy(cpi->common.fc.coef_counts_4x4, cpi->coef_counts_4x4); vp9_copy(cpi->common.fc.coef_counts_8x8, cpi->coef_counts_8x8); @@ -3589,6 +3624,9 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi, xd->update_mb_segmentation_data = 0; xd->mode_ref_lf_delta_update = 0; + // keep track of the last coded dimensions + cm->last_width = cm->Width; + cm->last_height = cm->Height; // Dont increment frame counters if this was an altref buffer update not a real frame if (cm->show_frame) { @@ -4083,18 +4121,32 @@ int vp9_set_active_map(VP9_PTR comp, unsigned char *map, int vp9_set_internal_size(VP9_PTR comp, VPX_SCALING horiz_mode, VPX_SCALING vert_mode) { VP9_COMP *cpi = (VP9_COMP *) comp; + VP9_COMMON *cm = &cpi->common; if (horiz_mode <= ONETWO) - cpi->horiz_scale = horiz_mode; + cm->horiz_scale = horiz_mode; else return -1; if (vert_mode <= ONETWO) - cpi->vert_scale = vert_mode; + cm->vert_scale = vert_mode; else return -1; - vp9_change_config(comp, &cpi->oxcf); + if (cm->horiz_scale != NORMAL || cm->vert_scale != NORMAL) { + int UNINITIALIZED_IS_SAFE(hr), UNINITIALIZED_IS_SAFE(hs); + int UNINITIALIZED_IS_SAFE(vr), UNINITIALIZED_IS_SAFE(vs); + + Scale2Ratio(cm->horiz_scale, &hr, &hs); + Scale2Ratio(cm->vert_scale, &vr, &vs); + + // always go to the next whole number + cm->Width = (hs - 1 + cpi->oxcf.Width * hr) / hs; + cm->Height = (vs - 1 + cpi->oxcf.Height * vr) / vs; + } + assert(cm->Width <= cpi->initial_width); + assert(cm->Height <= cpi->initial_height); + update_frame_size(cpi); return 0; } diff --git a/vp9/encoder/vp9_onyx_int.h b/vp9/encoder/vp9_onyx_int.h index 9b509ea0b..02a371964 100644 --- a/vp9/encoder/vp9_onyx_int.h +++ b/vp9/encoder/vp9_onyx_int.h @@ -332,6 +332,7 @@ typedef struct VP9_COMP { int alt_is_last; // Alt reference frame same as last ( short circuit altref search) int gold_is_alt; // don't do both alt and gold search ( just do gold). + int scaled_ref_idx[3]; int lst_fb_idx; int gld_fb_idx; int alt_fb_idx; diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 9d51e1cba..bc075d9be 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -3115,6 +3115,7 @@ static void setup_buffer_inter(VP9_COMP *cpi, MACROBLOCK *x, YV12_BUFFER_CONFIG *yv12 = &cm->yv12_fb[cpi->common.active_ref_idx[idx]]; MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi; + int use_prev_in_find_mv_refs, use_prev_in_find_best_ref; // set up scaling factors scale[frame_type] = cpi->common.active_ref_scale[frame_type - 1]; @@ -3129,18 +3130,24 @@ static void setup_buffer_inter(VP9_COMP *cpi, MACROBLOCK *x, &scale[frame_type], &scale[frame_type]); // Gets an initial list of candidate vectors from neighbours and orders them + use_prev_in_find_mv_refs = cm->Width == cm->last_width && + cm->Height == cm->last_height && + !cpi->common.error_resilient_mode; vp9_find_mv_refs(&cpi->common, xd, xd->mode_info_context, - cpi->common.error_resilient_mode ? - 0 : xd->prev_mode_info_context, + use_prev_in_find_mv_refs ? xd->prev_mode_info_context : NULL, frame_type, mbmi->ref_mvs[frame_type], cpi->common.ref_frame_sign_bias); // Candidate refinement carried out at encoder and decoder + use_prev_in_find_best_ref = + scale[frame_type].x_num == scale[frame_type].x_den && + scale[frame_type].y_num == scale[frame_type].y_den && + !cm->error_resilient_mode && + !cm->frame_parallel_decoding_mode; vp9_find_best_ref_mvs(xd, - cpi->common.error_resilient_mode || - cpi->common.frame_parallel_decoding_mode ? - 0 : yv12_mb[frame_type].y_buffer, + use_prev_in_find_best_ref ? + yv12_mb[frame_type].y_buffer : NULL, yv12->y_stride, mbmi->ref_mvs[frame_type], &frame_nearest_mv[frame_type], @@ -3212,6 +3219,7 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, INTERPOLATIONFILTERTYPE *best_filter, int_mv frame_mv[MB_MODE_COUNT] [MAX_REF_FRAMES], + YV12_BUFFER_CONFIG *scaled_ref_frame, int mb_row, int mb_col) { VP9_COMMON *cm = &cpi->common; MACROBLOCKD *xd = &x->e_mbd; @@ -3256,6 +3264,7 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, x->nmvjointcost, x->mvcost, 96, x->e_mbd.allow_high_precision_mv); } else { + YV12_BUFFER_CONFIG backup_yv12 = xd->pre; int bestsme = INT_MAX; int further_steps, step_param = cpi->sf.first_step; int sadpb = x->sadperbit16; @@ -3267,6 +3276,16 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, int tmp_row_min = x->mv_row_min; int tmp_row_max = x->mv_row_max; + if (scaled_ref_frame) { + // Swap out the reference frame for a version that's been scaled to + // match the resolution of the current frame, allowing the existing + // motion search code to be used without additional modifications. + xd->pre = *scaled_ref_frame; + xd->pre.y_buffer += mb_row * 16 * xd->pre.y_stride + mb_col * 16; + xd->pre.u_buffer += mb_row * 8 * xd->pre.uv_stride + mb_col * 8; + xd->pre.v_buffer += mb_row * 8 * xd->pre.uv_stride + mb_col * 8; + } + vp9_clamp_mv_min_max(x, &ref_mv[0]); // mvp_full.as_int = ref_mv[0].as_int; @@ -3309,6 +3328,11 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, *rate2 += vp9_mv_bit_cost(&tmp_mv, &ref_mv[0], x->nmvjointcost, x->mvcost, 96, xd->allow_high_precision_mv); + + // restore the predictor, if required + if (scaled_ref_frame) { + xd->pre = backup_yv12; + } } break; case NEARMV: @@ -3963,6 +3987,7 @@ static void rd_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, #endif int mode_excluded = 0; int64_t txfm_cache[NB_TXFM_MODES] = { 0 }; + YV12_BUFFER_CONFIG *scaled_ref_frame; // These variables hold are rolling total cost and distortion for this mode rate2 = 0; @@ -4042,12 +4067,25 @@ static void rd_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, } /* everything but intra */ + scaled_ref_frame = NULL; if (mbmi->ref_frame) { int ref = mbmi->ref_frame; + int fb; xd->pre = yv12_mb[ref]; best_ref_mv = mbmi->ref_mvs[ref][0]; vpx_memcpy(mdcounts, frame_mdcounts[ref], sizeof(mdcounts)); + + if (mbmi->ref_frame == LAST_FRAME) { + fb = cpi->lst_fb_idx; + } else if (mbmi->ref_frame == GOLDEN_FRAME) { + fb = cpi->gld_fb_idx; + } else { + fb = cpi->alt_fb_idx; + } + + if (cpi->scaled_ref_idx[fb] != cm->active_ref_idx[fb]) + scaled_ref_frame = &cm->yv12_fb[cpi->scaled_ref_idx[fb]]; } if (mbmi->second_ref_frame > 0) { @@ -4371,7 +4409,7 @@ static void rd_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, &rate_uv, &distortion_uv, &mode_excluded, &disable_skip, mode_index, &tmp_best_filter, frame_mv, - mb_row, mb_col); + scaled_ref_frame, mb_row, mb_col); if (this_rd == INT64_MAX) continue; } @@ -5025,17 +5063,6 @@ static int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, mbmi->interintra_uv_mode = (MB_PREDICTION_MODE)(DC_PRED - 1); #endif - if (mbmi->ref_frame > 0 && - (yv12_mb[mbmi->ref_frame].y_width != cm->mb_cols * 16 || - yv12_mb[mbmi->ref_frame].y_height != cm->mb_rows * 16) && - this_mode != ZEROMV) - continue; - if (mbmi->second_ref_frame > 0 && - (yv12_mb[mbmi->second_ref_frame].y_width != cm->mb_cols * 16 || - yv12_mb[mbmi->second_ref_frame].y_height != cm->mb_rows * 16) && - this_mode != ZEROMV) - continue; - // Evaluate all sub-pel filters irrespective of whether we can use // them for this frame. mbmi->interp_filter = cm->mcomp_filter_type; @@ -5139,6 +5166,20 @@ static int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, rate2 = rate_y + x->mbmode_cost[cm->frame_type][mbmi->mode] + rate_uv; distortion2 = distortion_y + distortion_uv; } else { + YV12_BUFFER_CONFIG *scaled_ref_frame = NULL; + int fb; + + if (mbmi->ref_frame == LAST_FRAME) { + fb = cpi->lst_fb_idx; + } else if (mbmi->ref_frame == GOLDEN_FRAME) { + fb = cpi->gld_fb_idx; + } else { + fb = cpi->alt_fb_idx; + } + + if (cpi->scaled_ref_idx[fb] != cm->active_ref_idx[fb]) + scaled_ref_frame = &cm->yv12_fb[cpi->scaled_ref_idx[fb]]; + #if CONFIG_COMP_INTERINTRA_PRED if (mbmi->second_ref_frame == INTRA_FRAME) { if (best_intra16_mode == DC_PRED - 1) continue; @@ -5161,7 +5202,7 @@ static int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x, &rate_uv, &distortion_uv, &mode_excluded, &disable_skip, mode_index, &tmp_best_filter, frame_mv, - mb_row, mb_col); + scaled_ref_frame, mb_row, mb_col); if (this_rd == INT64_MAX) continue; } -- 2.40.0