From 5a3e3c6d3fa308066e2cef1f8cbc407cd540c176 Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Wed, 25 Jun 2014 16:53:07 -0700 Subject: [PATCH] Adaptive txfm size selection depending on residual sse/variance This commit enables an adaptive transform size selection method for speed -6. It uses largest transform size when the sse is more than 4 times of variance, i.e., most energy is compacted in the DC coefficient. Otherwise, use the default TX_8X8. It improves the compression efficiency for rtc set of speed -6 by 0.8%, no speed change observed. Change-Id: Ie6ed1e728ff7bf88ebe940a60811361cdd19969c --- vp9/encoder/vp9_encodeframe.c | 2 +- vp9/encoder/vp9_pickmode.c | 32 +++++++++++++++++++++++++++----- vp9/encoder/vp9_speed_features.c | 3 ++- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c index d1c882443..5387e681c 100644 --- a/vp9/encoder/vp9_encodeframe.c +++ b/vp9/encoder/vp9_encodeframe.c @@ -2390,7 +2390,7 @@ static TX_MODE select_tx_mode(const VP9_COMP *cpi) { rd_opt->tx_select_threshes[frame_type][TX_MODE_SELECT] ? ALLOW_32X32 : TX_MODE_SELECT; } else if (cpi->sf.tx_size_search_method == USE_TX_8X8) { - return ALLOW_8X8; + return TX_MODE_SELECT; } else { unsigned int total = 0; int i; diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c index 4c340ea0b..5a97aea36 100644 --- a/vp9/encoder/vp9_pickmode.c +++ b/vp9/encoder/vp9_pickmode.c @@ -252,6 +252,17 @@ static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize, else x->skip_txfm = 0; + if (cpi->common.tx_mode == TX_MODE_SELECT) { + if (sse > (var << 2)) + xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize], + tx_mode_to_biggest_tx_size[cpi->common.tx_mode]); + else + xd->mi[0]->mbmi.tx_size = TX_8X8; + } else { + xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize], + tx_mode_to_biggest_tx_size[cpi->common.tx_mode]); + } + vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize], dc_quant >> 3, &rate, &dist); *out_rate_sum = rate >> 1; @@ -293,6 +304,8 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, struct macroblockd_plane *const pd = &xd->plane[0]; PREDICTION_MODE this_mode, best_mode = ZEROMV; MV_REFERENCE_FRAME ref_frame, best_ref_frame = LAST_FRAME; + TX_SIZE best_tx_size = MIN(max_txsize_lookup[bsize], + tx_mode_to_biggest_tx_size[cpi->common.tx_mode]); INTERP_FILTER best_pred_filter = EIGHTTAP; int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES]; struct buf_2d yv12_mb[4][MAX_MB_PLANE]; @@ -486,6 +499,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, int64_t pf_dist[3]; unsigned int pf_var[3]; unsigned int pf_sse[3]; + TX_SIZE pf_tx_size[3]; int64_t best_cost = INT64_MAX; INTERP_FILTER best_filter = SWITCHABLE, filter; PRED_BUFFER *current_pred = this_mode_pred; @@ -499,6 +513,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, cost = RDCOST(x->rdmult, x->rddiv, vp9_get_switchable_rate(cpi) + pf_rate[filter], pf_dist[filter]); + pf_tx_size[filter] = mbmi->tx_size; if (cost < best_cost) { best_filter = filter; best_cost = cost; @@ -523,6 +538,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, free_pred_buffer(current_pred); mbmi->interp_filter = best_filter; + mbmi->tx_size = pf_tx_size[mbmi->interp_filter]; rate = pf_rate[mbmi->interp_filter]; dist = pf_dist[mbmi->interp_filter]; var_y = pf_var[mbmi->interp_filter]; @@ -624,6 +640,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, *returndistortion = dist; best_mode = this_mode; best_pred_filter = mbmi->interp_filter; + best_tx_size = mbmi->tx_size; best_ref_frame = ref_frame; skip_txfm = x->skip_txfm; @@ -657,10 +674,11 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, bw, bh); } - mbmi->mode = best_mode; + mbmi->mode = best_mode; mbmi->interp_filter = best_pred_filter; - mbmi->ref_frame[0] = best_ref_frame; - mbmi->mv[0].as_int = frame_mv[best_mode][best_ref_frame].as_int; + mbmi->tx_size = best_tx_size; + mbmi->ref_frame[0] = best_ref_frame; + mbmi->mv[0].as_int = frame_mv[best_mode][best_ref_frame].as_int; xd->mi[0]->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int; x->skip_txfm = skip_txfm; @@ -669,7 +687,6 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, if (!x->skip && best_rd > inter_mode_thresh && bsize <= cpi->sf.max_intra_bsize) { int i, j; - const int step = 1 << mbmi->tx_size; const int width = num_4x4_blocks_wide_lookup[bsize]; const int height = num_4x4_blocks_high_lookup[bsize]; @@ -679,6 +696,10 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, const int src_stride = p->src.stride; int block_idx = 0; + TX_SIZE tmp_tx_size = MIN(max_txsize_lookup[bsize], + tx_mode_to_biggest_tx_size[cpi->common.tx_mode]); + const int step = 1 << tmp_tx_size; + for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) { if (cpi->sf.reuse_inter_pred_sby) { pd->dst.buf = tmp[0].data; @@ -688,7 +709,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, for (j = 0; j < height; j += step) { for (i = 0; i < width; i += step) { vp9_predict_intra_block(xd, block_idx, b_width_log2(bsize), - mbmi->tx_size, this_mode, + tmp_tx_size, this_mode, &p->src.buf[4 * (j * dst_stride + i)], src_stride, &pd->dst.buf[4 * (j * dst_stride + i)], @@ -715,6 +736,7 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, *returnrate = rate; *returndistortion = dist; mbmi->mode = this_mode; + mbmi->tx_size = tmp_tx_size; mbmi->ref_frame[0] = INTRA_FRAME; mbmi->uv_mode = this_mode; mbmi->mv[0].as_int = INVALID_MV; diff --git a/vp9/encoder/vp9_speed_features.c b/vp9/encoder/vp9_speed_features.c index 1a14da3c8..1730389a7 100644 --- a/vp9/encoder/vp9_speed_features.c +++ b/vp9/encoder/vp9_speed_features.c @@ -272,7 +272,8 @@ static void set_rt_speed_feature(VP9_COMP *cpi, SPEED_FEATURES *sf, sf->search_type_check_frequency = 50; sf->source_var_thresh = 360; - sf->tx_size_search_method = USE_TX_8X8; + sf->tx_size_search_method = (cm->frame_type == KEY_FRAME) ? + USE_LARGESTALL : USE_TX_8X8; sf->max_intra_bsize = BLOCK_8X8; // This feature is only enabled when partition search is disabled. -- 2.40.0