From 4b594d3d0053ef90fce59bbea44077cdcf944f9f Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Mon, 2 Nov 2015 12:05:47 -0800 Subject: [PATCH] Incorporate flexible tx type and tx partition in RD scheme This commit hooks up the rate-distortion optimization system to fully exploit recursive transform block partition and multiple transform type. The compression performance of the two experiments largely adds up. For derf set, ext-tx provides additional 2.1% coding gains on top of the gains due to recursive transform block partition (0.69%). Change-Id: I1091fb9545f74e489a6a2489dc3c12f5abd05043 --- vp10/common/blockd.h | 1 + vp10/decoder/decodeframe.c | 5 ++ vp10/encoder/encodeframe.c | 1 + vp10/encoder/rdopt.c | 104 ++++++++++++++++++++++++++++++++++--- 4 files changed, 105 insertions(+), 6 deletions(-) diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h index b02e7c92c..529fcdbf1 100644 --- a/vp10/common/blockd.h +++ b/vp10/common/blockd.h @@ -223,6 +223,7 @@ typedef struct macroblockd { // dimension in the unit of 8x8 block of the current block int16_t n8_w, n8_h; + TX_SIZE max_tx_size; #endif #if CONFIG_VP9_HIGHBITDEPTH diff --git a/vp10/decoder/decodeframe.c b/vp10/decoder/decodeframe.c index a6613f1f7..d807268ef 100644 --- a/vp10/decoder/decodeframe.c +++ b/vp10/decoder/decodeframe.c @@ -869,6 +869,11 @@ static MB_MODE_INFO *set_offsets(VP10_COMMON *const cm, MACROBLOCKD *const xd, set_skip_context(xd, mi_row, mi_col); + +#if CONFIG_VAR_TX + xd->max_tx_size = max_txsize_lookup[bsize]; +#endif + // Distance of Mb to the various image edges. These are specified to 8th pel // as they are always compared to values that are in 1/8th pel units set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols); diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c index 44a57e346..c3a85ef24 100644 --- a/vp10/encoder/encodeframe.c +++ b/vp10/encoder/encodeframe.c @@ -199,6 +199,7 @@ static void set_offsets(VP10_COMP *cpi, const TileInfo *const tile, #if CONFIG_VAR_TX xd->above_txfm_context = cm->above_txfm_context + mi_col; xd->left_txfm_context = xd->left_txfm_context_buffer + (mi_row & 0x07); + xd->max_tx_size = max_txsize_lookup[bsize]; #endif mbmi = &xd->mi[0]->mbmi; diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index 942303b89..dc15f99d9 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -1650,8 +1650,6 @@ static void tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, *bsse += (int64_t)tmp_sse * 16; if (p->eobs[block] > 0) { - // TODO(jingning): integrate multiple transform type experiment - TX_TYPE tx_type = DCT_DCT; switch (tx_size) { case TX_32X32: vp10_inv_txfm_add_32x32(dqcoeff, rec_buffer, 32, p->eobs[block], @@ -1878,10 +1876,6 @@ static void inter_block_yrd(const VP10_COMP *cpi, MACROBLOCK *x, *sse = 0; *skippable = 1; -#if CONFIG_EXT_TX - xd->mi[0]->mbmi.tx_type = DCT_DCT; -#endif - if (is_cost_valid) { const struct macroblockd_plane *const pd = &xd->plane[0]; const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd); @@ -1936,6 +1930,99 @@ static void inter_block_yrd(const VP10_COMP *cpi, MACROBLOCK *x, } } +#if CONFIG_EXT_TX +static void select_tx_type_yrd(const VP10_COMP *cpi, MACROBLOCK *x, + int *rate, int64_t *distortion, int *skippable, + int64_t *sse, BLOCK_SIZE bsize, + int64_t ref_best_rd) { + const TX_SIZE max_tx_size = max_txsize_lookup[bsize]; + const VP10_COMMON *const cm = &cpi->common; + MACROBLOCKD *const xd = &x->e_mbd; + MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; + int64_t rd = INT64_MAX; + int64_t best_rd = INT64_MAX; + TX_TYPE tx_type, best_tx_type = DCT_DCT; + int ext_tx_set; + const int is_inter = is_inter_block(mbmi); + vpx_prob skip_prob = vp10_get_skip_prob(cm, xd); + int s0 = vp10_cost_bit(skip_prob, 0); + int s1 = vp10_cost_bit(skip_prob, 1); + + *distortion = INT64_MAX; + *rate = INT_MAX; + *skippable = 0; + *sse = INT64_MAX; + + for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { + int this_rate = 0; + int this_skip = 1; + int64_t this_dist = 0; + int64_t this_sse = 0; + + ext_tx_set = get_ext_tx_set(max_tx_size, bsize, is_inter); + if (is_inter) { + if (!ext_tx_used_inter[ext_tx_set][tx_type]) + continue; + } else { + if (!ext_tx_used_intra[ext_tx_set][tx_type]) + continue; + } + + mbmi->tx_type = tx_type; + + if (ext_tx_set == 1 && + mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX && + best_tx_type == DCT_DCT) { + tx_type = IDTX - 1; + break; + } + + inter_block_yrd(cpi, x, &this_rate, &this_dist, &this_skip, &this_sse, + bsize, ref_best_rd); + + if (get_ext_tx_types(max_tx_size, bsize, is_inter) > 1 && + !xd->lossless[xd->mi[0]->mbmi.segment_id] && + this_rate != INT_MAX) { + if (is_inter) { + if (ext_tx_set > 0) + this_rate += cpi->inter_tx_type_costs[ext_tx_set] + [mbmi->tx_size][mbmi->tx_type]; + } else { + if (ext_tx_set > 0) + this_rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size] + [mbmi->mode][mbmi->tx_type]; + } + } + + if (this_rate == INT_MAX) + continue; + + if (this_skip) + rd = RDCOST(x->rdmult, x->rddiv, s1, this_sse); + else + rd = RDCOST(x->rdmult, x->rddiv, this_rate + s0, this_dist); + + if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] && !this_skip) + rd = VPXMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, this_sse)); + + if (rd < + (is_inter && best_tx_type == DCT_DCT ? ext_tx_th : 1) * + best_rd) { + best_rd = rd; + *distortion = this_dist; + *rate = this_rate; + *skippable = this_skip; + *sse = this_sse; + best_tx_type = mbmi->tx_type; + } + } + + mbmi->tx_type = best_tx_type; + inter_block_yrd(cpi, x, rate, distortion, skippable, sse, + bsize, ref_best_rd); +} +#endif + static void tx_block_rd(const VP10_COMP *cpi, MACROBLOCK *x, int blk_row, int blk_col, int plane, int block, TX_SIZE tx_size, BLOCK_SIZE plane_bsize, @@ -3704,8 +3791,13 @@ static int64_t handle_inter_mode(VP10_COMP *cpi, MACROBLOCK *x, vp10_subtract_plane(x, bsize, 0); #if CONFIG_VAR_TX if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) { +#if CONFIG_EXT_TX + select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse, + bsize, ref_best_rd); +#else inter_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse, bsize, ref_best_rd); +#endif } else { super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse, bsize, ref_best_rd); -- 2.40.0