From 4c6c82a2e856df8c3740b7d27c9b11d633716be2 Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Tue, 9 Feb 2016 17:51:49 -0800 Subject: [PATCH] Resolve conflict between var-tx and super-tx This commit aligns the rate-distortion metric for the recursive transform block partitioning and the super transform. It resolves the conflicts between these two experiments. The coding performance gains of the combined experiments (var-tx + super-tx) has been improved: derf 0.89% -> 1.9% hevcmr 1.06% -> 1.8% stdhd 0.29% -> 1.4% hevchr 0.80% -> 2.3% Change-Id: I7e33994ad70c1b2751435620815f867d82172f41 --- vp10/encoder/encodeframe.c | 26 +++++++++++++++++++++----- vp10/encoder/rdopt.c | 16 ++++++++-------- vp10/encoder/rdopt.h | 7 +++++++ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c index 18a49809b..9e20000ba 100644 --- a/vp10/encoder/encodeframe.c +++ b/vp10/encoder/encodeframe.c @@ -3164,7 +3164,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td, #if CONFIG_VAR_TX xd->above_txfm_context = cm->above_txfm_context + mi_col; xd->left_txfm_context = - xd->left_txfm_context_buffer + (mi_row & 0x07); + xd->left_txfm_context_buffer + (mi_row & MI_MASK); restore_context(x, mi_row, mi_col, a, l, sa, sl, ta, tl, bsize); #else restore_context(x, mi_row, mi_col, a, l, sa, sl, bsize); @@ -5188,6 +5188,11 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td, ext_tx_set = get_ext_tx_set(tx_size, bsize, 1); #endif // CONFIG_EXT_TX for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { +#if CONFIG_VAR_TX + ENTROPY_CONTEXT ctxa[16], ctxl[16]; + const struct macroblockd_plane *const pd = &xd->plane[0]; + int coeff_ctx = 1; +#endif // CONFIG_VAR_TX #if CONFIG_EXT_TX if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue; @@ -5201,12 +5206,23 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td, continue; #endif // CONFIG_EXT_TX mbmi->tx_type = tx_type; - vp10_txfm_rd_in_plane_supertx(x, + #if CONFIG_VAR_TX - cpi, -#endif - &this_rate, &this_dist, &pnskip, + this_rate = 0; + this_dist = 0; + pnsse = 0; + pnskip = 1; + + vp10_get_entropy_contexts(bsize, tx_size, pd, ctxa, ctxl); + coeff_ctx = combine_entropy_contexts(ctxa[0], ctxl[0]); + vp10_tx_block_rd_b(cpi, x, tx_size, + 0, 0, 0, 0, + bsize, coeff_ctx, + &this_rate, &this_dist, &pnsse, &pnskip); +#else + vp10_txfm_rd_in_plane_supertx(x, &this_rate, &this_dist, &pnskip, &pnsse, INT64_MAX, 0, bsize, tx_size, 0); +#endif // CONFIG_VAR_TX #if CONFIG_EXT_TX if (get_ext_tx_types(tx_size, bsize, 1) > 1 && diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index f22727220..9f21c5d34 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -2317,10 +2317,10 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x, } #if CONFIG_VAR_TX -static void tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, - int blk_row, int blk_col, int plane, int block, - int plane_bsize, int coeff_ctx, - int *rate, int64_t *dist, int64_t *bsse, int *skip) { +void vp10_tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, + int blk_row, int blk_col, int plane, int block, + int plane_bsize, int coeff_ctx, + int *rate, int64_t *dist, int64_t *bsse, int *skip) { MACROBLOCKD *xd = &x->e_mbd; const struct macroblock_plane *const p = &x->plane[plane]; struct macroblockd_plane *const pd = &xd->plane[plane]; @@ -2531,8 +2531,8 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x, if (cpi->common.tx_mode == TX_MODE_SELECT || tx_size == TX_4X4) { mbmi->inter_tx_size[tx_idx] = tx_size; - tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block, - plane_bsize, coeff_ctx, rate, dist, bsse, skip); + vp10_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block, + plane_bsize, coeff_ctx, rate, dist, bsse, skip); if ((RDCOST(x->rdmult, x->rddiv, *rate, *dist) >= RDCOST(x->rdmult, x->rddiv, zero_blk_rate, *bsse) || *skip == 1) && @@ -2863,8 +2863,8 @@ static void tx_block_rd(const VP10_COMP *cpi, MACROBLOCK *x, break; } coeff_ctx = combine_entropy_contexts(ta[0], tl[0]); - tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block, - plane_bsize, coeff_ctx, rate, dist, bsse, skip); + vp10_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block, + plane_bsize, coeff_ctx, rate, dist, bsse, skip); for (i = 0; i < (1 << tx_size); ++i) { ta[i] = !(p->eobs[block] == 0); tl[i] = !(p->eobs[block] == 0); diff --git a/vp10/encoder/rdopt.h b/vp10/encoder/rdopt.h index 62b0aeaeb..a6394fa45 100644 --- a/vp10/encoder/rdopt.h +++ b/vp10/encoder/rdopt.h @@ -74,6 +74,13 @@ void vp10_rd_pick_inter_mode_sub8x8(struct VP10_COMP *cpi, int64_t best_rd_so_far); #if CONFIG_SUPERTX +#if CONFIG_VAR_TX +void vp10_tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, + int blk_row, int blk_col, int plane, int block, + int plane_bsize, int coeff_ctx, + int *rate, int64_t *dist, int64_t *bsse, int *skip); +#endif + void vp10_txfm_rd_in_plane_supertx(MACROBLOCK *x, #if CONFIG_VAR_TX const VP10_COMP *cpi, -- 2.40.0