From: hui su Date: Tue, 29 Sep 2015 01:38:00 +0000 (-0700) Subject: Extend ext_tx experiment to intra blocks X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=3fa0129cafb2162404bf64e1a580afbe3651e5fe;p=libvpx Extend ext_tx experiment to intra blocks ext-tx on derflr +2.30% (was +1.84%) Change-Id: Ic91565cacc38e7a8e1200d054ed7bf99295fe19e --- diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h index 715d2e6fd..d01067a32 100644 --- a/vp10/common/blockd.h +++ b/vp10/common/blockd.h @@ -269,11 +269,16 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, #if CONFIG_EXT_TX if (xd->lossless || tx_size >= TX_32X32) return DCT_DCT; - if (is_inter_block(&mi->mbmi)) { - return ext_tx_to_txtype[mi->mbmi.ext_txfrm]; + if (mbmi->sb_type >= BLOCK_8X8) { + if (plane_type == PLANE_TYPE_Y || is_inter_block(mbmi)) + return ext_tx_to_txtype[mbmi->ext_txfrm]; } - return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ? - get_y_mode(mi, block_idx) : mbmi->uv_mode]; + + if (is_inter_block(mbmi)) + return DCT_DCT; + else + return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ? + get_y_mode(mi, block_idx) : mbmi->uv_mode]; #else if (plane_type != PLANE_TYPE_Y || xd->lossless || is_inter_block(mbmi) || tx_size >= TX_32X32) diff --git a/vp10/common/entropymode.c b/vp10/common/entropymode.c index 5bbe045ef..8f59fd8f2 100644 --- a/vp10/common/entropymode.c +++ b/vp10/common/entropymode.c @@ -334,7 +334,8 @@ const vpx_tree_index vp10_ext_tx_tree[TREE_SIZE(EXT_TX_TYPES)] = { -ALT7, -ALT8 }; -static const vpx_prob default_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = { +static const vpx_prob +default_inter_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = { { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }, { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, @@ -342,6 +343,76 @@ static const vpx_prob default_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = { { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }, }; + +static const vpx_prob +default_intra_ext_tx_prob[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES - 1] = { + { + { 8, 11, 24, 112, 87, 137, 127, 134, + 128, 86, 128, 124, 125, 133, 176, 123, }, + { 10, 9, 39, 106, 73, 155, 163, 228, + 35, 62, 129, 127, 133, 114, 213, 234, }, + { 10, 9, 14, 88, 91, 127, 151, 51, + 210, 89, 126, 58, 52, 116, 217, 24, }, + { 9, 6, 29, 113, 98, 131, 149, 210, + 119, 60, 124, 93, 90, 143, 170, 197, }, + { 8, 8, 38, 101, 111, 166, 167, 141, + 130, 105, 128, 75, 75, 118, 197, 117, }, + { 7, 8, 39, 91, 101, 153, 166, 200, + 99, 77, 123, 90, 83, 144, 224, 192, }, + { 7, 10, 26, 86, 119, 154, 130, 101, + 152, 91, 129, 75, 79, 137, 219, 77, }, + { 10, 13, 20, 86, 102, 162, 112, 76, + 171, 86, 134, 122, 106, 124, 196, 44, }, + { 8, 9, 33, 108, 100, 144, 148, 215, + 77, 60, 125, 125, 128, 126, 198, 220, }, + { 3, 10, 29, 111, 69, 141, 204, 141, + 139, 93, 120, 75, 77, 163, 242, 124, }, + }, + { + { 2, 53, 18, 147, 96, 98, 136, 133, + 131, 120, 153, 163, 169, 137, 173, 124, }, + { 4, 18, 34, 133, 54, 130, 179, 228, + 28, 72, 153, 164, 168, 118, 227, 239, }, + { 4, 18, 13, 125, 72, 110, 176, 36, + 221, 104, 148, 75, 72, 117, 225, 19, }, + { 8, 33, 24, 162, 113, 99, 147, 226, + 103, 85, 153, 143, 153, 124, 155, 210, }, + { 2, 15, 35, 107, 127, 158, 192, 128, + 126, 116, 151, 95, 88, 182, 241, 119, }, + { 3, 15, 36, 112, 100, 146, 194, 189, + 90, 98, 152, 99, 100, 165, 235, 175, }, + { 3, 16, 29, 109, 103, 140, 182, 76, + 173, 104, 147, 82, 85, 159, 235, 70, }, + { 9, 24, 14, 120, 86, 156, 161, 34, + 177, 121, 142, 128, 128, 126, 185, 37, }, + { 5, 24, 29, 152, 98, 99, 174, 228, + 82, 76, 147, 149, 128, 132, 191, 225, }, + { 2, 15, 29, 111, 77, 126, 200, 135, + 117, 93, 152, 96, 84, 191, 245, 135, }, + }, + { + { 2, 69, 13, 173, 111, 69, 137, 159, + 159, 146, 151, 193, 203, 131, 180, 123, }, + { 1, 12, 33, 164, 32, 98, 204, 242, + 23, 99, 149, 215, 232, 110, 239, 245, }, + { 1, 17, 9, 136, 82, 83, 171, 28, + 231, 128, 135, 76, 64, 118, 235, 17, }, + { 4, 41, 17, 195, 131, 58, 161, 237, + 141, 97, 153, 189, 191, 117, 182, 202, }, + { 2, 17, 36, 104, 149, 137, 217, 139, + 191, 119, 125, 107, 115, 223, 249, 110, }, + { 2, 14, 24, 127, 91, 135, 219, 198, + 113, 91, 164, 125, 173, 211, 250, 116, }, + { 3, 19, 24, 120, 102, 130, 209, 81, + 187, 95, 143, 102, 50, 190, 244, 56, }, + { 4, 27, 10, 128, 91, 157, 181, 33, + 181, 150, 141, 141, 166, 114, 215, 25, }, + { 2, 34, 27, 187, 102, 77, 210, 245, + 113, 107, 136, 184, 188, 121, 210, 234, }, + { 1, 15, 22, 141, 59, 94, 208, 133, + 154, 95, 152, 112, 105, 191, 242, 111, }, + }, +}; #endif // CONFIG_EXT_TX static void init_mode_probs(FRAME_CONTEXT *fc) { @@ -357,7 +428,8 @@ static void init_mode_probs(FRAME_CONTEXT *fc) { vp10_copy(fc->skip_probs, default_skip_probs); vp10_copy(fc->inter_mode_probs, default_inter_mode_probs); #if CONFIG_EXT_TX - vp10_copy(fc->ext_tx_prob, default_ext_tx_prob); + vp10_copy(fc->inter_ext_tx_prob, default_inter_ext_tx_prob); + vp10_copy(fc->intra_ext_tx_prob, default_intra_ext_tx_prob); #endif // CONFIG_EXT_TX } @@ -441,8 +513,13 @@ void vp10_adapt_mode_probs(VP10_COMMON *cm) { #if CONFIG_EXT_TX for (i = TX_4X4; i <= TX_16X16; ++i) { - vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->ext_tx_prob[i], - counts->ext_tx[i], fc->ext_tx_prob[i]); + vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->inter_ext_tx_prob[i], + counts->inter_ext_tx[i], fc->inter_ext_tx_prob[i]); + + for (j = 0; j < INTRA_MODES; ++j) + vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->intra_ext_tx_prob[i][j], + counts->intra_ext_tx[i][j], + fc->intra_ext_tx_prob[i][j]); } #endif // CONFIG_EXT_TX } diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h index bdae9ee6f..edd9b3a0a 100644 --- a/vp10/common/entropymode.h +++ b/vp10/common/entropymode.h @@ -57,7 +57,8 @@ typedef struct frame_contexts { vpx_prob skip_probs[SKIP_CONTEXTS]; nmv_context nmvc; #if CONFIG_EXT_TX - vpx_prob ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1]; + vpx_prob inter_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1]; + vpx_prob intra_ext_tx_prob[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES - 1]; #endif // CONFIG_EXT_TX int initialized; } FRAME_CONTEXT; @@ -80,7 +81,8 @@ typedef struct FRAME_COUNTS { unsigned int skip[SKIP_CONTEXTS][2]; nmv_context_counts mv; #if CONFIG_EXT_TX - unsigned int ext_tx[EXT_TX_SIZES][EXT_TX_TYPES]; + unsigned int inter_ext_tx[EXT_TX_SIZES][EXT_TX_TYPES]; + unsigned int intra_ext_tx[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES]; #endif // CONFIG_EXT_TX } FRAME_COUNTS; diff --git a/vp10/decoder/decodeframe.c b/vp10/decoder/decodeframe.c index a5551299f..4797d32d2 100644 --- a/vp10/decoder/decodeframe.c +++ b/vp10/decoder/decodeframe.c @@ -2070,11 +2070,18 @@ static size_t read_uncompressed_header(VP10Decoder *pbi, #if CONFIG_EXT_TX static void read_ext_tx_probs(FRAME_CONTEXT *fc, vpx_reader *r) { - int i, j; + int i, j, k; + if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) { + for (i = TX_4X4; i <= TX_16X16; ++i) + for (j = 0; j < EXT_TX_TYPES - 1; ++j) + vp10_diff_update_prob(r, &fc->inter_ext_tx_prob[i][j]); + } + if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) { - for (j = TX_4X4; j <= TX_16X16; ++j) - for (i = 0; i < EXT_TX_TYPES - 1; ++i) - vp10_diff_update_prob(r, &fc->ext_tx_prob[j][i]); + for (i = TX_4X4; i <= TX_16X16; ++i) + for (j = 0; j < INTRA_MODES; ++j) + for (k = 0; k < EXT_TX_TYPES - 1; ++k) + vp10_diff_update_prob(r, &fc->intra_ext_tx_prob[i][j][k]); } } #endif // CONFIG_EXT_TX diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c index 4d8a1eba3..beb29a065 100644 --- a/vp10/decoder/decodemv.c +++ b/vp10/decoder/decodemv.c @@ -244,6 +244,18 @@ static void read_intra_frame_mode_info(VP10_COMMON *const cm, } mbmi->uv_mode = read_intra_mode(r, vp10_kf_uv_mode_prob[mbmi->mode]); + +#if CONFIG_EXT_TX + if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && + mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip && + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + mbmi->ext_txfrm = + vpx_read_tree(r, vp10_ext_tx_tree, + cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode]); + } else { + mbmi->ext_txfrm = NORM; + } +#endif // CONFIG_EXT_TX } static int read_mv_component(vpx_reader *r, @@ -595,28 +607,37 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi, mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r); inter_block = read_is_inter_block(cm, xd, mbmi->segment_id, r); mbmi->tx_size = read_tx_size(cm, xd, !mbmi->skip || !inter_block, r); + + if (inter_block) + read_inter_block_mode_info(pbi, xd, mi, mi_row, mi_col, r); + else + read_intra_block_mode_info(cm, xd, mi, r); + #if CONFIG_EXT_TX - if (inter_block && - mbmi->tx_size <= TX_16X16 && - cm->base_qindex > 0 && - mbmi->sb_type >= BLOCK_8X8 && - !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) && - !mbmi->skip) { + if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && + mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip && + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { FRAME_COUNTS *counts = xd->counts; - mbmi->ext_txfrm = vpx_read_tree(r, - vp10_ext_tx_tree, - cm->fc->ext_tx_prob[mbmi->tx_size]); - if (counts) - ++counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm]; + + if (inter_block) + mbmi->ext_txfrm = + vpx_read_tree(r, vp10_ext_tx_tree, + cm->fc->inter_ext_tx_prob[mbmi->tx_size]); + else + mbmi->ext_txfrm = + vpx_read_tree(r, vp10_ext_tx_tree, + cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode]); + + if (counts) { + if (inter_block) + ++counts->inter_ext_tx[mbmi->tx_size][mbmi->ext_txfrm]; + else + ++counts->intra_ext_tx[mbmi->tx_size][mbmi->mode][mbmi->ext_txfrm]; + } } else { mbmi->ext_txfrm = NORM; } #endif // CONFIG_EXT_TX - - if (inter_block) - read_inter_block_mode_info(pbi, xd, mi, mi_row, mi_col, r); - else - read_intra_block_mode_info(cm, xd, mi, r); } void vp10_read_mode_info(VP10Decoder *const pbi, MACROBLOCKD *xd, diff --git a/vp10/encoder/bitstream.c b/vp10/encoder/bitstream.c index 85057258f..f8c274e14 100644 --- a/vp10/encoder/bitstream.c +++ b/vp10/encoder/bitstream.c @@ -164,22 +164,39 @@ static void update_switchable_interp_probs(VP10_COMMON *cm, vpx_writer *w, static void update_ext_tx_probs(VP10_COMMON *cm, vpx_writer *w) { const int savings_thresh = vp10_cost_one(GROUP_DIFF_UPDATE_PROB) - vp10_cost_zero(GROUP_DIFF_UPDATE_PROB); - int i; + int i, j; int savings = 0; int do_update = 0; for (i = TX_4X4; i <= TX_16X16; ++i) { savings += prob_diff_update_savings( - vp10_ext_tx_tree, cm->fc->ext_tx_prob[i], - cm->counts.ext_tx[i], EXT_TX_TYPES); + vp10_ext_tx_tree, cm->fc->inter_ext_tx_prob[i], + cm->counts.inter_ext_tx[i], EXT_TX_TYPES); } do_update = savings > savings_thresh; vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB); if (do_update) { for (i = TX_4X4; i <= TX_16X16; ++i) { - prob_diff_update(vp10_ext_tx_tree, cm->fc->ext_tx_prob[i], - cm->counts.ext_tx[i], EXT_TX_TYPES, w); + prob_diff_update(vp10_ext_tx_tree, cm->fc->inter_ext_tx_prob[i], + cm->counts.inter_ext_tx[i], EXT_TX_TYPES, w); } } + + savings = 0; + do_update = 0; + + for (i = TX_4X4; i <= TX_16X16; ++i) + for (j = 0; j < INTRA_MODES; ++j) + savings += prob_diff_update_savings( + vp10_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j], + cm->counts.intra_ext_tx[i][j], EXT_TX_TYPES); + do_update = savings > savings_thresh; + vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB); + if (do_update) { + for (i = TX_4X4; i <= TX_16X16; ++i) + for (j = 0; j < INTRA_MODES; ++j) + prob_diff_update(vp10_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j], + cm->counts.intra_ext_tx[i][j], EXT_TX_TYPES, w); + } } #endif // CONFIG_EXT_TX @@ -337,18 +354,6 @@ static void pack_inter_mode_mvs(VP10_COMP *cpi, const MODE_INFO *mi, write_selected_tx_size(cm, xd, w); } -#if CONFIG_EXT_TX - if (is_inter && - mbmi->tx_size <= TX_16X16 && - cm->base_qindex > 0 && - bsize >= BLOCK_8X8 && - !mbmi->skip && - !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { - vp10_write_token(w, vp10_ext_tx_tree, cm->fc->ext_tx_prob[mbmi->tx_size], - &ext_tx_encodings[mbmi->ext_txfrm]); - } -#endif // CONFIG_EXT_TX - if (!is_inter) { if (bsize >= BLOCK_8X8) { write_intra_mode(w, mode, cm->fc->y_mode_prob[size_group_lookup[bsize]]); @@ -412,6 +417,21 @@ static void pack_inter_mode_mvs(VP10_COMP *cpi, const MODE_INFO *mi, } } } + +#if CONFIG_EXT_TX + if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && + bsize >= BLOCK_8X8 && !mbmi->skip && + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + if (is_inter) + vp10_write_token(w, vp10_ext_tx_tree, + cm->fc->inter_ext_tx_prob[mbmi->tx_size], + &ext_tx_encodings[mbmi->ext_txfrm]); + else + vp10_write_token(w, vp10_ext_tx_tree, + cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode], + &ext_tx_encodings[mbmi->ext_txfrm]); + } +#endif // CONFIG_EXT_TX } static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd, @@ -448,6 +468,16 @@ static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd, } write_intra_mode(w, mbmi->uv_mode, vp10_kf_uv_mode_prob[mbmi->mode]); + +#if CONFIG_EXT_TX + if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && + bsize >= BLOCK_8X8 && !mbmi->skip && + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + vp10_write_token(w, vp10_ext_tx_tree, + cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode], + &ext_tx_encodings[mbmi->ext_txfrm]); + } +#endif // CONFIG_EXT_TX } static void write_modes_b(VP10_COMP *cpi, const TileInfo *const tile, diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c index ceb9eb471..356baf586 100644 --- a/vp10/encoder/encodeframe.c +++ b/vp10/encoder/encodeframe.c @@ -2989,13 +2989,13 @@ static void encode_superblock(VP10_COMP *cpi, ThreadData *td, ++td->counts->tx.tx_totals[mbmi->tx_size]; ++td->counts->tx.tx_totals[get_uv_tx_size(mbmi, &xd->plane[1])]; #if CONFIG_EXT_TX - if (is_inter_block(mbmi) && - mbmi->tx_size <= TX_16X16 && - cm->base_qindex > 0 && - bsize >= BLOCK_8X8 && - !mbmi->skip && + if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && + bsize >= BLOCK_8X8 && !mbmi->skip && !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { - ++td->counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm]; + if (is_inter_block(mbmi)) + ++td->counts->inter_ext_tx[mbmi->tx_size][mbmi->ext_txfrm]; + else + ++td->counts->intra_ext_tx[mbmi->tx_size][mbmi->mode][mbmi->ext_txfrm]; } #endif // CONFIG_EXT_TX } diff --git a/vp10/encoder/encoder.h b/vp10/encoder/encoder.h index e9779103b..0f8076db0 100644 --- a/vp10/encoder/encoder.h +++ b/vp10/encoder/encoder.h @@ -459,7 +459,8 @@ typedef struct VP10_COMP { int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS]; int partition_cost[PARTITION_CONTEXTS][PARTITION_TYPES]; #if CONFIG_EXT_TX - int ext_tx_costs[EXT_TX_SIZES][EXT_TX_TYPES]; + int inter_ext_tx_costs[EXT_TX_SIZES][EXT_TX_TYPES]; + int intra_ext_tx_costs[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES]; #endif // CONFIG_EXT_TX int multi_arf_allowed; diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c index 82dfd2310..2b549af96 100644 --- a/vp10/encoder/rd.c +++ b/vp10/encoder/rd.c @@ -85,9 +85,13 @@ static void fill_mode_costs(VP10_COMP *cpi) { vp10_cost_tokens(cpi->switchable_interp_costs[i], fc->switchable_interp_prob[i], vp10_switchable_interp_tree); #if CONFIG_EXT_TX - for (i = TX_4X4; i <= TX_16X16; ++i) - vp10_cost_tokens(cpi->ext_tx_costs[i], fc->ext_tx_prob[i], + for (i = TX_4X4; i <= TX_16X16; ++i) { + vp10_cost_tokens(cpi->inter_ext_tx_costs[i], fc->inter_ext_tx_prob[i], vp10_ext_tx_tree); + for (j = 0; j < INTRA_MODES; ++j) + vp10_cost_tokens(cpi->intra_ext_tx_costs[i][j], + fc->intra_ext_tx_prob[i][j], vp10_ext_tx_tree); + } #endif // CONFIG_EXT_TX } diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index 1f1405e8e..bec1103eb 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -603,6 +603,11 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x, #if CONFIG_EXT_TX if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && !xd->lossless) { for (tx_type = NORM; tx_type < EXT_TX_TYPES - 1; ++tx_type) { + if (mbmi->ext_txfrm >= ALT11 && mbmi->ext_txfrm < ALT16 && + best_tx_type == NORM) { + tx_type = ALT16 - 1; + continue; + } if (tx_type >= GET_EXT_TX_TYPES(mbmi->tx_size)) continue; @@ -613,8 +618,13 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x, if (r == INT_MAX) continue; - if (mbmi->tx_size <= TX_16X16) - r += cpi->ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm]; + if (mbmi->tx_size <= TX_16X16) { + if (is_inter_block(mbmi)) + r += cpi->inter_ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm]; + else + r += cpi->intra_ext_tx_costs[mbmi->tx_size] + [mbmi->mode][mbmi->ext_txfrm]; + } if (s) this_rd = RDCOST(x->rdmult, x->rddiv, s1, psse); @@ -638,10 +648,14 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x, mbmi->tx_size, cpi->sf.use_fast_coef_costing); #if CONFIG_EXT_TX - if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && - mbmi->tx_size <= TX_16X16 && - !xd->lossless && *rate != INT_MAX) - *rate += cpi->ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm]; + if (bs >= BLOCK_8X8 && mbmi->tx_size <= TX_16X16 && + !xd->lossless && *rate != INT_MAX) { + if (is_inter_block(mbmi)) + *rate += cpi->inter_ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm]; + else + *rate += cpi->intra_ext_tx_costs[mbmi->tx_size] + [mbmi->mode][mbmi->ext_txfrm]; + } #endif // CONFIG_EXT_TX } @@ -693,7 +707,7 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, #if CONFIG_EXT_TX start_tx_type = NORM; - if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && !xd->lossless) + if (bs >= BLOCK_8X8 && !xd->lossless) end_tx_type = EXT_TX_TYPES - 1; else end_tx_type = NORM; @@ -726,9 +740,12 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, &sse, ref_best_rd, 0, bs, n, cpi->sf.use_fast_coef_costing); #if CONFIG_EXT_TX - if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && - !xd->lossless && r != INT_MAX && n < TX_32X32) - r += cpi->ext_tx_costs[n][mbmi->ext_txfrm]; + if (bs >= BLOCK_8X8 && !xd->lossless && r != INT_MAX && n < TX_32X32) { + if (is_inter_block(mbmi)) + r += cpi->inter_ext_tx_costs[n][mbmi->ext_txfrm]; + else + r += cpi->intra_ext_tx_costs[n][mbmi->mode][mbmi->ext_txfrm]; + } #endif // CONFIG_EXT_TX if (r == INT_MAX) @@ -1136,6 +1153,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x, int this_rate, this_rate_tokenonly, s; int64_t this_distortion, this_rd; TX_SIZE best_tx = TX_4X4; +#if CONFIG_EXT_TX + EXT_TX_TYPE best_tx_type = NORM; +#endif // CONFIG_EXT_TX int *bmode_costs; const MODE_INFO *above_mi = xd->above_mi; const MODE_INFO *left_mi = xd->left_mi; @@ -1161,6 +1181,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x, mode_selected = mode; best_rd = this_rd; best_tx = mic->mbmi.tx_size; +#if CONFIG_EXT_TX + best_tx_type = mic->mbmi.ext_txfrm; +#endif // CONFIG_EXT_TX *rate = this_rate; *rate_tokenonly = this_rate_tokenonly; *distortion = this_distortion; @@ -1170,6 +1193,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x, mic->mbmi.mode = mode_selected; mic->mbmi.tx_size = best_tx; +#if CONFIG_EXT_TX + mic->mbmi.ext_txfrm = best_tx_type; +#endif // CONFIG_EXT_TX return best_rd; } @@ -3295,9 +3321,6 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi, if (ref_frame == INTRA_FRAME) { TX_SIZE uv_tx; struct macroblockd_plane *const pd = &xd->plane[1]; -#if CONFIG_EXT_TX - mbmi->ext_txfrm = NORM; -#endif // CONFIG_EXT_TX memset(x->skip_txfm, 0, sizeof(x->skip_txfm)); super_block_yrd(cpi, x, &rate_y, &distortion_y, &skippable, NULL, bsize, best_rd); @@ -3322,9 +3345,6 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi, rate2 += intra_cost_penalty; distortion2 = distortion_y + distortion_uv; } else { -#if CONFIG_EXT_TX - mbmi->ext_txfrm = NORM; -#endif this_rd = handle_inter_mode(cpi, x, bsize, &rate2, &distortion2, &skippable, &rate_y, &rate_uv, @@ -3755,9 +3775,6 @@ void vp10_rd_pick_inter_mode_sub8x8(VP10_COMP *cpi, int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS]; int internal_active_edge = vp10_active_edge_sb(cpi, mi_row, mi_col) && vp10_internal_image_edge(cpi); -#if CONFIG_EXT_TX - mbmi->ext_txfrm = NORM; -#endif memset(x->zcoeff_blk[TX_4X4], 0, 4); vp10_zero(best_mbmode); @@ -3916,9 +3933,6 @@ void vp10_rd_pick_inter_mode_sub8x8(VP10_COMP *cpi, if (ref_frame == INTRA_FRAME) { int rate; -#if CONFIG_EXT_TX - mbmi->ext_txfrm = NORM; -#endif if (rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate, &rate_y, &distortion_y, best_rd) >= best_rd) continue;