From: Debargha Mukherjee Date: Mon, 12 Oct 2015 19:30:55 +0000 (-0700) Subject: Refactoring tx-types to add more flexibility X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=8a4292441f874ee5f3afc4a1d767283d3081038e;p=libvpx Refactoring tx-types to add more flexibility Allows inter and intra tx_types to have different sets of transforms for different tx_size/sb_type combinations. Change-Id: Ic0ac1daef7a9fb15c4210271e4d04cd36e5cec8e --- diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h index 392b5d233..4bf06d186 100644 --- a/vp10/common/blockd.h +++ b/vp10/common/blockd.h @@ -32,11 +32,6 @@ extern "C" { #define MAX_MB_PLANE 3 -#if CONFIG_EXT_TX -#define GET_TX_TYPES(tx_size) \ - ((tx_size) >= TX_32X32 ? 1 : TX_TYPES) -#endif // CONFIG_EXT_TX - typedef enum { KEY_FRAME = 0, INTER_FRAME = 1, @@ -250,6 +245,58 @@ static const TX_TYPE intra_mode_to_tx_type_lookup[INTRA_MODES] = { ADST_ADST, // TM }; +#if CONFIG_EXT_TX +static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = { + 1, 17, 10, 2 +}; +static const int num_ext_tx_set_intra[EXT_TX_SETS_INTRA] = { + 1, 17, +}; + +#define USE_IDTX_FOR_32X32 0 +static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, + int is_inter) { + (void) is_inter; + if (tx_size > TX_32X32 || bs < BLOCK_8X8) return 0; +#if USE_IDTX_FOR_32X32 + if (tx_size == TX_32X32) return is_inter ? 3 : 0; +#else + if (tx_size == TX_32X32) return 0; +#endif + return 1; +} + +static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, + int is_inter) { + const int set = get_ext_tx_set(tx_size, bs, is_inter); + return is_inter ? num_ext_tx_set_inter[set] : num_ext_tx_set_intra[set]; +} + +static const int use_intra_ext_tx_for_tx[EXT_TX_SETS_INTRA][TX_SIZES] = { + { 0, 0, 0, 0, }, // unused + { 1, 1, 1, 0, }, +}; + +static const int use_inter_ext_tx_for_tx[EXT_TX_SETS_INTER][TX_SIZES] = { + { 0, 0, 0, 0, }, // unused + { 1, 1, 1, 0, }, + { 0, 0, 0, 0, }, + { 0, 0, 0, USE_IDTX_FOR_32X32, }, +}; + +static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = { + { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }, + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }, +}; + +static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = { + { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }, + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }, + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, }, + { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, }, +}; +#endif // CONFIG_EXT_TX + static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd, int block_idx, TX_SIZE tx_size) { @@ -257,16 +304,27 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, const MB_MODE_INFO *const mbmi = &mi->mbmi; #if CONFIG_EXT_TX - if ( xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32) +#if USE_IDTX_FOR_32X32 + if (xd->lossless[mbmi->segment_id] || tx_size > TX_32X32 || + (tx_size >= TX_32X32 && !is_inter_block(mbmi))) +#else + if (xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32) +#endif return DCT_DCT; if (mbmi->sb_type >= BLOCK_8X8) { - if (plane_type == PLANE_TYPE_Y || is_inter_block(mbmi)) + if (plane_type == PLANE_TYPE_Y) return mbmi->tx_type; + if (is_inter_block(mbmi)) + // UV Inter only + return (mbmi->tx_type == IDTX && tx_size == TX_32X32 ? + DCT_DCT : mbmi->tx_type); + // return mbmi->tx_type; } - if (is_inter_block(mbmi)) + // Sub8x8-Inter/Intra OR UV-Intra + if (is_inter_block(mbmi)) // Sub8x8-Inter return DCT_DCT; - else + else // Sub8x8 Intra OR UV-Intra return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ? get_y_mode(mi, block_idx) : mbmi->uv_mode]; #else diff --git a/vp10/common/entropymode.c b/vp10/common/entropymode.c index ccfd6de4a..59e6df8f3 100644 --- a/vp10/common/entropymode.c +++ b/vp10/common/entropymode.c @@ -755,103 +755,190 @@ static const vpx_prob default_switchable_interp_prob[SWITCHABLE_FILTER_CONTEXTS] }; #if CONFIG_EXT_TX -const vpx_tree_index vp10_tx_type_tree[TREE_SIZE(TX_TYPES)] = { - -IDTX, 2, - -DCT_DCT, 4, - -DST_DST, 6, - 8, 18, - 10, 12, - -DST_DCT, -DCT_DST, - 14, 16, - -ADST_DCT, -DCT_ADST, - -FLIPADST_DCT, -DCT_FLIPADST, - 20, 26, - 22, 24, - -DST_ADST, -ADST_DST, - -DST_FLIPADST, -FLIPADST_DST, - 28, 30, - -ADST_ADST, -FLIPADST_FLIPADST, - -ADST_FLIPADST, -FLIPADST_ADST +const vpx_tree_index vp10_ext_tx_inter_tree[EXT_TX_SETS_INTER] + [TREE_SIZE(TX_TYPES)] = { + { + }, { + -IDTX, 2, + -DCT_DCT, 4, + -DST_DST, 6, + 8, 18, + 10, 12, + -DST_DCT, -DCT_DST, + 14, 16, + -ADST_DCT, -DCT_ADST, + -FLIPADST_DCT, -DCT_FLIPADST, + 20, 26, + 22, 24, + -DST_ADST, -ADST_DST, + -DST_FLIPADST, -FLIPADST_DST, + 28, 30, + -ADST_ADST, -FLIPADST_FLIPADST, + -ADST_FLIPADST, -FLIPADST_ADST, + }, { + -IDTX, 2, + -DCT_DCT, 4, + 6, 12, + 8, 10, + -ADST_DCT, -DCT_ADST, + -FLIPADST_DCT, -DCT_FLIPADST, + 14, 16, + -ADST_ADST, -FLIPADST_FLIPADST, + -ADST_FLIPADST, -FLIPADST_ADST + }, { + -IDTX, -DCT_DCT, + } +}; + +const vpx_tree_index vp10_ext_tx_intra_tree[EXT_TX_SETS_INTRA] + [TREE_SIZE(TX_TYPES)] = { + { + }, { + -IDTX, 2, + -DCT_DCT, 4, + -DST_DST, 6, + 8, 18, + 10, 12, + -DST_DCT, -DCT_DST, + 14, 16, + -ADST_DCT, -DCT_ADST, + -FLIPADST_DCT, -DCT_FLIPADST, + 20, 26, + 22, 24, + -DST_ADST, -ADST_DST, + -DST_FLIPADST, -FLIPADST_DST, + 28, 30, + -ADST_ADST, -FLIPADST_FLIPADST, + -ADST_FLIPADST, -FLIPADST_ADST, + } }; static const vpx_prob -default_inter_tx_type_prob[EXT_TX_SIZES][TX_TYPES - 1] = { - { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128 }, - { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128 }, - { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128 }, +default_inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1] = { + { + // unused + }, { + { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128 }, + { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128 }, + { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128 }, +#if EXT_TX_SIZES == 4 + { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, + 128 }, +#endif + }, { + { 12, 112, 128, 128, 128, 128, 128, 128, 128 }, + { 12, 112, 128, 128, 128, 128, 128, 128, 128 }, + { 12, 112, 128, 128, 128, 128, 128, 128, 128 }, +#if EXT_TX_SIZES == 4 + { 12, 112, 128, 128, 128, 128, 128, 128, 128 }, +#endif + }, { + { 12, }, + { 12, }, + { 12, }, +#if EXT_TX_SIZES == 4 + { 12, }, +#endif + } }; static const vpx_prob -default_intra_tx_type_prob[EXT_TX_SIZES][INTRA_MODES][TX_TYPES - 1] = { - { - { 8, 11, 24, 112, 87, 137, 127, 134, - 128, 86, 128, 124, 125, 133, 176, 123, }, - { 10, 9, 39, 106, 73, 155, 163, 228, - 35, 62, 129, 127, 133, 114, 213, 234, }, - { 10, 9, 14, 88, 91, 127, 151, 51, - 210, 89, 126, 58, 52, 116, 217, 24, }, - { 9, 6, 29, 113, 98, 131, 149, 210, - 119, 60, 124, 93, 90, 143, 170, 197, }, - { 8, 8, 38, 101, 111, 166, 167, 141, - 130, 105, 128, 75, 75, 118, 197, 117, }, - { 7, 8, 39, 91, 101, 153, 166, 200, - 99, 77, 123, 90, 83, 144, 224, 192, }, - { 7, 10, 26, 86, 119, 154, 130, 101, - 152, 91, 129, 75, 79, 137, 219, 77, }, - { 10, 13, 20, 86, 102, 162, 112, 76, - 171, 86, 134, 122, 106, 124, 196, 44, }, - { 8, 9, 33, 108, 100, 144, 148, 215, - 77, 60, 125, 125, 128, 126, 198, 220, }, - { 3, 10, 29, 111, 69, 141, 204, 141, - 139, 93, 120, 75, 77, 163, 242, 124, }, - }, +default_intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES] + [INTRA_MODES][TX_TYPES - 1] = { + { + // unused + }, { { - { 2, 53, 18, 147, 96, 98, 136, 133, - 131, 120, 153, 163, 169, 137, 173, 124, }, - { 4, 18, 34, 133, 54, 130, 179, 228, - 28, 72, 153, 164, 168, 118, 227, 239, }, - { 4, 18, 13, 125, 72, 110, 176, 36, - 221, 104, 148, 75, 72, 117, 225, 19, }, - { 8, 33, 24, 162, 113, 99, 147, 226, - 103, 85, 153, 143, 153, 124, 155, 210, }, - { 2, 15, 35, 107, 127, 158, 192, 128, - 126, 116, 151, 95, 88, 182, 241, 119, }, - { 3, 15, 36, 112, 100, 146, 194, 189, - 90, 98, 152, 99, 100, 165, 235, 175, }, - { 3, 16, 29, 109, 103, 140, 182, 76, - 173, 104, 147, 82, 85, 159, 235, 70, }, - { 9, 24, 14, 120, 86, 156, 161, 34, - 177, 121, 142, 128, 128, 126, 185, 37, }, - { 5, 24, 29, 152, 98, 99, 174, 228, - 82, 76, 147, 149, 128, 132, 191, 225, }, - { 2, 15, 29, 111, 77, 126, 200, 135, - 117, 93, 152, 96, 84, 191, 245, 135, }, - }, - { - { 2, 69, 13, 173, 111, 69, 137, 159, - 159, 146, 151, 193, 203, 131, 180, 123, }, - { 1, 12, 33, 164, 32, 98, 204, 242, - 23, 99, 149, 215, 232, 110, 239, 245, }, - { 1, 17, 9, 136, 82, 83, 171, 28, - 231, 128, 135, 76, 64, 118, 235, 17, }, - { 4, 41, 17, 195, 131, 58, 161, 237, - 141, 97, 153, 189, 191, 117, 182, 202, }, - { 2, 17, 36, 104, 149, 137, 217, 139, - 191, 119, 125, 107, 115, 223, 249, 110, }, - { 2, 14, 24, 127, 91, 135, 219, 198, - 113, 91, 164, 125, 173, 211, 250, 116, }, - { 3, 19, 24, 120, 102, 130, 209, 81, - 187, 95, 143, 102, 50, 190, 244, 56, }, - { 4, 27, 10, 128, 91, 157, 181, 33, - 181, 150, 141, 141, 166, 114, 215, 25, }, - { 2, 34, 27, 187, 102, 77, 210, 245, - 113, 107, 136, 184, 188, 121, 210, 234, }, - { 1, 15, 22, 141, 59, 94, 208, 133, - 154, 95, 152, 112, 105, 191, 242, 111, }, + { 8, 11, 24, 112, 87, 137, 127, 134, + 128, 86, 128, 124, 125, 133, 176, 123, }, + { 10, 9, 39, 106, 73, 155, 163, 228, + 35, 62, 129, 127, 133, 114, 213, 234, }, + { 10, 9, 14, 88, 91, 127, 151, 51, + 210, 89, 126, 58, 52, 116, 217, 24, }, + { 9, 6, 29, 113, 98, 131, 149, 210, + 119, 60, 124, 93, 90, 143, 170, 197, }, + { 8, 8, 38, 101, 111, 166, 167, 141, + 130, 105, 128, 75, 75, 118, 197, 117, }, + { 7, 8, 39, 91, 101, 153, 166, 200, + 99, 77, 123, 90, 83, 144, 224, 192, }, + { 7, 10, 26, 86, 119, 154, 130, 101, + 152, 91, 129, 75, 79, 137, 219, 77, }, + { 10, 13, 20, 86, 102, 162, 112, 76, + 171, 86, 134, 122, 106, 124, 196, 44, }, + { 8, 9, 33, 108, 100, 144, 148, 215, + 77, 60, 125, 125, 128, 126, 198, 220, }, + { 3, 10, 29, 111, 69, 141, 204, 141, + 139, 93, 120, 75, 77, 163, 242, 124, }, + }, { + { 2, 53, 18, 147, 96, 98, 136, 133, + 131, 120, 153, 163, 169, 137, 173, 124, }, + { 4, 18, 34, 133, 54, 130, 179, 228, + 28, 72, 153, 164, 168, 118, 227, 239, }, + { 4, 18, 13, 125, 72, 110, 176, 36, + 221, 104, 148, 75, 72, 117, 225, 19, }, + { 8, 33, 24, 162, 113, 99, 147, 226, + 103, 85, 153, 143, 153, 124, 155, 210, }, + { 2, 15, 35, 107, 127, 158, 192, 128, + 126, 116, 151, 95, 88, 182, 241, 119, }, + { 3, 15, 36, 112, 100, 146, 194, 189, + 90, 98, 152, 99, 100, 165, 235, 175, }, + { 3, 16, 29, 109, 103, 140, 182, 76, + 173, 104, 147, 82, 85, 159, 235, 70, }, + { 9, 24, 14, 120, 86, 156, 161, 34, + 177, 121, 142, 128, 128, 126, 185, 37, }, + { 5, 24, 29, 152, 98, 99, 174, 228, + 82, 76, 147, 149, 128, 132, 191, 225, }, + { 2, 15, 29, 111, 77, 126, 200, 135, + 117, 93, 152, 96, 84, 191, 245, 135, }, + }, { + { 2, 69, 13, 173, 111, 69, 137, 159, + 159, 146, 151, 193, 203, 131, 180, 123, }, + { 1, 12, 33, 164, 32, 98, 204, 242, + 23, 99, 149, 215, 232, 110, 239, 245, }, + { 1, 17, 9, 136, 82, 83, 171, 28, + 231, 128, 135, 76, 64, 118, 235, 17, }, + { 4, 41, 17, 195, 131, 58, 161, 237, + 141, 97, 153, 189, 191, 117, 182, 202, }, + { 2, 17, 36, 104, 149, 137, 217, 139, + 191, 119, 125, 107, 115, 223, 249, 110, }, + { 2, 14, 24, 127, 91, 135, 219, 198, + 113, 91, 164, 125, 173, 211, 250, 116, }, + { 3, 19, 24, 120, 102, 130, 209, 81, + 187, 95, 143, 102, 50, 190, 244, 56, }, + { 4, 27, 10, 128, 91, 157, 181, 33, + 181, 150, 141, 141, 166, 114, 215, 25, }, + { 2, 34, 27, 187, 102, 77, 210, 245, + 113, 107, 136, 184, 188, 121, 210, 234, }, + { 1, 15, 22, 141, 59, 94, 208, 133, + 154, 95, 152, 112, 105, 191, 242, 111, }, +#if EXT_TX_SIZES == 4 + }, { + { 2, 69, 13, 173, 111, 69, 137, 159, + 159, 146, 151, 193, 203, 131, 180, 123, }, + { 1, 12, 33, 164, 32, 98, 204, 242, + 23, 99, 149, 215, 232, 110, 239, 245, }, + { 1, 17, 9, 136, 82, 83, 171, 28, + 231, 128, 135, 76, 64, 118, 235, 17, }, + { 4, 41, 17, 195, 131, 58, 161, 237, + 141, 97, 153, 189, 191, 117, 182, 202, }, + { 2, 17, 36, 104, 149, 137, 217, 139, + 191, 119, 125, 107, 115, 223, 249, 110, }, + { 2, 14, 24, 127, 91, 135, 219, 198, + 113, 91, 164, 125, 173, 211, 250, 116, }, + { 3, 19, 24, 120, 102, 130, 209, 81, + 187, 95, 143, 102, 50, 190, 244, 56, }, + { 4, 27, 10, 128, 91, 157, 181, 33, + 181, 150, 141, 141, 166, 114, 215, 25, }, + { 2, 34, 27, 187, 102, 77, 210, 245, + 113, 107, 136, 184, 188, 121, 210, 234, }, + { 1, 15, 22, 141, 59, 94, 208, 133, + 154, 95, 152, 112, 105, 191, 242, 111, }, +#endif }, + } }; #endif // CONFIG_EXT_TX #if CONFIG_MISC_FIXES @@ -875,8 +962,8 @@ static void init_mode_probs(FRAME_CONTEXT *fc) { vp10_copy(fc->skip_probs, default_skip_probs); vp10_copy(fc->inter_mode_probs, default_inter_mode_probs); #if CONFIG_EXT_TX - vp10_copy(fc->inter_tx_type_prob, default_inter_tx_type_prob); - vp10_copy(fc->intra_tx_type_prob, default_intra_tx_type_prob); + vp10_copy(fc->inter_ext_tx_prob, default_inter_ext_tx_prob); + vp10_copy(fc->intra_ext_tx_prob, default_intra_ext_tx_prob); #endif // CONFIG_EXT_TX #if CONFIG_MISC_FIXES vp10_copy(fc->seg.tree_probs, default_seg_probs.tree_probs); @@ -885,7 +972,7 @@ static void init_mode_probs(FRAME_CONTEXT *fc) { } const vpx_tree_index vp10_switchable_interp_tree - [TREE_SIZE(SWITCHABLE_FILTERS)] = { +[TREE_SIZE(SWITCHABLE_FILTERS)] = { -EIGHTTAP, 2, -EIGHTTAP_SMOOTH, -EIGHTTAP_SHARP }; @@ -972,15 +1059,26 @@ void vp10_adapt_intra_frame_probs(VP10_COMMON *cm) { pre_fc->skip_probs[i], counts->skip[i]); #if CONFIG_EXT_TX - for (i = TX_4X4; i <= TX_16X16; ++i) { - int j; - vpx_tree_merge_probs(vp10_tx_type_tree, pre_fc->inter_tx_type_prob[i], - counts->inter_tx_type[i], fc->inter_tx_type_prob[i]); - - for (j = 0; j < INTRA_MODES; ++j) - vpx_tree_merge_probs(vp10_tx_type_tree, pre_fc->intra_tx_type_prob[i][j], - counts->intra_tx_type[i][j], - fc->intra_tx_type_prob[i][j]); + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + int s; + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + if (use_inter_ext_tx_for_tx[s][i]) { + vpx_tree_merge_probs(vp10_ext_tx_inter_tree[s], + pre_fc->inter_ext_tx_prob[s][i], + counts->inter_ext_tx[s][i], + fc->inter_ext_tx_prob[s][i]); + } + } + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + if (use_intra_ext_tx_for_tx[s][i]) { + int j; + for (j = 0; j < INTRA_MODES; ++j) + vpx_tree_merge_probs(vp10_ext_tx_intra_tree[s], + pre_fc->intra_ext_tx_prob[s][i][j], + counts->intra_ext_tx[s][i][j], + fc->intra_ext_tx_prob[s][i][j]); + } + } } #endif // CONFIG_EXT_TX #if CONFIG_MISC_FIXES diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h index 3780b4874..b53b4e157 100644 --- a/vp10/common/entropymode.h +++ b/vp10/common/entropymode.h @@ -70,8 +70,9 @@ typedef struct frame_contexts { vpx_prob skip_probs[SKIP_CONTEXTS]; nmv_context nmvc; #if CONFIG_EXT_TX - vpx_prob inter_tx_type_prob[EXT_TX_SIZES][TX_TYPES - 1]; - vpx_prob intra_tx_type_prob[EXT_TX_SIZES][INTRA_MODES][TX_TYPES - 1]; + vpx_prob inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1]; + vpx_prob intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES] + [TX_TYPES - 1]; #endif // CONFIG_EXT_TX #if CONFIG_MISC_FIXES struct segmentation_probs seg; @@ -98,8 +99,9 @@ typedef struct FRAME_COUNTS { unsigned int skip[SKIP_CONTEXTS][2]; nmv_context_counts mv; #if CONFIG_EXT_TX - unsigned int inter_tx_type[EXT_TX_SIZES][TX_TYPES]; - unsigned int intra_tx_type[EXT_TX_SIZES][INTRA_MODES][TX_TYPES]; + unsigned int inter_ext_tx[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES]; + unsigned int intra_ext_tx[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES] + [TX_TYPES]; #endif // CONFIG_EXT_TX #if CONFIG_MISC_FIXES struct seg_counts seg; @@ -147,7 +149,10 @@ void vp10_tx_counts_to_branch_counts_8x8(const unsigned int *tx_count_8x8p, unsigned int (*ct_8x8p)[2]); #if CONFIG_EXT_TX -extern const vpx_tree_index vp10_tx_type_tree[TREE_SIZE(TX_TYPES)]; +extern const vpx_tree_index + vp10_ext_tx_inter_tree[EXT_TX_SETS_INTER][TREE_SIZE(TX_TYPES)]; +extern const vpx_tree_index + vp10_ext_tx_intra_tree[EXT_TX_SETS_INTRA][TREE_SIZE(TX_TYPES)]; #endif // CONFIG_EXT_TX static INLINE int vp10_ceil_log2(int n) { diff --git a/vp10/common/enums.h b/vp10/common/enums.h index 36fa591a4..33c5eaf20 100644 --- a/vp10/common/enums.h +++ b/vp10/common/enums.h @@ -113,7 +113,12 @@ typedef enum { } TX_TYPE; #if CONFIG_EXT_TX -#define EXT_TX_SIZES 3 // number of sizes that use extended transforms +#define EXT_TX_SIZES 3 // number of sizes that use extended transforms +#define EXT_TX_SETS_INTER 4 // Sets of transform selections for INTER +#define EXT_TX_SETS_INTRA 2 // Sets of transform selections for INTRA + +#define TX_TYPES_10 10 +#define TX_TYPES_2 2 #endif // CONFIG_EXT_TX typedef enum { diff --git a/vp10/common/idct.c b/vp10/common/idct.c index 6f8df8e74..94e75643b 100644 --- a/vp10/common/idct.c +++ b/vp10/common/idct.c @@ -13,6 +13,7 @@ #include "./vp10_rtcd.h" #include "./vpx_dsp_rtcd.h" #include "vp10/common/blockd.h" +#include "vp10/common/enums.h" #include "vp10/common/idct.h" #include "vpx_dsp/inv_txfm.h" #include "vpx_ports/mem.h" @@ -864,6 +865,11 @@ void vp10_inv_txfm_add_32x32(const tran_low_t *input, uint8_t *dest, case DCT_DCT: vp10_idct32x32_add(input, dest, stride, eob); break; +#if CONFIG_EXT_TX + case IDTX: + inv_idtx_add_c(input, dest, stride, 32); + break; +#endif // CONFIG_EXT_TX case ADST_DCT: case DCT_ADST: case ADST_ADST: @@ -1295,6 +1301,11 @@ void vp10_highbd_inv_txfm_add_32x32(const tran_low_t *input, uint8_t *dest, case DCT_DCT: vp10_highbd_idct32x32_add(input, dest, stride, eob, bd); break; +#if CONFIG_EXT_TX + case IDTX: + highbd_inv_idtx_add_c(input, dest, stride, 32, bd); + break; +#endif // CONFIG_EXT_TX case ADST_DCT: case DCT_ADST: case ADST_ADST: diff --git a/vp10/common/thread_common.c b/vp10/common/thread_common.c index 5523abbbd..7508195f3 100644 --- a/vp10/common/thread_common.c +++ b/vp10/common/thread_common.c @@ -436,15 +436,25 @@ void vp10_accumulate_frame_counts(VP10_COMMON *cm, FRAME_COUNTS *counts, } #if CONFIG_EXT_TX - for (i = 0; i < EXT_TX_SIZES; i++) - for (j = 0; j < TX_TYPES; j++) - cm->counts.inter_tx_type[i][j] += counts->inter_tx_type[i][j]; - - for (i = 0; i < EXT_TX_SIZES; i++) - for (j = 0; j < INTRA_MODES; j++) - for (k = 0; k < TX_TYPES; k++) - cm->counts.intra_tx_type[i][j][k] += counts->intra_tx_type[i][j][k]; -#endif + for (i = 0; i < EXT_TX_SIZES; i++) { + int s, k; + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + if (use_inter_ext_tx_for_tx[s][i]) { + for (k = 0; k < TX_TYPES; k++) + cm->counts.inter_ext_tx[s][i][k] += counts->inter_ext_tx[s][i][k]; + } + } + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + if (use_intra_ext_tx_for_tx[s][i]) { + int j; + for (j = 0; j < INTRA_MODES; ++j) + for (k = 0; k < TX_TYPES; k++) + cm->counts.intra_ext_tx[s][i][j][k] += + counts->intra_ext_tx[s][i][j][k]; + } + } + } +#endif // CONFIG_EXT_TX #if CONFIG_MISC_FIXES for (i = 0; i < PREDICTION_PROBS; i++) diff --git a/vp10/decoder/decodeframe.c b/vp10/decoder/decodeframe.c index b3f59ff6d..eaa31f181 100644 --- a/vp10/decoder/decodeframe.c +++ b/vp10/decoder/decodeframe.c @@ -2224,17 +2224,26 @@ static size_t read_uncompressed_header(VP10Decoder *pbi, #if CONFIG_EXT_TX static void read_ext_tx_probs(FRAME_CONTEXT *fc, vpx_reader *r) { int i, j, k; - if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) { - for (i = TX_4X4; i <= TX_16X16; ++i) - for (j = 0; j < TX_TYPES - 1; ++j) - vp10_diff_update_prob(r, &fc->inter_tx_type_prob[i][j]); + int s; + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_inter_ext_tx_for_tx[s][i]) continue; + for (j = 0; j < num_ext_tx_set_inter[s] - 1; ++j) + vp10_diff_update_prob(r, &fc->inter_ext_tx_prob[s][i][j]); + } + } } - if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) { - for (i = TX_4X4; i <= TX_16X16; ++i) - for (j = 0; j < INTRA_MODES; ++j) - for (k = 0; k < TX_TYPES - 1; ++k) - vp10_diff_update_prob(r, &fc->intra_tx_type_prob[i][j][k]); + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_intra_ext_tx_for_tx[s][i]) continue; + for (j = 0; j < INTRA_MODES; ++j) + for (k = 0; k < num_ext_tx_set_intra[s] - 1; ++k) + vp10_diff_update_prob(r, &fc->intra_ext_tx_prob[s][i][j][k]); + } + } } } #endif // CONFIG_EXT_TX @@ -2365,11 +2374,11 @@ static void debug_check_frame_counts(const VP10_COMMON *const cm) { assert(!memcmp(&cm->counts.mv, &zero_counts.mv, sizeof(cm->counts.mv))); #if CONFIG_EXT_TX - assert(!memcmp(cm->counts.inter_tx_type, - zero_counts.inter_tx_type, sizeof(cm->counts.inter_tx_type))); - assert(!memcmp(cm->counts.intra_tx_type, - zero_counts.intra_tx_type, sizeof(cm->counts.intra_tx_type))); -#endif + assert(!memcmp(cm->counts.inter_ext_tx, zero_counts.inter_ext_tx, + sizeof(cm->counts.inter_ext_tx))); + assert(!memcmp(cm->counts.intra_ext_tx, zero_counts.intra_ext_tx, + sizeof(cm->counts.intra_ext_tx))); +#endif // CONFIG_EXT_TX } #endif // NDEBUG diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c index 907457470..803d0df9d 100644 --- a/vp10/decoder/decodemv.c +++ b/vp10/decoder/decodemv.c @@ -381,15 +381,19 @@ static void read_intra_frame_mode_info(VP10_COMMON *const cm, read_palette_mode_info(cm, xd, r); #if CONFIG_EXT_TX - if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && - mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip && + if (get_ext_tx_types(mbmi->tx_size, mbmi->sb_type, 0) > 1 && + cm->base_qindex > 0 && !mbmi->skip && !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { FRAME_COUNTS *counts = xd->counts; - mbmi->tx_type = - vpx_read_tree(r, vp10_tx_type_tree, - cm->fc->intra_tx_type_prob[mbmi->tx_size][mbmi->mode]); - if (counts) - ++counts->intra_tx_type[mbmi->tx_size][mbmi->mode][mbmi->tx_type]; + int eset = get_ext_tx_set(mbmi->tx_size, mbmi->sb_type, 0); + if (eset > 0) { + mbmi->tx_type = vpx_read_tree( + r, vp10_ext_tx_intra_tree[eset], + cm->fc->intra_ext_tx_prob[eset][mbmi->tx_size][mbmi->mode]); + if (counts) + ++counts->intra_ext_tx[eset][mbmi->tx_size][mbmi->mode] + [mbmi->tx_type]; + } } else { mbmi->tx_type = DCT_DCT; } @@ -790,25 +794,30 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi, read_intra_block_mode_info(cm, xd, mi, r); #if CONFIG_EXT_TX - if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && - mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip && + if (get_ext_tx_types(mbmi->tx_size, mbmi->sb_type, inter_block) > 1 && + cm->base_qindex > 0 && !mbmi->skip && !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { + int eset = get_ext_tx_set(mbmi->tx_size, mbmi->sb_type, + inter_block); FRAME_COUNTS *counts = xd->counts; - if (inter_block) - mbmi->tx_type = - vpx_read_tree(r, vp10_tx_type_tree, - cm->fc->inter_tx_type_prob[mbmi->tx_size]); - else - mbmi->tx_type = vpx_read_tree(r, vp10_tx_type_tree, - cm->fc->intra_tx_type_prob - [mbmi->tx_size][mbmi->mode]); - - if (counts) { - if (inter_block) - ++counts->inter_tx_type[mbmi->tx_size][mbmi->tx_type]; - else - ++counts->intra_tx_type[mbmi->tx_size][mbmi->mode][mbmi->tx_type]; + if (inter_block) { + if (eset > 0) { + mbmi->tx_type = + vpx_read_tree(r, vp10_ext_tx_inter_tree[eset], + cm->fc->inter_ext_tx_prob[eset][mbmi->tx_size]); + if (counts) + ++counts->inter_ext_tx[eset][mbmi->tx_size][mbmi->tx_type]; + } + } else { + if (eset > 0) { + mbmi->tx_type = vpx_read_tree(r, vp10_ext_tx_intra_tree[eset], + cm->fc->intra_ext_tx_prob[eset] + [mbmi->tx_size][mbmi->mode]); + if (counts) + ++counts->intra_ext_tx[eset][mbmi->tx_size] + [mbmi->mode][mbmi->tx_type]; + } } } else { mbmi->tx_type = DCT_DCT; diff --git a/vp10/encoder/bitstream.c b/vp10/encoder/bitstream.c index b6e827d06..e4dcc00aa 100644 --- a/vp10/encoder/bitstream.c +++ b/vp10/encoder/bitstream.c @@ -73,12 +73,19 @@ static INLINE void write_uniform(vpx_writer *w, int n, int v) { } #if CONFIG_EXT_TX -static struct vp10_token tx_type_encodings[TX_TYPES]; +static struct vp10_token ext_tx_inter_encodings[EXT_TX_SETS_INTER][TX_TYPES]; +static struct vp10_token ext_tx_intra_encodings[EXT_TX_SETS_INTRA][TX_TYPES]; #endif // CONFIG_EXT_TX void vp10_encode_token_init() { #if CONFIG_EXT_TX - vp10_tokens_from_tree(tx_type_encodings, vp10_tx_type_tree); + int s; + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + vp10_tokens_from_tree(ext_tx_inter_encodings[s], vp10_ext_tx_inter_tree[s]); + } + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + vp10_tokens_from_tree(ext_tx_intra_encodings[s], vp10_ext_tx_intra_tree[s]); + } #endif // CONFIG_EXT_TX } @@ -231,37 +238,51 @@ static void update_ext_tx_probs(VP10_COMMON *cm, vpx_writer *w) { const int savings_thresh = vp10_cost_one(GROUP_DIFF_UPDATE_PROB) - vp10_cost_zero(GROUP_DIFF_UPDATE_PROB); int i, j; - int savings = 0; - int do_update = 0; - for (i = TX_4X4; i <= TX_16X16; ++i) { - savings += prob_diff_update_savings( - vp10_tx_type_tree, cm->fc->inter_tx_type_prob[i], - cm->counts.inter_tx_type[i], TX_TYPES); - } - do_update = savings > savings_thresh; - vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB); - if (do_update) { - for (i = TX_4X4; i <= TX_16X16; ++i) { - prob_diff_update(vp10_tx_type_tree, cm->fc->inter_tx_type_prob[i], - cm->counts.inter_tx_type[i], TX_TYPES, w); + int s; + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + int savings = 0; + int do_update = 0; + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_inter_ext_tx_for_tx[s][i]) continue; + savings += prob_diff_update_savings( + vp10_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i], + cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s]); + } + do_update = savings > savings_thresh; + vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB); + if (do_update) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_inter_ext_tx_for_tx[s][i]) continue; + prob_diff_update(vp10_ext_tx_inter_tree[s], + cm->fc->inter_ext_tx_prob[s][i], + cm->counts.inter_ext_tx[s][i], + num_ext_tx_set_inter[s], w); + } } } - savings = 0; - do_update = 0; - - for (i = TX_4X4; i <= TX_16X16; ++i) - for (j = 0; j < INTRA_MODES; ++j) - savings += prob_diff_update_savings( - vp10_tx_type_tree, cm->fc->intra_tx_type_prob[i][j], - cm->counts.intra_tx_type[i][j], TX_TYPES); - do_update = savings > savings_thresh; - vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB); - if (do_update) { - for (i = TX_4X4; i <= TX_16X16; ++i) + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + int savings = 0; + int do_update = 0; + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_intra_ext_tx_for_tx[s][i]) continue; for (j = 0; j < INTRA_MODES; ++j) - prob_diff_update(vp10_tx_type_tree, cm->fc->intra_tx_type_prob[i][j], - cm->counts.intra_tx_type[i][j], TX_TYPES, w); + savings += prob_diff_update_savings( + vp10_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j], + cm->counts.intra_ext_tx[s][i][j], num_ext_tx_set_intra[s]); + } + do_update = savings > savings_thresh; + vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB); + if (do_update) { + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + if (!use_intra_ext_tx_for_tx[s][i]) continue; + for (j = 0; j < INTRA_MODES; ++j) + prob_diff_update(vp10_ext_tx_intra_tree[s], + cm->fc->intra_ext_tx_prob[s][i][j], + cm->counts.intra_ext_tx[s][i][j], + num_ext_tx_set_intra[s], w); + } + } } } #endif // CONFIG_EXT_TX @@ -598,17 +619,22 @@ static void pack_inter_mode_mvs(VP10_COMP *cpi, const MODE_INFO *mi, } #if CONFIG_EXT_TX - if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && - bsize >= BLOCK_8X8 && !mbmi->skip && + if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter) > 1 && + cm->base_qindex > 0 && !mbmi->skip && !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { - if (is_inter) - vp10_write_token(w, vp10_tx_type_tree, - cm->fc->inter_tx_type_prob[mbmi->tx_size], - &tx_type_encodings[mbmi->tx_type]); - else - vp10_write_token(w, vp10_tx_type_tree, - cm->fc->intra_tx_type_prob[mbmi->tx_size][mbmi->mode], - &tx_type_encodings[mbmi->tx_type]); + int eset = get_ext_tx_set(mbmi->tx_size, bsize, is_inter); + if (is_inter) { + if (eset > 0) + vp10_write_token(w, vp10_ext_tx_inter_tree[eset], + cm->fc->inter_ext_tx_prob[eset][mbmi->tx_size], + &ext_tx_inter_encodings[eset][mbmi->tx_type]); + } else { + if (eset > 0) + vp10_write_token( + w, vp10_ext_tx_intra_tree[eset], + cm->fc->intra_ext_tx_prob[eset][mbmi->tx_size][mbmi->mode], + &ext_tx_intra_encodings[eset][mbmi->tx_type]); + } } #endif // CONFIG_EXT_TX } @@ -690,12 +716,15 @@ static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd, #if CONFIG_EXT_TX - if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && - bsize >= BLOCK_8X8 && !mbmi->skip && + if (get_ext_tx_types(mbmi->tx_size, bsize, 0) > 1 && + cm->base_qindex > 0 && !mbmi->skip && !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { - vp10_write_token(w, vp10_tx_type_tree, - cm->fc->intra_tx_type_prob[mbmi->tx_size][mbmi->mode], - &tx_type_encodings[mbmi->tx_type]); + int eset = get_ext_tx_set(mbmi->tx_size, bsize, 0); + if (eset > 0) + vp10_write_token( + w, vp10_ext_tx_intra_tree[eset], + cm->fc->intra_ext_tx_prob[eset][mbmi->tx_size][mbmi->mode], + &ext_tx_intra_encodings[eset][mbmi->tx_type]); } #endif // CONFIG_EXT_TX } diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c index 095c6c28e..ea09f6e0b 100644 --- a/vp10/encoder/encodeframe.c +++ b/vp10/encoder/encodeframe.c @@ -2430,7 +2430,6 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td, (void) best_rd; *rd_cost = best_rdc; - if (best_rdc.rate < INT_MAX && best_rdc.dist < INT64_MAX && pc_tree->index != 3) { int output_enabled = (bsize == BLOCK_64X64); @@ -3057,13 +3056,19 @@ static void encode_superblock(VP10_COMP *cpi, ThreadData *td, ++td->counts->tx.tx_totals[mbmi->tx_size]; ++td->counts->tx.tx_totals[get_uv_tx_size(mbmi, &xd->plane[1])]; #if CONFIG_EXT_TX - if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 && - bsize >= BLOCK_8X8 && !mbmi->skip && + if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter_block(mbmi)) > 1 && + cm->base_qindex > 0 && !mbmi->skip && !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) { - if (is_inter_block(mbmi)) - ++td->counts->inter_tx_type[mbmi->tx_size][mbmi->tx_type]; - else - ++td->counts->intra_tx_type[mbmi->tx_size][mbmi->mode][mbmi->tx_type]; + int eset = get_ext_tx_set(mbmi->tx_size, bsize, + is_inter_block(mbmi)); + if (eset > 0) { + if (is_inter_block(mbmi)) { + ++td->counts->inter_ext_tx[eset][mbmi->tx_size][mbmi->tx_type]; + } else { + ++td->counts->intra_ext_tx[eset][mbmi->tx_size][mbmi->mode] + [mbmi->tx_type]; + } + } } #endif // CONFIG_EXT_TX } diff --git a/vp10/encoder/encodemb.c b/vp10/encoder/encodemb.c index 7a31dc78f..966d05c4d 100644 --- a/vp10/encoder/encodemb.c +++ b/vp10/encoder/encodemb.c @@ -709,6 +709,11 @@ static void fwd_txfm_32x32(int rd_transform, const int16_t *src_diff, case DCT_DCT: fdct32x32(rd_transform, src_diff, coeff, diff_stride); break; +#if CONFIG_EXT_TX + case IDTX: + fwd_idtx_c(src_diff, coeff, diff_stride, 32); + break; +#endif // CONFIG_EXT_TX case ADST_DCT: case DCT_ADST: case ADST_ADST: @@ -727,6 +732,11 @@ static void fwd_txfm_32x32_1(const int16_t *src_diff, case DCT_DCT: vpx_fdct32x32_1(src_diff, coeff, diff_stride); break; +#if CONFIG_EXT_TX + case IDTX: + fwd_idtx_c(src_diff, coeff, diff_stride, 32); + break; +#endif // CONFIG_EXT_TX case ADST_DCT: case DCT_ADST: case ADST_ADST: @@ -1056,6 +1066,11 @@ static void highbd_fwd_txfm_32x32(int rd_transform, const int16_t *src_diff, case DCT_DCT: highbd_fdct32x32(rd_transform, src_diff, coeff, diff_stride); break; +#if CONFIG_EXT_TX + case IDTX: + fwd_idtx_c(src_diff, coeff, diff_stride, 32); + break; +#endif // CONFIG_EXT_TX case ADST_DCT: case DCT_ADST: case ADST_ADST: @@ -1074,6 +1089,11 @@ static void highbd_fwd_txfm_32x32_1(const int16_t *src_diff, case DCT_DCT: vpx_highbd_fdct32x32_1(src_diff, coeff, diff_stride); break; +#if CONFIG_EXT_TX + case IDTX: + fwd_idtx_c(src_diff, coeff, diff_stride, 32); + break; +#endif // CONFIG_EXT_TX case ADST_DCT: case DCT_ADST: case ADST_ADST: diff --git a/vp10/encoder/encoder.h b/vp10/encoder/encoder.h index d399932f1..d63c123d3 100644 --- a/vp10/encoder/encoder.h +++ b/vp10/encoder/encoder.h @@ -467,8 +467,9 @@ typedef struct VP10_COMP { int palette_uv_color_cost[PALETTE_MAX_SIZE - 1][PALETTE_COLOR_CONTEXTS] [PALETTE_COLORS]; #if CONFIG_EXT_TX - int inter_tx_type_costs[EXT_TX_SIZES][TX_TYPES]; - int intra_tx_type_costs[EXT_TX_SIZES][INTRA_MODES][TX_TYPES]; + int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES]; + int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES] + [TX_TYPES]; #endif // CONFIG_EXT_TX int multi_arf_allowed; diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c index 963ff5071..817eb0aed 100644 --- a/vp10/encoder/rd.c +++ b/vp10/encoder/rd.c @@ -102,12 +102,23 @@ static void fill_mode_costs(VP10_COMP *cpi) { vp10_palette_color_tree[i]); } #if CONFIG_EXT_TX - for (i = TX_4X4; i <= TX_16X16; ++i) { - vp10_cost_tokens(cpi->inter_tx_type_costs[i], fc->inter_tx_type_prob[i], - vp10_tx_type_tree); - for (j = 0; j < INTRA_MODES; ++j) - vp10_cost_tokens(cpi->intra_tx_type_costs[i][j], - fc->intra_tx_type_prob[i][j], vp10_tx_type_tree); + for (i = TX_4X4; i < EXT_TX_SIZES; ++i) { + int s; + for (s = 1; s < EXT_TX_SETS_INTER; ++s) { + if (use_inter_ext_tx_for_tx[s][i]) { + vp10_cost_tokens(cpi->inter_tx_type_costs[s][i], + fc->inter_ext_tx_prob[s][i], + vp10_ext_tx_inter_tree[s]); + } + } + for (s = 1; s < EXT_TX_SETS_INTRA; ++s) { + if (use_intra_ext_tx_for_tx[s][i]) { + for (j = 0; j < INTRA_MODES; ++j) + vp10_cost_tokens(cpi->intra_tx_type_costs[s][i][j], + fc->intra_ext_tx_prob[s][i][j], + vp10_ext_tx_intra_tree[s]); + } + } } #endif // CONFIG_EXT_TX } diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c index 223c67ae2..fc7a107d6 100644 --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@ -668,28 +668,41 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x, MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; #if CONFIG_EXT_TX - int tx_type, best_tx_type = DCT_DCT; + TX_TYPE tx_type, best_tx_type = DCT_DCT; int r, s; int64_t d, psse, this_rd, best_rd = INT64_MAX; vpx_prob skip_prob = vp10_get_skip_prob(cm, xd); int s0 = vp10_cost_bit(skip_prob, 0); int s1 = vp10_cost_bit(skip_prob, 1); + int ext_tx_set; + const int is_inter = is_inter_block(mbmi); #endif // CONFIG_EXT_TX mbmi->tx_size = VPXMIN(max_tx_size, largest_tx_size); + #if CONFIG_EXT_TX - if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && + ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter); + + if (is_inter && + get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1 && !xd->lossless[mbmi->segment_id]) { - for (tx_type = DCT_DCT; tx_type < TX_TYPES - 1; ++tx_type) { - if (mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX && + for (tx_type = 0; tx_type < TX_TYPES; ++tx_type) { + if (is_inter) { + if (!ext_tx_used_inter[ext_tx_set][tx_type]) + continue; + } else { + if (!ext_tx_used_intra[ext_tx_set][tx_type]) + continue; + } + + mbmi->tx_type = tx_type; + if (ext_tx_set == 1 && + mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX && best_tx_type == DCT_DCT) { tx_type = IDTX - 1; continue; } - if (tx_type >= GET_TX_TYPES(mbmi->tx_size)) - continue; - mbmi->tx_type = tx_type; txfm_rd_in_plane(x, #if CONFIG_VAR_TX cpi, @@ -700,12 +713,16 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x, if (r == INT_MAX) continue; - if (mbmi->tx_size <= TX_16X16) { - if (is_inter_block(mbmi)) - r += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type]; - else - r += cpi->intra_tx_type_costs[mbmi->tx_size] - [mbmi->mode][mbmi->tx_type]; + if (get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1) { + if (is_inter) { + if (ext_tx_set > 0) + r += cpi->inter_tx_type_costs[ext_tx_set] + [mbmi->tx_size][mbmi->tx_type]; + } else { + if (ext_tx_set > 0) + r += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size] + [mbmi->mode][mbmi->tx_type]; + } } if (s) @@ -734,13 +751,15 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x, mbmi->tx_size, cpi->sf.use_fast_coef_costing); #if CONFIG_EXT_TX - if (bs >= BLOCK_8X8 && mbmi->tx_size <= TX_16X16 && + if (get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1 && !xd->lossless[mbmi->segment_id] && *rate != INT_MAX) { - if (is_inter_block(mbmi)) - *rate += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type]; + int ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter); + if (is_inter) + *rate += cpi->inter_tx_type_costs[ext_tx_set][mbmi->tx_size] + [mbmi->tx_type]; else - *rate += cpi->intra_tx_type_costs[mbmi->tx_size] - [mbmi->mode][mbmi->tx_type]; + *rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size] + [mbmi->mode][mbmi->tx_type]; } #endif // CONFIG_EXT_TX } @@ -786,9 +805,10 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, int start_tx, end_tx; const int tx_select = cm->tx_mode == TX_MODE_SELECT; #if CONFIG_EXT_TX - int tx_type, best_tx_type = DCT_DCT; - int start_tx_type, end_tx_type; + TX_TYPE tx_type, best_tx_type = DCT_DCT; + int ext_tx_set; #endif // CONFIG_EXT_TX + const int is_inter = is_inter_block(mbmi); const vpx_prob *tx_probs = get_tx_probs2(max_tx_size, xd, &cm->fc->tx_probs); assert(skip_prob > 0); @@ -799,8 +819,8 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, start_tx = max_tx_size; end_tx = 0; } else { - const TX_SIZE chosen_tx_size = VPXMIN(max_tx_size, - tx_mode_to_biggest_tx_size[cm->tx_mode]); + const TX_SIZE chosen_tx_size = + VPXMIN(max_tx_size, tx_mode_to_biggest_tx_size[cm->tx_mode]); start_tx = chosen_tx_size; end_tx = chosen_tx_size; } @@ -811,29 +831,11 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, *psse = INT64_MAX; #if CONFIG_EXT_TX - start_tx_type = DCT_DCT; - if (bs >= BLOCK_8X8 && !xd->lossless[mbmi->segment_id]) - end_tx_type = TX_TYPES - 1; - else - end_tx_type = DCT_DCT; - - for (tx_type = start_tx_type; tx_type <= end_tx_type; ++tx_type) { - mbmi->tx_type = tx_type; - // TODO(huisu): clean up the logic. - if (mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX && - best_tx_type == DCT_DCT) { - tx_type = IDTX - 1; - continue; - } + for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) { #endif // CONFIG_EXT_TX + last_rd = INT64_MAX; for (n = start_tx; n >= end_tx; --n) { int r_tx_size = 0; - -#if CONFIG_EXT_TX - if (mbmi->tx_type >= GET_TX_TYPES(n)) - continue; -#endif // CONFIG_EXT_TX - for (m = 0; m <= n - (n == (int) max_tx_size); ++m) { if (m == n) r_tx_size += vp10_cost_zero(tx_probs[m]); @@ -841,6 +843,22 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, r_tx_size += vp10_cost_one(tx_probs[m]); } +#if CONFIG_EXT_TX + ext_tx_set = get_ext_tx_set(n, bs, is_inter); + if (is_inter) { + if (!ext_tx_used_inter[ext_tx_set][tx_type]) + continue; + } else { + if (!ext_tx_used_intra[ext_tx_set][tx_type]) + continue; + } + mbmi->tx_type = tx_type; + if (ext_tx_set == 1 && + mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX && + best_tx_type == DCT_DCT) { + tx_type = IDTX - 1; + break; + } txfm_rd_in_plane(x, #if CONFIG_VAR_TX cpi, @@ -848,52 +866,65 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x, &r, &d, &s, &sse, ref_best_rd, 0, bs, n, cpi->sf.use_fast_coef_costing); -#if CONFIG_EXT_TX - if (bs >= BLOCK_8X8 && !xd->lossless[mbmi->segment_id] && - r != INT_MAX && n < TX_32X32) { - if (is_inter_block(mbmi)) - r += cpi->inter_tx_type_costs[n][mbmi->tx_type]; - else - r += cpi->intra_tx_type_costs[n][mbmi->mode][mbmi->tx_type]; + if (get_ext_tx_types(n, bs, is_inter) > 1 && + !xd->lossless[xd->mi[0]->mbmi.segment_id] && + r != INT_MAX) { + if (is_inter) { + if (ext_tx_set > 0) + r += cpi->inter_tx_type_costs[ext_tx_set] + [mbmi->tx_size][mbmi->tx_type]; + } else { + if (ext_tx_set > 0) + r += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size] + [mbmi->mode][mbmi->tx_type]; + } } +#else // CONFIG_EXT_TX + txfm_rd_in_plane(x, +#if CONFIG_VAR_TX + cpi, +#endif + &r, &d, &s, + &sse, ref_best_rd, 0, bs, n, + cpi->sf.use_fast_coef_costing); #endif // CONFIG_EXT_TX if (r == INT_MAX) continue; - if (tx_select) - r += r_tx_size; - if (s) { - if (is_inter_block(mbmi)) { + if (is_inter) { rd = RDCOST(x->rdmult, x->rddiv, s1, sse); - if (tx_select) - r -= r_tx_size; } else { rd = RDCOST(x->rdmult, x->rddiv, s1 + r_tx_size * tx_select, sse); } } else { - rd = RDCOST(x->rdmult, x->rddiv, r + s0, d); + rd = RDCOST(x->rdmult, x->rddiv, r + s0 + r_tx_size * tx_select, d); } - if (is_inter_block(mbmi) && !xd->lossless[mbmi->segment_id] && !s) + if (tx_select && !(s && is_inter)) + r += r_tx_size; + + if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] && !s) rd = VPXMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, sse)); // Early termination in transform size search. - if (0 && cpi->sf.tx_size_search_breakout && - (rd== INT64_MAX || - (n < (int) max_tx_size && rd > last_rd) || - s == 1)) + if (cpi->sf.tx_size_search_breakout && + (rd == INT64_MAX || +#if CONFIG_EXT_TX + (s == 1 && tx_type != DCT_DCT) || +#else + (s == 1) || +#endif + (n < (int) max_tx_size && rd > last_rd))) break; last_rd = rd; -#if CONFIG_EXT_TX if (rd < - (is_inter_block(mbmi) && best_tx_type == DCT_DCT ? ext_tx_th : 1) * - best_rd) { -#else - if (rd < best_rd) { +#if CONFIG_EXT_TX + (is_inter && best_tx_type == DCT_DCT ? ext_tx_th : 1) * #endif // CONFIG_EXT_TX + best_rd) { best_tx = n; best_rd = rd; *distortion = d; @@ -1170,7 +1201,7 @@ static int64_t rd_pick_intra4x4block(VP10_COMP *cpi, MACROBLOCK *x, // one of the neighboring directional modes if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) { if (conditional_skipintra(mode, *best_mode)) - continue; + continue; } memcpy(tempa, ta, sizeof(ta)); @@ -1182,8 +1213,8 @@ static int64_t rd_pick_intra4x4block(VP10_COMP *cpi, MACROBLOCK *x, const uint8_t *const src = &src_init[idx * 4 + idy * 4 * src_stride]; uint8_t *const dst = &dst_init[idx * 4 + idy * 4 * dst_stride]; int16_t *const src_diff = vp10_raster_block_offset_int16(BLOCK_8X8, - block, - p->src_diff); + block, + p->src_diff); tran_low_t *const coeff = BLOCK_OFFSET(x->plane[0].coeff, block); xd->mi[0]->bmi[block].as_mode = mode; vp10_predict_intra_block(xd, 1, 1, TX_4X4, mode, dst, dst_stride, @@ -1262,7 +1293,7 @@ static int64_t rd_pick_intra4x4block(VP10_COMP *cpi, MACROBLOCK *x, num_4x4_blocks_wide * 4 * sizeof(uint16_t)); } } - next_highbd: +next_highbd: {} } if (best_rd >= rd_thresh) @@ -1291,7 +1322,7 @@ static int64_t rd_pick_intra4x4block(VP10_COMP *cpi, MACROBLOCK *x, // one of the neighboring directional modes if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) { if (conditional_skipintra(mode, *best_mode)) - continue; + continue; } memcpy(tempa, ta, sizeof(ta));