Refactoring tx-types to add more flexibility

Allows inter and intra tx_types to have different sets of
transforms for different tx_size/sb_type combinations.

Change-Id: Ic0ac1daef7a9fb15c4210271e4d04cd36e5cec8e

diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index 392b5d233fddf33a3928b7e5b4f94d9e48607a0f..4bf06d186d6bdfaa6dfa94dc195faf92927abeea 100644 (file)
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -32,11 +32,6 @@ extern "C" {
 
 #define MAX_MB_PLANE 3
 
-#if CONFIG_EXT_TX
-#define GET_TX_TYPES(tx_size) \
-    ((tx_size) >= TX_32X32 ? 1 : TX_TYPES)
-#endif  // CONFIG_EXT_TX
-
 typedef enum {
   KEY_FRAME = 0,
   INTER_FRAME = 1,
@@ -250,6 +245,58 @@ static const TX_TYPE intra_mode_to_tx_type_lookup[INTRA_MODES] = {
   ADST_ADST,  // TM
 };
 
+#if CONFIG_EXT_TX
+static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = {
+  1, 17, 10, 2
+};
+static const int num_ext_tx_set_intra[EXT_TX_SETS_INTRA] = {
+  1, 17,
+};
+
+#define USE_IDTX_FOR_32X32 0
+static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs,
+                                 int is_inter) {
+  (void) is_inter;
+  if (tx_size > TX_32X32 || bs < BLOCK_8X8) return 0;
+#if USE_IDTX_FOR_32X32
+  if (tx_size == TX_32X32) return is_inter ? 3 : 0;
+#else
+  if (tx_size == TX_32X32) return 0;
+#endif
+  return 1;
+}
+
+static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs,
+                                   int is_inter) {
+  const int set = get_ext_tx_set(tx_size, bs, is_inter);
+  return is_inter ? num_ext_tx_set_inter[set] : num_ext_tx_set_intra[set];
+}
+
+static const int use_intra_ext_tx_for_tx[EXT_TX_SETS_INTRA][TX_SIZES] = {
+  { 0, 0, 0, 0, },  // unused
+  { 1, 1, 1, 0, },
+};
+
+static const int use_inter_ext_tx_for_tx[EXT_TX_SETS_INTER][TX_SIZES] = {
+  { 0, 0, 0, 0, },  // unused
+  { 1, 1, 1, 0, },
+  { 0, 0, 0, 0, },
+  { 0, 0, 0, USE_IDTX_FOR_32X32, },
+};
+
+static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
+  { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, },
+  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, },
+};
+
+static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = {
+  { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, },
+  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, },
+  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, },
+  { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, },
+};
+#endif  // CONFIG_EXT_TX
+
 static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type,
                                   const MACROBLOCKD *xd,
                                   int block_idx, TX_SIZE tx_size) {
@@ -257,16 +304,27 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type,
   const MB_MODE_INFO *const mbmi = &mi->mbmi;
 
 #if CONFIG_EXT_TX
-  if ( xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32)
+#if USE_IDTX_FOR_32X32
+  if (xd->lossless[mbmi->segment_id] || tx_size > TX_32X32 ||
+      (tx_size >= TX_32X32 && !is_inter_block(mbmi)))
+#else
+  if (xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32)
+#endif
     return DCT_DCT;
   if (mbmi->sb_type >= BLOCK_8X8) {
-    if (plane_type == PLANE_TYPE_Y || is_inter_block(mbmi))
+    if (plane_type == PLANE_TYPE_Y)
       return mbmi->tx_type;
+    if (is_inter_block(mbmi))
+      // UV Inter only
+      return (mbmi->tx_type == IDTX && tx_size == TX_32X32 ?
+              DCT_DCT : mbmi->tx_type);
+      // return mbmi->tx_type;
   }
 
-  if (is_inter_block(mbmi))
+  // Sub8x8-Inter/Intra OR UV-Intra
+  if (is_inter_block(mbmi))  // Sub8x8-Inter
     return DCT_DCT;
-  else
+  else  // Sub8x8 Intra OR UV-Intra
     return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ?
         get_y_mode(mi, block_idx) : mbmi->uv_mode];
 #else

diff --git a/vp10/common/entropymode.c b/vp10/common/entropymode.c
index ccfd6de4a87b5f9f3bac05be096aa75a1b6da82d..59e6df8f317c50aac40875ebc4ed1d8a63c55242 100644 (file)
--- a/vp10/common/entropymode.c
+++ b/vp10/common/entropymode.c
@@ -755,103 +755,190 @@ static const vpx_prob default_switchable_interp_prob[SWITCHABLE_FILTER_CONTEXTS]
 };
 
 #if CONFIG_EXT_TX
-const vpx_tree_index vp10_tx_type_tree[TREE_SIZE(TX_TYPES)] = {
-  -IDTX, 2,
-  -DCT_DCT, 4,
-  -DST_DST, 6,
-  8, 18,
-  10, 12,
-  -DST_DCT, -DCT_DST,
-  14, 16,
-  -ADST_DCT, -DCT_ADST,
-  -FLIPADST_DCT, -DCT_FLIPADST,
-  20, 26,
-  22, 24,
-  -DST_ADST, -ADST_DST,
-  -DST_FLIPADST, -FLIPADST_DST,
-  28, 30,
-  -ADST_ADST, -FLIPADST_FLIPADST,
-  -ADST_FLIPADST, -FLIPADST_ADST
+const vpx_tree_index vp10_ext_tx_inter_tree[EXT_TX_SETS_INTER]
+                                           [TREE_SIZE(TX_TYPES)] = {
+  {
+  }, {
+    -IDTX, 2,
+    -DCT_DCT, 4,
+    -DST_DST, 6,
+    8, 18,
+    10, 12,
+    -DST_DCT, -DCT_DST,
+    14, 16,
+    -ADST_DCT, -DCT_ADST,
+    -FLIPADST_DCT, -DCT_FLIPADST,
+    20, 26,
+    22, 24,
+    -DST_ADST, -ADST_DST,
+    -DST_FLIPADST, -FLIPADST_DST,
+    28, 30,
+    -ADST_ADST, -FLIPADST_FLIPADST,
+    -ADST_FLIPADST, -FLIPADST_ADST,
+  }, {
+    -IDTX, 2,
+    -DCT_DCT, 4,
+    6, 12,
+    8, 10,
+    -ADST_DCT, -DCT_ADST,
+    -FLIPADST_DCT, -DCT_FLIPADST,
+    14, 16,
+    -ADST_ADST, -FLIPADST_FLIPADST,
+    -ADST_FLIPADST, -FLIPADST_ADST
+  }, {
+    -IDTX, -DCT_DCT,
+  }
+};
+
+const vpx_tree_index vp10_ext_tx_intra_tree[EXT_TX_SETS_INTRA]
+                                           [TREE_SIZE(TX_TYPES)] = {
+  {
+  }, {
+    -IDTX, 2,
+    -DCT_DCT, 4,
+    -DST_DST, 6,
+    8, 18,
+    10, 12,
+    -DST_DCT, -DCT_DST,
+    14, 16,
+    -ADST_DCT, -DCT_ADST,
+    -FLIPADST_DCT, -DCT_FLIPADST,
+    20, 26,
+    22, 24,
+    -DST_ADST, -ADST_DST,
+    -DST_FLIPADST, -FLIPADST_DST,
+    28, 30,
+    -ADST_ADST, -FLIPADST_FLIPADST,
+    -ADST_FLIPADST, -FLIPADST_ADST,
+  }
 };
 
 static const vpx_prob
-default_inter_tx_type_prob[EXT_TX_SIZES][TX_TYPES - 1] = {
-  {  12, 112,  16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
-    128 },
-  {  12, 112,  16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
-    128 },
-  {  12, 112,  16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
-    128 },
+default_inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1] = {
+  {
+    // unused
+  }, {
+    { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+      128 },
+    { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+      128 },
+    { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+      128 },
+#if EXT_TX_SIZES == 4
+    { 12, 112, 16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+      128 },
+#endif
+  }, {
+    { 12, 112, 128, 128, 128, 128, 128, 128, 128 },
+    { 12, 112, 128, 128, 128, 128, 128, 128, 128 },
+    { 12, 112, 128, 128, 128, 128, 128, 128, 128 },
+#if EXT_TX_SIZES == 4
+    { 12, 112, 128, 128, 128, 128, 128, 128, 128 },
+#endif
+  }, {
+    { 12, },
+    { 12, },
+    { 12, },
+#if EXT_TX_SIZES == 4
+    { 12, },
+#endif
+  }
 };
 
 static const vpx_prob
-default_intra_tx_type_prob[EXT_TX_SIZES][INTRA_MODES][TX_TYPES - 1] = {
-    {
-        {   8,  11,  24, 112,  87, 137, 127, 134,
-          128,  86, 128, 124, 125, 133, 176, 123, },
-        {  10,   9,  39, 106,  73, 155, 163, 228,
-           35,  62, 129, 127, 133, 114, 213, 234, },
-        {  10,   9,  14,  88,  91, 127, 151,  51,
-          210,  89, 126,  58,  52, 116, 217,  24, },
-        {   9,   6,  29, 113,  98, 131, 149, 210,
-          119,  60, 124,  93,  90, 143, 170, 197, },
-        {   8,   8,  38, 101, 111, 166, 167, 141,
-          130, 105, 128,  75,  75, 118, 197, 117, },
-        {   7,   8,  39,  91, 101, 153, 166, 200,
-           99,  77, 123,  90,  83, 144, 224, 192, },
-        {   7,  10,  26,  86, 119, 154, 130, 101,
-          152,  91, 129,  75,  79, 137, 219,  77, },
-        {  10,  13,  20,  86, 102, 162, 112,  76,
-          171,  86, 134, 122, 106, 124, 196,  44, },
-        {   8,   9,  33, 108, 100, 144, 148, 215,
-           77,  60, 125, 125, 128, 126, 198, 220, },
-        {   3,  10,  29, 111,  69, 141, 204, 141,
-          139,  93, 120,  75,  77, 163, 242, 124, },
-    },
+default_intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES]
+                         [INTRA_MODES][TX_TYPES - 1] = {
+  {
+    // unused
+  }, {
     {
-        {   2,  53,  18, 147,  96,  98, 136, 133,
-          131, 120, 153, 163, 169, 137, 173, 124, },
-        {   4,  18,  34, 133,  54, 130, 179, 228,
-           28,  72, 153, 164, 168, 118, 227, 239, },
-        {   4,  18,  13, 125,  72, 110, 176,  36,
-          221, 104, 148,  75,  72, 117, 225,  19, },
-        {   8,  33,  24, 162, 113,  99, 147, 226,
-          103,  85, 153, 143, 153, 124, 155, 210, },
-        {   2,  15,  35, 107, 127, 158, 192, 128,
-          126, 116, 151,  95,  88, 182, 241, 119, },
-        {   3,  15,  36, 112, 100, 146, 194, 189,
-           90,  98, 152,  99, 100, 165, 235, 175, },
-        {   3,  16,  29, 109, 103, 140, 182,  76,
-          173, 104, 147,  82,  85, 159, 235,  70, },
-        {   9,  24,  14, 120,  86, 156, 161,  34,
-          177, 121, 142, 128, 128, 126, 185,  37, },
-        {   5,  24,  29, 152,  98,  99, 174, 228,
-           82,  76, 147, 149, 128, 132, 191, 225, },
-        {   2,  15,  29, 111,  77, 126, 200, 135,
-          117,  93, 152,  96,  84, 191, 245, 135, },
-    },
-    {
-        {   2,  69,  13, 173, 111,  69, 137, 159,
-          159, 146, 151, 193, 203, 131, 180, 123, },
-        {   1,  12,  33, 164,  32,  98, 204, 242,
-           23,  99, 149, 215, 232, 110, 239, 245, },
-        {   1,  17,   9, 136,  82,  83, 171,  28,
-          231, 128, 135,  76,  64, 118, 235,  17, },
-        {   4,  41,  17, 195, 131,  58, 161, 237,
-          141,  97, 153, 189, 191, 117, 182, 202, },
-        {   2,  17,  36, 104, 149, 137, 217, 139,
-          191, 119, 125, 107, 115, 223, 249, 110, },
-        {   2,  14,  24, 127,  91, 135, 219, 198,
-          113,  91, 164, 125, 173, 211, 250, 116, },
-        {   3,  19,  24, 120, 102, 130, 209,  81,
-          187,  95, 143, 102,  50, 190, 244,  56, },
-        {   4,  27,  10, 128,  91, 157, 181,  33,
-          181, 150, 141, 141, 166, 114, 215,  25, },
-        {   2,  34,  27, 187, 102,  77, 210, 245,
-          113, 107, 136, 184, 188, 121, 210, 234, },
-        {   1,  15,  22, 141,  59,  94, 208, 133,
-          154,  95, 152, 112, 105, 191, 242, 111, },
+      {   8,  11,  24, 112,  87, 137, 127, 134,
+        128,  86, 128, 124, 125, 133, 176, 123, },
+      {  10,   9,  39, 106,  73, 155, 163, 228,
+        35,  62, 129, 127, 133, 114, 213, 234, },
+      {  10,   9,  14,  88,  91, 127, 151,  51,
+        210,  89, 126,  58,  52, 116, 217,  24, },
+      {   9,   6,  29, 113,  98, 131, 149, 210,
+        119,  60, 124,  93,  90, 143, 170, 197, },
+      {   8,   8,  38, 101, 111, 166, 167, 141,
+        130, 105, 128,  75,  75, 118, 197, 117, },
+      {   7,   8,  39,  91, 101, 153, 166, 200,
+        99,  77, 123,  90,  83, 144, 224, 192, },
+      {   7,  10,  26,  86, 119, 154, 130, 101,
+        152,  91, 129,  75,  79, 137, 219,  77, },
+      {  10,  13,  20,  86, 102, 162, 112,  76,
+        171,  86, 134, 122, 106, 124, 196,  44, },
+      {   8,   9,  33, 108, 100, 144, 148, 215,
+        77,  60, 125, 125, 128, 126, 198, 220, },
+      {   3,  10,  29, 111,  69, 141, 204, 141,
+        139,  93, 120,  75,  77, 163, 242, 124, },
+    }, {
+      {   2,  53,  18, 147,  96,  98, 136, 133,
+        131, 120, 153, 163, 169, 137, 173, 124, },
+      {   4,  18,  34, 133,  54, 130, 179, 228,
+        28,  72, 153, 164, 168, 118, 227, 239, },
+      {   4,  18,  13, 125,  72, 110, 176,  36,
+        221, 104, 148,  75,  72, 117, 225,  19, },
+      {   8,  33,  24, 162, 113,  99, 147, 226,
+        103,  85, 153, 143, 153, 124, 155, 210, },
+      {   2,  15,  35, 107, 127, 158, 192, 128,
+        126, 116, 151,  95,  88, 182, 241, 119, },
+      {   3,  15,  36, 112, 100, 146, 194, 189,
+        90,  98, 152,  99, 100, 165, 235, 175, },
+      {   3,  16,  29, 109, 103, 140, 182,  76,
+        173, 104, 147,  82,  85, 159, 235,  70, },
+      {   9,  24,  14, 120,  86, 156, 161,  34,
+        177, 121, 142, 128, 128, 126, 185,  37, },
+      {   5,  24,  29, 152,  98,  99, 174, 228,
+        82,  76, 147, 149, 128, 132, 191, 225, },
+      {   2,  15,  29, 111,  77, 126, 200, 135,
+        117,  93, 152,  96,  84, 191, 245, 135, },
+    }, {
+      {   2,  69,  13, 173, 111,  69, 137, 159,
+        159, 146, 151, 193, 203, 131, 180, 123, },
+      {   1,  12,  33, 164,  32,  98, 204, 242,
+         23,  99, 149, 215, 232, 110, 239, 245, },
+      {   1,  17,   9, 136,  82,  83, 171,  28,
+        231, 128, 135,  76,  64, 118, 235,  17, },
+      {   4,  41,  17, 195, 131,  58, 161, 237,
+        141,  97, 153, 189, 191, 117, 182, 202, },
+      {   2,  17,  36, 104, 149, 137, 217, 139,
+        191, 119, 125, 107, 115, 223, 249, 110, },
+      {   2,  14,  24, 127,  91, 135, 219, 198,
+        113,  91, 164, 125, 173, 211, 250, 116, },
+      {   3,  19,  24, 120, 102, 130, 209,  81,
+        187,  95, 143, 102,  50, 190, 244,  56, },
+      {   4,  27,  10, 128,  91, 157, 181,  33,
+        181, 150, 141, 141, 166, 114, 215,  25, },
+      {   2,  34,  27, 187, 102,  77, 210, 245,
+        113, 107, 136, 184, 188, 121, 210, 234, },
+      {   1,  15,  22, 141,  59,  94, 208, 133,
+        154,  95, 152, 112, 105, 191, 242, 111, },
+#if EXT_TX_SIZES == 4
+    }, {
+      {   2,  69,  13, 173, 111,  69, 137, 159,
+        159, 146, 151, 193, 203, 131, 180, 123, },
+      {   1,  12,  33, 164,  32,  98, 204, 242,
+        23,  99, 149, 215, 232, 110, 239, 245, },
+      {   1,  17,   9, 136,  82,  83, 171,  28,
+        231, 128, 135,  76,  64, 118, 235,  17, },
+      {   4,  41,  17, 195, 131,  58, 161, 237,
+        141,  97, 153, 189, 191, 117, 182, 202, },
+      {   2,  17,  36, 104, 149, 137, 217, 139,
+        191, 119, 125, 107, 115, 223, 249, 110, },
+      {   2,  14,  24, 127,  91, 135, 219, 198,
+        113,  91, 164, 125, 173, 211, 250, 116, },
+      {   3,  19,  24, 120, 102, 130, 209,  81,
+        187,  95, 143, 102,  50, 190, 244,  56, },
+      {   4,  27,  10, 128,  91, 157, 181,  33,
+        181, 150, 141, 141, 166, 114, 215,  25, },
+      {   2,  34,  27, 187, 102,  77, 210, 245,
+        113, 107, 136, 184, 188, 121, 210, 234, },
+      {   1,  15,  22, 141,  59,  94, 208, 133,
+        154,  95, 152, 112, 105, 191, 242, 111, },
+#endif
     },
+  }
 };
 #endif  // CONFIG_EXT_TX
 #if CONFIG_MISC_FIXES
@@ -875,8 +962,8 @@ static void init_mode_probs(FRAME_CONTEXT *fc) {
   vp10_copy(fc->skip_probs, default_skip_probs);
   vp10_copy(fc->inter_mode_probs, default_inter_mode_probs);
 #if CONFIG_EXT_TX
-  vp10_copy(fc->inter_tx_type_prob, default_inter_tx_type_prob);
-  vp10_copy(fc->intra_tx_type_prob, default_intra_tx_type_prob);
+  vp10_copy(fc->inter_ext_tx_prob, default_inter_ext_tx_prob);
+  vp10_copy(fc->intra_ext_tx_prob, default_intra_ext_tx_prob);
 #endif  // CONFIG_EXT_TX
 #if CONFIG_MISC_FIXES
   vp10_copy(fc->seg.tree_probs, default_seg_probs.tree_probs);
@@ -885,7 +972,7 @@ static void init_mode_probs(FRAME_CONTEXT *fc) {
 }
 
 const vpx_tree_index vp10_switchable_interp_tree
-                         [TREE_SIZE(SWITCHABLE_FILTERS)] = {
+[TREE_SIZE(SWITCHABLE_FILTERS)] = {
   -EIGHTTAP, 2,
   -EIGHTTAP_SMOOTH, -EIGHTTAP_SHARP
 };
@@ -972,15 +1059,26 @@ void vp10_adapt_intra_frame_probs(VP10_COMMON *cm) {
         pre_fc->skip_probs[i], counts->skip[i]);
 
 #if CONFIG_EXT_TX
-  for (i = TX_4X4; i <= TX_16X16; ++i) {
-    int j;
-    vpx_tree_merge_probs(vp10_tx_type_tree, pre_fc->inter_tx_type_prob[i],
-                         counts->inter_tx_type[i], fc->inter_tx_type_prob[i]);
-
-    for (j = 0; j < INTRA_MODES; ++j)
-      vpx_tree_merge_probs(vp10_tx_type_tree, pre_fc->intra_tx_type_prob[i][j],
-                           counts->intra_tx_type[i][j],
-                           fc->intra_tx_type_prob[i][j]);
+  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
+    int s;
+    for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
+      if (use_inter_ext_tx_for_tx[s][i]) {
+        vpx_tree_merge_probs(vp10_ext_tx_inter_tree[s],
+                             pre_fc->inter_ext_tx_prob[s][i],
+                             counts->inter_ext_tx[s][i],
+                             fc->inter_ext_tx_prob[s][i]);
+      }
+    }
+    for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
+      if (use_intra_ext_tx_for_tx[s][i]) {
+        int j;
+        for (j = 0; j < INTRA_MODES; ++j)
+          vpx_tree_merge_probs(vp10_ext_tx_intra_tree[s],
+                               pre_fc->intra_ext_tx_prob[s][i][j],
+                               counts->intra_ext_tx[s][i][j],
+                               fc->intra_ext_tx_prob[s][i][j]);
+      }
+    }
   }
 #endif  // CONFIG_EXT_TX
 #if CONFIG_MISC_FIXES

diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h
index 3780b4874db994a848bbac212c2f4aa897e87f9e..b53b4e157394080c3b992d00e2d6b2afe13bb4d6 100644 (file)
--- a/vp10/common/entropymode.h
+++ b/vp10/common/entropymode.h
@@ -70,8 +70,9 @@ typedef struct frame_contexts {
   vpx_prob skip_probs[SKIP_CONTEXTS];
   nmv_context nmvc;
 #if CONFIG_EXT_TX
-  vpx_prob inter_tx_type_prob[EXT_TX_SIZES][TX_TYPES - 1];
-  vpx_prob intra_tx_type_prob[EXT_TX_SIZES][INTRA_MODES][TX_TYPES - 1];
+  vpx_prob inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1];
+  vpx_prob intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
+                            [TX_TYPES - 1];
 #endif  // CONFIG_EXT_TX
 #if CONFIG_MISC_FIXES
   struct segmentation_probs seg;
@@ -98,8 +99,9 @@ typedef struct FRAME_COUNTS {
   unsigned int skip[SKIP_CONTEXTS][2];
   nmv_context_counts mv;
 #if CONFIG_EXT_TX
-  unsigned int inter_tx_type[EXT_TX_SIZES][TX_TYPES];
-  unsigned int intra_tx_type[EXT_TX_SIZES][INTRA_MODES][TX_TYPES];
+  unsigned int inter_ext_tx[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
+  unsigned int intra_ext_tx[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
+                           [TX_TYPES];
 #endif  // CONFIG_EXT_TX
 #if CONFIG_MISC_FIXES
   struct seg_counts seg;
@@ -147,7 +149,10 @@ void vp10_tx_counts_to_branch_counts_8x8(const unsigned int *tx_count_8x8p,
                                     unsigned int (*ct_8x8p)[2]);
 
 #if CONFIG_EXT_TX
-extern const vpx_tree_index vp10_tx_type_tree[TREE_SIZE(TX_TYPES)];
+extern const vpx_tree_index
+    vp10_ext_tx_inter_tree[EXT_TX_SETS_INTER][TREE_SIZE(TX_TYPES)];
+extern const vpx_tree_index
+    vp10_ext_tx_intra_tree[EXT_TX_SETS_INTRA][TREE_SIZE(TX_TYPES)];
 #endif  // CONFIG_EXT_TX
 
 static INLINE int vp10_ceil_log2(int n) {

diff --git a/vp10/common/enums.h b/vp10/common/enums.h
index 36fa591a402134ff9bb61f69d926c79a33d97f2d..33c5eaf20e9dc1b7709d401a22ada3352dcf502f 100644 (file)
--- a/vp10/common/enums.h
+++ b/vp10/common/enums.h
@@ -113,7 +113,12 @@ typedef enum {
 } TX_TYPE;
 
 #if CONFIG_EXT_TX
-#define EXT_TX_SIZES   3  // number of sizes that use extended transforms
+#define EXT_TX_SIZES       3  // number of sizes that use extended transforms
+#define EXT_TX_SETS_INTER  4  // Sets of transform selections for INTER
+#define EXT_TX_SETS_INTRA  2  // Sets of transform selections for INTRA
+
+#define TX_TYPES_10    10
+#define TX_TYPES_2      2
 #endif  // CONFIG_EXT_TX
 
 typedef enum {

diff --git a/vp10/common/idct.c b/vp10/common/idct.c
index 6f8df8e7455fc1e26ce901fd93bf52426e2470af..94e75643b82a89a7fa79b244e4132d7f02b6315e 100644 (file)
--- a/vp10/common/idct.c
+++ b/vp10/common/idct.c
@@ -13,6 +13,7 @@
 #include "./vp10_rtcd.h"
 #include "./vpx_dsp_rtcd.h"
 #include "vp10/common/blockd.h"
+#include "vp10/common/enums.h"
 #include "vp10/common/idct.h"
 #include "vpx_dsp/inv_txfm.h"
 #include "vpx_ports/mem.h"
@@ -864,6 +865,11 @@ void vp10_inv_txfm_add_32x32(const tran_low_t *input, uint8_t *dest,
     case DCT_DCT:
       vp10_idct32x32_add(input, dest, stride, eob);
       break;
+#if CONFIG_EXT_TX
+    case IDTX:
+      inv_idtx_add_c(input, dest, stride, 32);
+      break;
+#endif  // CONFIG_EXT_TX
     case ADST_DCT:
     case DCT_ADST:
     case ADST_ADST:
@@ -1295,6 +1301,11 @@ void vp10_highbd_inv_txfm_add_32x32(const tran_low_t *input, uint8_t *dest,
     case DCT_DCT:
       vp10_highbd_idct32x32_add(input, dest, stride, eob, bd);
       break;
+#if CONFIG_EXT_TX
+    case IDTX:
+      highbd_inv_idtx_add_c(input, dest, stride, 32, bd);
+      break;
+#endif  // CONFIG_EXT_TX
     case ADST_DCT:
     case DCT_ADST:
     case ADST_ADST:

diff --git a/vp10/common/thread_common.c b/vp10/common/thread_common.c
index 5523abbbd982146992f3249f7a99267cc6c1c809..7508195f37842c3e745e908f0bf7d15a5f115eb3 100644 (file)
--- a/vp10/common/thread_common.c
+++ b/vp10/common/thread_common.c
@@ -436,15 +436,25 @@ void vp10_accumulate_frame_counts(VP10_COMMON *cm, FRAME_COUNTS *counts,
   }
 
 #if CONFIG_EXT_TX
-  for (i = 0; i < EXT_TX_SIZES; i++)
-    for (j = 0; j < TX_TYPES; j++)
-      cm->counts.inter_tx_type[i][j] += counts->inter_tx_type[i][j];
-
-  for (i = 0; i < EXT_TX_SIZES; i++)
-    for (j = 0; j < INTRA_MODES; j++)
-      for (k = 0; k < TX_TYPES; k++)
-        cm->counts.intra_tx_type[i][j][k] += counts->intra_tx_type[i][j][k];
-#endif
+  for (i = 0; i < EXT_TX_SIZES; i++) {
+    int s, k;
+    for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
+      if (use_inter_ext_tx_for_tx[s][i]) {
+        for (k = 0; k < TX_TYPES; k++)
+          cm->counts.inter_ext_tx[s][i][k] += counts->inter_ext_tx[s][i][k];
+      }
+    }
+    for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
+      if (use_intra_ext_tx_for_tx[s][i]) {
+        int j;
+        for (j = 0; j < INTRA_MODES; ++j)
+          for (k = 0; k < TX_TYPES; k++)
+            cm->counts.intra_ext_tx[s][i][j][k] +=
+                counts->intra_ext_tx[s][i][j][k];
+      }
+    }
+  }
+#endif  // CONFIG_EXT_TX
 
 #if CONFIG_MISC_FIXES
   for (i = 0; i < PREDICTION_PROBS; i++)

diff --git a/vp10/decoder/decodeframe.c b/vp10/decoder/decodeframe.c
index b3f59ff6d881c2d0aaf3bfcb794829a9a3b2f456..eaa31f1815ca2bf982b73737048fb8deb4851bf4 100644 (file)
--- a/vp10/decoder/decodeframe.c
+++ b/vp10/decoder/decodeframe.c
@@ -2224,17 +2224,26 @@ static size_t read_uncompressed_header(VP10Decoder *pbi,
 #if CONFIG_EXT_TX
 static void read_ext_tx_probs(FRAME_CONTEXT *fc, vpx_reader *r) {
   int i, j, k;
-  if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
-    for (i = TX_4X4; i <= TX_16X16; ++i)
-      for (j = 0; j < TX_TYPES - 1; ++j)
-        vp10_diff_update_prob(r, &fc->inter_tx_type_prob[i][j]);
+  int s;
+  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
+    if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
+      for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
+        if (!use_inter_ext_tx_for_tx[s][i]) continue;
+        for (j = 0; j < num_ext_tx_set_inter[s] - 1; ++j)
+          vp10_diff_update_prob(r, &fc->inter_ext_tx_prob[s][i][j]);
+      }
+    }
   }
 
-  if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
-    for (i = TX_4X4; i <= TX_16X16; ++i)
-      for (j = 0; j < INTRA_MODES; ++j)
-        for (k = 0; k < TX_TYPES - 1; ++k)
-          vp10_diff_update_prob(r, &fc->intra_tx_type_prob[i][j][k]);
+  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
+    if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
+      for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
+        if (!use_intra_ext_tx_for_tx[s][i]) continue;
+        for (j = 0; j < INTRA_MODES; ++j)
+          for (k = 0; k < num_ext_tx_set_intra[s] - 1; ++k)
+            vp10_diff_update_prob(r, &fc->intra_ext_tx_prob[s][i][j][k]);
+      }
+    }
   }
 }
 #endif  // CONFIG_EXT_TX
@@ -2365,11 +2374,11 @@ static void debug_check_frame_counts(const VP10_COMMON *const cm) {
   assert(!memcmp(&cm->counts.mv, &zero_counts.mv, sizeof(cm->counts.mv)));
 
 #if CONFIG_EXT_TX
-  assert(!memcmp(cm->counts.inter_tx_type,
-                 zero_counts.inter_tx_type, sizeof(cm->counts.inter_tx_type)));
-  assert(!memcmp(cm->counts.intra_tx_type,
-                 zero_counts.intra_tx_type, sizeof(cm->counts.intra_tx_type)));
-#endif
+  assert(!memcmp(cm->counts.inter_ext_tx, zero_counts.inter_ext_tx,
+                 sizeof(cm->counts.inter_ext_tx)));
+  assert(!memcmp(cm->counts.intra_ext_tx, zero_counts.intra_ext_tx,
+                 sizeof(cm->counts.intra_ext_tx)));
+#endif  // CONFIG_EXT_TX
 }
 #endif  // NDEBUG
 

diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c
index 907457470de0eead4dc564a84c0b36a37a4af1b0..803d0df9de955ba1be881d0d0a1b468b3c7073d3 100644 (file)
--- a/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@ -381,15 +381,19 @@ static void read_intra_frame_mode_info(VP10_COMMON *const cm,
     read_palette_mode_info(cm, xd, r);
 
 #if CONFIG_EXT_TX
-    if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
-        mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip &&
+    if (get_ext_tx_types(mbmi->tx_size, mbmi->sb_type, 0) > 1 &&
+        cm->base_qindex > 0 && !mbmi->skip &&
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
       FRAME_COUNTS *counts = xd->counts;
-      mbmi->tx_type =
-          vpx_read_tree(r, vp10_tx_type_tree,
-                        cm->fc->intra_tx_type_prob[mbmi->tx_size][mbmi->mode]);
-      if (counts)
-        ++counts->intra_tx_type[mbmi->tx_size][mbmi->mode][mbmi->tx_type];
+      int eset = get_ext_tx_set(mbmi->tx_size, mbmi->sb_type, 0);
+      if (eset > 0) {
+        mbmi->tx_type = vpx_read_tree(
+            r, vp10_ext_tx_intra_tree[eset],
+            cm->fc->intra_ext_tx_prob[eset][mbmi->tx_size][mbmi->mode]);
+        if (counts)
+          ++counts->intra_ext_tx[eset][mbmi->tx_size][mbmi->mode]
+                                [mbmi->tx_type];
+      }
     } else {
       mbmi->tx_type = DCT_DCT;
     }
@@ -790,25 +794,30 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi,
     read_intra_block_mode_info(cm, xd, mi, r);
 
 #if CONFIG_EXT_TX
-    if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
-        mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip &&
+    if (get_ext_tx_types(mbmi->tx_size, mbmi->sb_type, inter_block) > 1 &&
+        cm->base_qindex > 0 && !mbmi->skip &&
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+      int eset = get_ext_tx_set(mbmi->tx_size, mbmi->sb_type,
+                                inter_block);
       FRAME_COUNTS *counts = xd->counts;
 
-      if (inter_block)
-        mbmi->tx_type =
-            vpx_read_tree(r, vp10_tx_type_tree,
-                          cm->fc->inter_tx_type_prob[mbmi->tx_size]);
-      else
-        mbmi->tx_type = vpx_read_tree(r, vp10_tx_type_tree,
-                                      cm->fc->intra_tx_type_prob
-                                      [mbmi->tx_size][mbmi->mode]);
-
-      if (counts) {
-        if (inter_block)
-          ++counts->inter_tx_type[mbmi->tx_size][mbmi->tx_type];
-        else
-          ++counts->intra_tx_type[mbmi->tx_size][mbmi->mode][mbmi->tx_type];
+      if (inter_block) {
+        if (eset > 0) {
+          mbmi->tx_type =
+              vpx_read_tree(r, vp10_ext_tx_inter_tree[eset],
+                            cm->fc->inter_ext_tx_prob[eset][mbmi->tx_size]);
+          if (counts)
+            ++counts->inter_ext_tx[eset][mbmi->tx_size][mbmi->tx_type];
+        }
+      } else {
+        if (eset > 0) {
+          mbmi->tx_type = vpx_read_tree(r, vp10_ext_tx_intra_tree[eset],
+                                        cm->fc->intra_ext_tx_prob[eset]
+                                        [mbmi->tx_size][mbmi->mode]);
+          if (counts)
+            ++counts->intra_ext_tx[eset][mbmi->tx_size]
+                                  [mbmi->mode][mbmi->tx_type];
+        }
       }
     } else {
       mbmi->tx_type = DCT_DCT;

diff --git a/vp10/encoder/bitstream.c b/vp10/encoder/bitstream.c
index b6e827d06344bb1d8f2d99e2982ea73b8777f181..e4dcc00aaa15500a82615820f2edd0dc2762aee3 100644 (file)
--- a/vp10/encoder/bitstream.c
+++ b/vp10/encoder/bitstream.c
@@ -73,12 +73,19 @@ static INLINE void write_uniform(vpx_writer *w, int n, int v) {
 }
 
 #if CONFIG_EXT_TX
-static struct vp10_token tx_type_encodings[TX_TYPES];
+static struct vp10_token ext_tx_inter_encodings[EXT_TX_SETS_INTER][TX_TYPES];
+static struct vp10_token ext_tx_intra_encodings[EXT_TX_SETS_INTRA][TX_TYPES];
 #endif  // CONFIG_EXT_TX
 
 void vp10_encode_token_init() {
 #if CONFIG_EXT_TX
-  vp10_tokens_from_tree(tx_type_encodings, vp10_tx_type_tree);
+  int s;
+  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
+    vp10_tokens_from_tree(ext_tx_inter_encodings[s], vp10_ext_tx_inter_tree[s]);
+  }
+  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
+    vp10_tokens_from_tree(ext_tx_intra_encodings[s], vp10_ext_tx_intra_tree[s]);
+  }
 #endif  // CONFIG_EXT_TX
 }
 
@@ -231,37 +238,51 @@ static void update_ext_tx_probs(VP10_COMMON *cm, vpx_writer *w) {
   const int savings_thresh = vp10_cost_one(GROUP_DIFF_UPDATE_PROB) -
                              vp10_cost_zero(GROUP_DIFF_UPDATE_PROB);
   int i, j;
-  int savings = 0;
-  int do_update = 0;
-  for (i = TX_4X4; i <= TX_16X16; ++i) {
-    savings += prob_diff_update_savings(
-        vp10_tx_type_tree, cm->fc->inter_tx_type_prob[i],
-        cm->counts.inter_tx_type[i], TX_TYPES);
-  }
-  do_update = savings > savings_thresh;
-  vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
-  if (do_update) {
-    for (i = TX_4X4; i <= TX_16X16; ++i) {
-      prob_diff_update(vp10_tx_type_tree, cm->fc->inter_tx_type_prob[i],
-                       cm->counts.inter_tx_type[i], TX_TYPES, w);
+  int s;
+  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
+    int savings = 0;
+    int do_update = 0;
+    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
+      if (!use_inter_ext_tx_for_tx[s][i]) continue;
+      savings += prob_diff_update_savings(
+          vp10_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
+          cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s]);
+    }
+    do_update = savings > savings_thresh;
+    vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
+    if (do_update) {
+      for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
+        if (!use_inter_ext_tx_for_tx[s][i]) continue;
+        prob_diff_update(vp10_ext_tx_inter_tree[s],
+                         cm->fc->inter_ext_tx_prob[s][i],
+                         cm->counts.inter_ext_tx[s][i],
+                         num_ext_tx_set_inter[s], w);
+      }
     }
   }
 
-  savings = 0;
-  do_update = 0;
-
-  for (i = TX_4X4; i <= TX_16X16; ++i)
-    for (j = 0; j < INTRA_MODES; ++j)
-      savings += prob_diff_update_savings(
-          vp10_tx_type_tree, cm->fc->intra_tx_type_prob[i][j],
-          cm->counts.intra_tx_type[i][j], TX_TYPES);
-  do_update = savings > savings_thresh;
-  vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
-  if (do_update) {
-    for (i = TX_4X4; i <= TX_16X16; ++i)
+  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
+    int savings = 0;
+    int do_update = 0;
+    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
+      if (!use_intra_ext_tx_for_tx[s][i]) continue;
       for (j = 0; j < INTRA_MODES; ++j)
-        prob_diff_update(vp10_tx_type_tree, cm->fc->intra_tx_type_prob[i][j],
-                         cm->counts.intra_tx_type[i][j], TX_TYPES, w);
+        savings += prob_diff_update_savings(
+            vp10_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j],
+            cm->counts.intra_ext_tx[s][i][j], num_ext_tx_set_intra[s]);
+    }
+    do_update = savings > savings_thresh;
+    vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
+    if (do_update) {
+      for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
+        if (!use_intra_ext_tx_for_tx[s][i]) continue;
+        for (j = 0; j < INTRA_MODES; ++j)
+          prob_diff_update(vp10_ext_tx_intra_tree[s],
+                           cm->fc->intra_ext_tx_prob[s][i][j],
+                           cm->counts.intra_ext_tx[s][i][j],
+                           num_ext_tx_set_intra[s], w);
+      }
+    }
   }
 }
 #endif  // CONFIG_EXT_TX
@@ -598,17 +619,22 @@ static void pack_inter_mode_mvs(VP10_COMP *cpi, const MODE_INFO *mi,
   }
 
 #if CONFIG_EXT_TX
-  if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
-      bsize >= BLOCK_8X8 && !mbmi->skip &&
+  if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter) > 1 &&
+      cm->base_qindex > 0 && !mbmi->skip &&
       !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
-    if (is_inter)
-      vp10_write_token(w, vp10_tx_type_tree,
-                       cm->fc->inter_tx_type_prob[mbmi->tx_size],
-                       &tx_type_encodings[mbmi->tx_type]);
-    else
-      vp10_write_token(w, vp10_tx_type_tree,
-                       cm->fc->intra_tx_type_prob[mbmi->tx_size][mbmi->mode],
-                       &tx_type_encodings[mbmi->tx_type]);
+    int eset = get_ext_tx_set(mbmi->tx_size, bsize, is_inter);
+    if (is_inter) {
+      if (eset > 0)
+        vp10_write_token(w, vp10_ext_tx_inter_tree[eset],
+                         cm->fc->inter_ext_tx_prob[eset][mbmi->tx_size],
+                         &ext_tx_inter_encodings[eset][mbmi->tx_type]);
+    } else {
+      if (eset > 0)
+        vp10_write_token(
+            w, vp10_ext_tx_intra_tree[eset],
+            cm->fc->intra_ext_tx_prob[eset][mbmi->tx_size][mbmi->mode],
+            &ext_tx_intra_encodings[eset][mbmi->tx_type]);
+    }
   }
 #endif  // CONFIG_EXT_TX
 }
@@ -690,12 +716,15 @@ static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd,
 
 
 #if CONFIG_EXT_TX
-  if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
-      bsize >= BLOCK_8X8 && !mbmi->skip &&
+  if (get_ext_tx_types(mbmi->tx_size, bsize, 0) > 1 &&
+      cm->base_qindex > 0 && !mbmi->skip &&
       !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
-    vp10_write_token(w, vp10_tx_type_tree,
-                     cm->fc->intra_tx_type_prob[mbmi->tx_size][mbmi->mode],
-                     &tx_type_encodings[mbmi->tx_type]);
+    int eset = get_ext_tx_set(mbmi->tx_size, bsize, 0);
+    if (eset > 0)
+      vp10_write_token(
+          w, vp10_ext_tx_intra_tree[eset],
+          cm->fc->intra_ext_tx_prob[eset][mbmi->tx_size][mbmi->mode],
+          &ext_tx_intra_encodings[eset][mbmi->tx_type]);
   }
 #endif  // CONFIG_EXT_TX
 }

diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 095c6c28ed36a408362ed14c4dd239b8501fe263..ea09f6e0bc31a52d0a31147700b602615e673517 100644 (file)
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -2430,7 +2430,6 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
   (void) best_rd;
   *rd_cost = best_rdc;
 
-
   if (best_rdc.rate < INT_MAX && best_rdc.dist < INT64_MAX &&
       pc_tree->index != 3) {
     int output_enabled = (bsize == BLOCK_64X64);
@@ -3057,13 +3056,19 @@ static void encode_superblock(VP10_COMP *cpi, ThreadData *td,
     ++td->counts->tx.tx_totals[mbmi->tx_size];
     ++td->counts->tx.tx_totals[get_uv_tx_size(mbmi, &xd->plane[1])];
 #if CONFIG_EXT_TX
-    if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
-        bsize >= BLOCK_8X8 && !mbmi->skip &&
+    if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter_block(mbmi)) > 1 &&
+        cm->base_qindex > 0 && !mbmi->skip &&
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
-      if (is_inter_block(mbmi))
-        ++td->counts->inter_tx_type[mbmi->tx_size][mbmi->tx_type];
-      else
-        ++td->counts->intra_tx_type[mbmi->tx_size][mbmi->mode][mbmi->tx_type];
+      int eset = get_ext_tx_set(mbmi->tx_size, bsize,
+                                is_inter_block(mbmi));
+      if (eset > 0) {
+        if (is_inter_block(mbmi)) {
+          ++td->counts->inter_ext_tx[eset][mbmi->tx_size][mbmi->tx_type];
+        } else {
+          ++td->counts->intra_ext_tx[eset][mbmi->tx_size][mbmi->mode]
+              [mbmi->tx_type];
+        }
+      }
     }
 #endif  // CONFIG_EXT_TX
   }

diff --git a/vp10/encoder/encodemb.c b/vp10/encoder/encodemb.c
index 7a31dc78f537189855569d649a404a68c39a098c..966d05c4ddae4e768aded78e0cf22b7d67daff5b 100644 (file)
--- a/vp10/encoder/encodemb.c
+++ b/vp10/encoder/encodemb.c
@@ -709,6 +709,11 @@ static void fwd_txfm_32x32(int rd_transform, const int16_t *src_diff,
     case DCT_DCT:
       fdct32x32(rd_transform, src_diff, coeff, diff_stride);
       break;
+#if CONFIG_EXT_TX
+    case IDTX:
+      fwd_idtx_c(src_diff, coeff, diff_stride, 32);
+      break;
+#endif  // CONFIG_EXT_TX
     case ADST_DCT:
     case DCT_ADST:
     case ADST_ADST:
@@ -727,6 +732,11 @@ static void fwd_txfm_32x32_1(const int16_t *src_diff,
     case DCT_DCT:
       vpx_fdct32x32_1(src_diff, coeff, diff_stride);
       break;
+#if CONFIG_EXT_TX
+    case IDTX:
+      fwd_idtx_c(src_diff, coeff, diff_stride, 32);
+      break;
+#endif  // CONFIG_EXT_TX
     case ADST_DCT:
     case DCT_ADST:
     case ADST_ADST:
@@ -1056,6 +1066,11 @@ static void highbd_fwd_txfm_32x32(int rd_transform, const int16_t *src_diff,
     case DCT_DCT:
       highbd_fdct32x32(rd_transform, src_diff, coeff, diff_stride);
       break;
+#if CONFIG_EXT_TX
+    case IDTX:
+      fwd_idtx_c(src_diff, coeff, diff_stride, 32);
+      break;
+#endif  // CONFIG_EXT_TX
     case ADST_DCT:
     case DCT_ADST:
     case ADST_ADST:
@@ -1074,6 +1089,11 @@ static void highbd_fwd_txfm_32x32_1(const int16_t *src_diff,
     case DCT_DCT:
       vpx_highbd_fdct32x32_1(src_diff, coeff, diff_stride);
       break;
+#if CONFIG_EXT_TX
+    case IDTX:
+      fwd_idtx_c(src_diff, coeff, diff_stride, 32);
+      break;
+#endif  // CONFIG_EXT_TX
     case ADST_DCT:
     case DCT_ADST:
     case ADST_ADST:

diff --git a/vp10/encoder/encoder.h b/vp10/encoder/encoder.h
index d399932f19c034f0c67f96ccf2e118ba8a7162d1..d63c123d32bdee4577a32a9a2692b811afeef971 100644 (file)
--- a/vp10/encoder/encoder.h
+++ b/vp10/encoder/encoder.h
@@ -467,8 +467,9 @@ typedef struct VP10_COMP {
   int palette_uv_color_cost[PALETTE_MAX_SIZE - 1][PALETTE_COLOR_CONTEXTS]
                                                   [PALETTE_COLORS];
 #if CONFIG_EXT_TX
-  int inter_tx_type_costs[EXT_TX_SIZES][TX_TYPES];
-  int intra_tx_type_costs[EXT_TX_SIZES][INTRA_MODES][TX_TYPES];
+  int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
+  int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
+                                                          [TX_TYPES];
 #endif  // CONFIG_EXT_TX
 
   int multi_arf_allowed;

diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c
index 963ff5071bb00a0ac1faab4dec6b464c3b3b6759..817eb0aed6a7a0316d104277e4fd0cb244574ece 100644 (file)
--- a/vp10/encoder/rd.c
+++ b/vp10/encoder/rd.c
@@ -102,12 +102,23 @@ static void fill_mode_costs(VP10_COMP *cpi) {
                        vp10_palette_color_tree[i]);
     }
 #if CONFIG_EXT_TX
-  for (i = TX_4X4; i <= TX_16X16; ++i) {
-    vp10_cost_tokens(cpi->inter_tx_type_costs[i], fc->inter_tx_type_prob[i],
-                     vp10_tx_type_tree);
-    for (j = 0; j < INTRA_MODES; ++j)
-      vp10_cost_tokens(cpi->intra_tx_type_costs[i][j],
-                       fc->intra_tx_type_prob[i][j], vp10_tx_type_tree);
+  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
+    int s;
+    for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
+      if (use_inter_ext_tx_for_tx[s][i]) {
+        vp10_cost_tokens(cpi->inter_tx_type_costs[s][i],
+                         fc->inter_ext_tx_prob[s][i],
+                         vp10_ext_tx_inter_tree[s]);
+      }
+    }
+    for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
+      if (use_intra_ext_tx_for_tx[s][i]) {
+        for (j = 0; j < INTRA_MODES; ++j)
+          vp10_cost_tokens(cpi->intra_tx_type_costs[s][i][j],
+                           fc->intra_ext_tx_prob[s][i][j],
+                           vp10_ext_tx_intra_tree[s]);
+      }
+    }
   }
 #endif  // CONFIG_EXT_TX
 }

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 223c67ae28e84b40e295633cbfe0d821f2f281f0..fc7a107d60ad86a3ea2a47141727450bbe3d0142 100644 (file)
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -668,28 +668,41 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
   MACROBLOCKD *const xd = &x->e_mbd;
   MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
 #if CONFIG_EXT_TX
-  int tx_type, best_tx_type = DCT_DCT;
+  TX_TYPE tx_type, best_tx_type = DCT_DCT;
   int r, s;
   int64_t d, psse, this_rd, best_rd = INT64_MAX;
   vpx_prob skip_prob = vp10_get_skip_prob(cm, xd);
   int  s0 = vp10_cost_bit(skip_prob, 0);
   int  s1 = vp10_cost_bit(skip_prob, 1);
+  int ext_tx_set;
+  const int is_inter = is_inter_block(mbmi);
 #endif  // CONFIG_EXT_TX
 
   mbmi->tx_size = VPXMIN(max_tx_size, largest_tx_size);
+
 #if CONFIG_EXT_TX
-  if (is_inter_block(mbmi) && bs >= BLOCK_8X8 &&
+  ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter);
+
+  if (is_inter &&
+      get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1 &&
       !xd->lossless[mbmi->segment_id]) {
-    for (tx_type = DCT_DCT; tx_type < TX_TYPES - 1; ++tx_type) {
-      if (mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX &&
+    for (tx_type = 0; tx_type < TX_TYPES; ++tx_type) {
+      if (is_inter) {
+        if (!ext_tx_used_inter[ext_tx_set][tx_type])
+          continue;
+      } else {
+        if (!ext_tx_used_intra[ext_tx_set][tx_type])
+          continue;
+      }
+
+      mbmi->tx_type = tx_type;
+      if (ext_tx_set == 1 &&
+          mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX &&
           best_tx_type == DCT_DCT) {
         tx_type = IDTX - 1;
         continue;
       }
-      if (tx_type >= GET_TX_TYPES(mbmi->tx_size))
-        continue;
 
-      mbmi->tx_type = tx_type;
       txfm_rd_in_plane(x,
 #if CONFIG_VAR_TX
                        cpi,
@@ -700,12 +713,16 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
 
       if (r == INT_MAX)
         continue;
-      if (mbmi->tx_size <= TX_16X16) {
-        if (is_inter_block(mbmi))
-          r += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
-        else
-          r += cpi->intra_tx_type_costs[mbmi->tx_size]
-                                        [mbmi->mode][mbmi->tx_type];
+      if (get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1) {
+        if (is_inter) {
+          if (ext_tx_set > 0)
+            r += cpi->inter_tx_type_costs[ext_tx_set]
+                                         [mbmi->tx_size][mbmi->tx_type];
+        } else {
+          if (ext_tx_set > 0)
+            r += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
+                                         [mbmi->mode][mbmi->tx_type];
+        }
       }
 
       if (s)
@@ -734,13 +751,15 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
                    mbmi->tx_size, cpi->sf.use_fast_coef_costing);
 
 #if CONFIG_EXT_TX
-  if (bs >= BLOCK_8X8 && mbmi->tx_size <= TX_16X16 &&
+  if (get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1 &&
       !xd->lossless[mbmi->segment_id] && *rate != INT_MAX) {
-    if (is_inter_block(mbmi))
-      *rate += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
+    int ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter);
+    if (is_inter)
+      *rate += cpi->inter_tx_type_costs[ext_tx_set][mbmi->tx_size]
+                                       [mbmi->tx_type];
     else
-      *rate += cpi->intra_tx_type_costs[mbmi->tx_size]
-                                        [mbmi->mode][mbmi->tx_type];
+      *rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
+                                       [mbmi->mode][mbmi->tx_type];
   }
 #endif  // CONFIG_EXT_TX
 }
@@ -786,9 +805,10 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
   int start_tx, end_tx;
   const int tx_select = cm->tx_mode == TX_MODE_SELECT;
 #if CONFIG_EXT_TX
-  int tx_type, best_tx_type = DCT_DCT;
-  int start_tx_type, end_tx_type;
+  TX_TYPE tx_type, best_tx_type = DCT_DCT;
+  int ext_tx_set;
 #endif  // CONFIG_EXT_TX
+  const int is_inter = is_inter_block(mbmi);
 
   const vpx_prob *tx_probs = get_tx_probs2(max_tx_size, xd, &cm->fc->tx_probs);
   assert(skip_prob > 0);
@@ -799,8 +819,8 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
     start_tx = max_tx_size;
     end_tx = 0;
   } else {
-    const TX_SIZE chosen_tx_size = VPXMIN(max_tx_size,
-                                    tx_mode_to_biggest_tx_size[cm->tx_mode]);
+    const TX_SIZE chosen_tx_size =
+        VPXMIN(max_tx_size, tx_mode_to_biggest_tx_size[cm->tx_mode]);
     start_tx = chosen_tx_size;
     end_tx = chosen_tx_size;
   }
@@ -811,29 +831,11 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
   *psse       = INT64_MAX;
 
 #if CONFIG_EXT_TX
-  start_tx_type = DCT_DCT;
-  if (bs >= BLOCK_8X8 && !xd->lossless[mbmi->segment_id])
-    end_tx_type = TX_TYPES - 1;
-  else
-    end_tx_type = DCT_DCT;
-
-  for (tx_type = start_tx_type; tx_type <= end_tx_type; ++tx_type) {
-    mbmi->tx_type = tx_type;
-    // TODO(huisu): clean up the logic.
-    if (mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX &&
-        best_tx_type == DCT_DCT) {
-      tx_type = IDTX - 1;
-      continue;
-    }
+  for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
 #endif  // CONFIG_EXT_TX
+    last_rd = INT64_MAX;
     for (n = start_tx; n >= end_tx; --n) {
       int r_tx_size = 0;
-
-#if CONFIG_EXT_TX
-      if (mbmi->tx_type >= GET_TX_TYPES(n))
-        continue;
-#endif  // CONFIG_EXT_TX
-
       for (m = 0; m <= n - (n == (int) max_tx_size); ++m) {
         if (m == n)
           r_tx_size += vp10_cost_zero(tx_probs[m]);
@@ -841,6 +843,22 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
           r_tx_size += vp10_cost_one(tx_probs[m]);
       }
 
+#if CONFIG_EXT_TX
+      ext_tx_set = get_ext_tx_set(n, bs, is_inter);
+      if (is_inter) {
+        if (!ext_tx_used_inter[ext_tx_set][tx_type])
+          continue;
+      } else {
+        if (!ext_tx_used_intra[ext_tx_set][tx_type])
+          continue;
+      }
+      mbmi->tx_type = tx_type;
+      if (ext_tx_set == 1 &&
+          mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX &&
+          best_tx_type == DCT_DCT) {
+        tx_type = IDTX - 1;
+        break;
+      }
       txfm_rd_in_plane(x,
 #if CONFIG_VAR_TX
                        cpi,
@@ -848,52 +866,65 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
                        &r, &d, &s,
                        &sse, ref_best_rd, 0, bs, n,
                        cpi->sf.use_fast_coef_costing);
-#if CONFIG_EXT_TX
-      if (bs >= BLOCK_8X8 && !xd->lossless[mbmi->segment_id] &&
-          r != INT_MAX && n < TX_32X32) {
-        if (is_inter_block(mbmi))
-          r += cpi->inter_tx_type_costs[n][mbmi->tx_type];
-        else
-          r += cpi->intra_tx_type_costs[n][mbmi->mode][mbmi->tx_type];
+      if (get_ext_tx_types(n, bs, is_inter) > 1 &&
+          !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+          r != INT_MAX) {
+        if (is_inter) {
+          if (ext_tx_set > 0)
+            r += cpi->inter_tx_type_costs[ext_tx_set]
+                                         [mbmi->tx_size][mbmi->tx_type];
+        } else {
+          if (ext_tx_set > 0)
+            r += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
+                                         [mbmi->mode][mbmi->tx_type];
+        }
       }
+#else  // CONFIG_EXT_TX
+      txfm_rd_in_plane(x,
+#if CONFIG_VAR_TX
+                       cpi,
+#endif
+                       &r, &d, &s,
+                       &sse, ref_best_rd, 0, bs, n,
+                       cpi->sf.use_fast_coef_costing);
 #endif  // CONFIG_EXT_TX
 
       if (r == INT_MAX)
         continue;
 
-      if (tx_select)
-        r += r_tx_size;
-
       if (s) {
-        if (is_inter_block(mbmi)) {
+        if (is_inter) {
           rd = RDCOST(x->rdmult, x->rddiv, s1, sse);
-          if (tx_select)
-            r -= r_tx_size;
         } else {
           rd =  RDCOST(x->rdmult, x->rddiv, s1 + r_tx_size * tx_select, sse);
         }
       } else {
-        rd = RDCOST(x->rdmult, x->rddiv, r + s0, d);
+        rd = RDCOST(x->rdmult, x->rddiv, r + s0 + r_tx_size * tx_select, d);
       }
 
-      if (is_inter_block(mbmi) && !xd->lossless[mbmi->segment_id] && !s)
+      if (tx_select && !(s && is_inter))
+        r += r_tx_size;
+
+      if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] && !s)
         rd = VPXMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, sse));
 
       // Early termination in transform size search.
-      if (0 && cpi->sf.tx_size_search_breakout &&
-          (rd== INT64_MAX ||
-              (n < (int) max_tx_size && rd > last_rd) ||
-              s == 1))
+      if (cpi->sf.tx_size_search_breakout &&
+          (rd == INT64_MAX ||
+#if CONFIG_EXT_TX
+           (s == 1 && tx_type != DCT_DCT) ||
+#else
+           (s == 1) ||
+#endif
+           (n < (int) max_tx_size && rd > last_rd)))
         break;
 
       last_rd = rd;
-#if CONFIG_EXT_TX
       if (rd <
-          (is_inter_block(mbmi) && best_tx_type == DCT_DCT ? ext_tx_th : 1) *
-          best_rd) {
-#else
-      if (rd < best_rd) {
+#if CONFIG_EXT_TX
+          (is_inter && best_tx_type == DCT_DCT ? ext_tx_th : 1) *
 #endif  // CONFIG_EXT_TX
+          best_rd) {
         best_tx = n;
         best_rd = rd;
         *distortion = d;
@@ -1170,7 +1201,7 @@ static int64_t rd_pick_intra4x4block(VP10_COMP *cpi, MACROBLOCK *x,
       // one of the neighboring directional modes
       if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
         if (conditional_skipintra(mode, *best_mode))
-            continue;
+          continue;
       }
 
       memcpy(tempa, ta, sizeof(ta));
@@ -1182,8 +1213,8 @@ static int64_t rd_pick_intra4x4block(VP10_COMP *cpi, MACROBLOCK *x,
           const uint8_t *const src = &src_init[idx * 4 + idy * 4 * src_stride];
           uint8_t *const dst = &dst_init[idx * 4 + idy * 4 * dst_stride];
           int16_t *const src_diff = vp10_raster_block_offset_int16(BLOCK_8X8,
-                                                                  block,
-                                                                  p->src_diff);
+                                                                   block,
+                                                                   p->src_diff);
           tran_low_t *const coeff = BLOCK_OFFSET(x->plane[0].coeff, block);
           xd->mi[0]->bmi[block].as_mode = mode;
           vp10_predict_intra_block(xd, 1, 1, TX_4X4, mode, dst, dst_stride,
@@ -1262,7 +1293,7 @@ static int64_t rd_pick_intra4x4block(VP10_COMP *cpi, MACROBLOCK *x,
                  num_4x4_blocks_wide * 4 * sizeof(uint16_t));
         }
       }
-    next_highbd:
+next_highbd:
       {}
     }
     if (best_rd >= rd_thresh)
@@ -1291,7 +1322,7 @@ static int64_t rd_pick_intra4x4block(VP10_COMP *cpi, MACROBLOCK *x,
     // one of the neighboring directional modes
     if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
       if (conditional_skipintra(mode, *best_mode))
-          continue;
+        continue;
     }
 
     memcpy(tempa, ta, sizeof(ta));