Extend ext_tx experiment to intra blocks

ext-tx on derflr +2.30% (was +1.84%)

Change-Id: Ic91565cacc38e7a8e1200d054ed7bf99295fe19e

diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index 715d2e6fd664b624536aa4ce7386ff78cf1ba3d1..d01067a32c52678735f7e9295e6c087c9f47c542 100644 (file)
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -269,11 +269,16 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type,
 #if CONFIG_EXT_TX
   if (xd->lossless || tx_size >= TX_32X32)
     return DCT_DCT;
-  if (is_inter_block(&mi->mbmi)) {
-    return ext_tx_to_txtype[mi->mbmi.ext_txfrm];
+  if (mbmi->sb_type >= BLOCK_8X8) {
+    if (plane_type == PLANE_TYPE_Y || is_inter_block(mbmi))
+      return ext_tx_to_txtype[mbmi->ext_txfrm];
   }
-  return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ?
-      get_y_mode(mi, block_idx) : mbmi->uv_mode];
+
+  if (is_inter_block(mbmi))
+    return DCT_DCT;
+  else
+    return intra_mode_to_tx_type_lookup[plane_type == PLANE_TYPE_Y ?
+        get_y_mode(mi, block_idx) : mbmi->uv_mode];
 #else
   if (plane_type != PLANE_TYPE_Y || xd->lossless || is_inter_block(mbmi) ||
       tx_size >= TX_32X32)

diff --git a/vp10/common/entropymode.c b/vp10/common/entropymode.c
index 5bbe045efe9f4de7b369d1f3fcc9b233c48b4704..8f59fd8f21e96e4a0973c54a8338008b5a6509f8 100644 (file)
--- a/vp10/common/entropymode.c
+++ b/vp10/common/entropymode.c
@@ -334,7 +334,8 @@ const vpx_tree_index vp10_ext_tx_tree[TREE_SIZE(EXT_TX_TYPES)] = {
   -ALT7, -ALT8
 };
 
-static const vpx_prob default_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = {
+static const vpx_prob
+default_inter_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = {
   {  12, 112,  16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
     128 },
   {  12, 112,  16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
@@ -342,6 +343,76 @@ static const vpx_prob default_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1] = {
   {  12, 112,  16, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
     128 },
 };
+
+static const vpx_prob
+default_intra_ext_tx_prob[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES - 1] = {
+    {
+        {   8,  11,  24, 112,  87, 137, 127, 134,
+          128,  86, 128, 124, 125, 133, 176, 123, },
+        {  10,   9,  39, 106,  73, 155, 163, 228,
+           35,  62, 129, 127, 133, 114, 213, 234, },
+        {  10,   9,  14,  88,  91, 127, 151,  51,
+          210,  89, 126,  58,  52, 116, 217,  24, },
+        {   9,   6,  29, 113,  98, 131, 149, 210,
+          119,  60, 124,  93,  90, 143, 170, 197, },
+        {   8,   8,  38, 101, 111, 166, 167, 141,
+          130, 105, 128,  75,  75, 118, 197, 117, },
+        {   7,   8,  39,  91, 101, 153, 166, 200,
+           99,  77, 123,  90,  83, 144, 224, 192, },
+        {   7,  10,  26,  86, 119, 154, 130, 101,
+          152,  91, 129,  75,  79, 137, 219,  77, },
+        {  10,  13,  20,  86, 102, 162, 112,  76,
+          171,  86, 134, 122, 106, 124, 196,  44, },
+        {   8,   9,  33, 108, 100, 144, 148, 215,
+           77,  60, 125, 125, 128, 126, 198, 220, },
+        {   3,  10,  29, 111,  69, 141, 204, 141,
+          139,  93, 120,  75,  77, 163, 242, 124, },
+    },
+    {
+        {   2,  53,  18, 147,  96,  98, 136, 133,
+          131, 120, 153, 163, 169, 137, 173, 124, },
+        {   4,  18,  34, 133,  54, 130, 179, 228,
+           28,  72, 153, 164, 168, 118, 227, 239, },
+        {   4,  18,  13, 125,  72, 110, 176,  36,
+          221, 104, 148,  75,  72, 117, 225,  19, },
+        {   8,  33,  24, 162, 113,  99, 147, 226,
+          103,  85, 153, 143, 153, 124, 155, 210, },
+        {   2,  15,  35, 107, 127, 158, 192, 128,
+          126, 116, 151,  95,  88, 182, 241, 119, },
+        {   3,  15,  36, 112, 100, 146, 194, 189,
+           90,  98, 152,  99, 100, 165, 235, 175, },
+        {   3,  16,  29, 109, 103, 140, 182,  76,
+          173, 104, 147,  82,  85, 159, 235,  70, },
+        {   9,  24,  14, 120,  86, 156, 161,  34,
+          177, 121, 142, 128, 128, 126, 185,  37, },
+        {   5,  24,  29, 152,  98,  99, 174, 228,
+           82,  76, 147, 149, 128, 132, 191, 225, },
+        {   2,  15,  29, 111,  77, 126, 200, 135,
+          117,  93, 152,  96,  84, 191, 245, 135, },
+    },
+    {
+        {   2,  69,  13, 173, 111,  69, 137, 159,
+          159, 146, 151, 193, 203, 131, 180, 123, },
+        {   1,  12,  33, 164,  32,  98, 204, 242,
+           23,  99, 149, 215, 232, 110, 239, 245, },
+        {   1,  17,   9, 136,  82,  83, 171,  28,
+          231, 128, 135,  76,  64, 118, 235,  17, },
+        {   4,  41,  17, 195, 131,  58, 161, 237,
+          141,  97, 153, 189, 191, 117, 182, 202, },
+        {   2,  17,  36, 104, 149, 137, 217, 139,
+          191, 119, 125, 107, 115, 223, 249, 110, },
+        {   2,  14,  24, 127,  91, 135, 219, 198,
+          113,  91, 164, 125, 173, 211, 250, 116, },
+        {   3,  19,  24, 120, 102, 130, 209,  81,
+          187,  95, 143, 102,  50, 190, 244,  56, },
+        {   4,  27,  10, 128,  91, 157, 181,  33,
+          181, 150, 141, 141, 166, 114, 215,  25, },
+        {   2,  34,  27, 187, 102,  77, 210, 245,
+          113, 107, 136, 184, 188, 121, 210, 234, },
+        {   1,  15,  22, 141,  59,  94, 208, 133,
+          154,  95, 152, 112, 105, 191, 242, 111, },
+    },
+};
 #endif  // CONFIG_EXT_TX
 
 static void init_mode_probs(FRAME_CONTEXT *fc) {
@@ -357,7 +428,8 @@ static void init_mode_probs(FRAME_CONTEXT *fc) {
   vp10_copy(fc->skip_probs, default_skip_probs);
   vp10_copy(fc->inter_mode_probs, default_inter_mode_probs);
 #if CONFIG_EXT_TX
-  vp10_copy(fc->ext_tx_prob, default_ext_tx_prob);
+  vp10_copy(fc->inter_ext_tx_prob, default_inter_ext_tx_prob);
+  vp10_copy(fc->intra_ext_tx_prob, default_intra_ext_tx_prob);
 #endif  // CONFIG_EXT_TX
 }
 
@@ -441,8 +513,13 @@ void vp10_adapt_mode_probs(VP10_COMMON *cm) {
 
 #if CONFIG_EXT_TX
   for (i = TX_4X4; i <= TX_16X16; ++i) {
-    vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->ext_tx_prob[i],
-                         counts->ext_tx[i], fc->ext_tx_prob[i]);
+    vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->inter_ext_tx_prob[i],
+                         counts->inter_ext_tx[i], fc->inter_ext_tx_prob[i]);
+
+    for (j = 0; j < INTRA_MODES; ++j)
+      vpx_tree_merge_probs(vp10_ext_tx_tree, pre_fc->intra_ext_tx_prob[i][j],
+                           counts->intra_ext_tx[i][j],
+                           fc->intra_ext_tx_prob[i][j]);
   }
 #endif  // CONFIG_EXT_TX
 }

diff --git a/vp10/common/entropymode.h b/vp10/common/entropymode.h
index bdae9ee6fc146951cce25ad1b3ad9707fde8ceba..edd9b3a0a0b6779bdf284dceaed713804fe864c1 100644 (file)
--- a/vp10/common/entropymode.h
+++ b/vp10/common/entropymode.h
@@ -57,7 +57,8 @@ typedef struct frame_contexts {
   vpx_prob skip_probs[SKIP_CONTEXTS];
   nmv_context nmvc;
 #if CONFIG_EXT_TX
-  vpx_prob ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1];
+  vpx_prob inter_ext_tx_prob[EXT_TX_SIZES][EXT_TX_TYPES - 1];
+  vpx_prob intra_ext_tx_prob[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES - 1];
 #endif  // CONFIG_EXT_TX
   int initialized;
 } FRAME_CONTEXT;
@@ -80,7 +81,8 @@ typedef struct FRAME_COUNTS {
   unsigned int skip[SKIP_CONTEXTS][2];
   nmv_context_counts mv;
 #if CONFIG_EXT_TX
-  unsigned int ext_tx[EXT_TX_SIZES][EXT_TX_TYPES];
+  unsigned int inter_ext_tx[EXT_TX_SIZES][EXT_TX_TYPES];
+  unsigned int intra_ext_tx[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES];
 #endif  // CONFIG_EXT_TX
 } FRAME_COUNTS;
 

diff --git a/vp10/decoder/decodeframe.c b/vp10/decoder/decodeframe.c
index a5551299fe8e1530a8feee06f4aca8fbd77a1354..4797d32d21db24ff73c5a2b01cd1a1ca4cb3cf41 100644 (file)
--- a/vp10/decoder/decodeframe.c
+++ b/vp10/decoder/decodeframe.c
@@ -2070,11 +2070,18 @@ static size_t read_uncompressed_header(VP10Decoder *pbi,
 
 #if CONFIG_EXT_TX
 static void read_ext_tx_probs(FRAME_CONTEXT *fc, vpx_reader *r) {
-  int i, j;
+  int i, j, k;
+  if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
+    for (i = TX_4X4; i <= TX_16X16; ++i)
+      for (j = 0; j < EXT_TX_TYPES - 1; ++j)
+        vp10_diff_update_prob(r, &fc->inter_ext_tx_prob[i][j]);
+  }
+
   if (vpx_read(r, GROUP_DIFF_UPDATE_PROB)) {
-    for (j = TX_4X4; j <= TX_16X16; ++j)
-      for (i = 0; i < EXT_TX_TYPES - 1; ++i)
-        vp10_diff_update_prob(r, &fc->ext_tx_prob[j][i]);
+    for (i = TX_4X4; i <= TX_16X16; ++i)
+      for (j = 0; j < INTRA_MODES; ++j)
+        for (k = 0; k < EXT_TX_TYPES - 1; ++k)
+          vp10_diff_update_prob(r, &fc->intra_ext_tx_prob[i][j][k]);
   }
 }
 #endif  // CONFIG_EXT_TX

diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c
index 4d8a1eba36f961be0b8759b4e761f4da9e568b93..beb29a0656c0f4383e8f24eb0a7ae912b8de6e81 100644 (file)
--- a/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@ -244,6 +244,18 @@ static void read_intra_frame_mode_info(VP10_COMMON *const cm,
   }
 
   mbmi->uv_mode = read_intra_mode(r, vp10_kf_uv_mode_prob[mbmi->mode]);
+
+#if CONFIG_EXT_TX
+    if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+        mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip &&
+        !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+      mbmi->ext_txfrm =
+          vpx_read_tree(r, vp10_ext_tx_tree,
+                        cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode]);
+    } else {
+      mbmi->ext_txfrm = NORM;
+    }
+#endif  // CONFIG_EXT_TX
 }
 
 static int read_mv_component(vpx_reader *r,
@@ -595,28 +607,37 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi,
   mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r);
   inter_block = read_is_inter_block(cm, xd, mbmi->segment_id, r);
   mbmi->tx_size = read_tx_size(cm, xd, !mbmi->skip || !inter_block, r);
+
+  if (inter_block)
+    read_inter_block_mode_info(pbi, xd, mi, mi_row, mi_col, r);
+  else
+    read_intra_block_mode_info(cm, xd, mi, r);
+
 #if CONFIG_EXT_TX
-    if (inter_block &&
-        mbmi->tx_size <= TX_16X16 &&
-        cm->base_qindex > 0 &&
-        mbmi->sb_type >= BLOCK_8X8 &&
-        !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) &&
-        !mbmi->skip) {
+    if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+        mbmi->sb_type >= BLOCK_8X8 && !mbmi->skip &&
+        !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
       FRAME_COUNTS *counts = xd->counts;
-      mbmi->ext_txfrm = vpx_read_tree(r,
-                                      vp10_ext_tx_tree,
-                                      cm->fc->ext_tx_prob[mbmi->tx_size]);
-      if (counts)
-        ++counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
+
+      if (inter_block)
+        mbmi->ext_txfrm =
+            vpx_read_tree(r, vp10_ext_tx_tree,
+                          cm->fc->inter_ext_tx_prob[mbmi->tx_size]);
+      else
+        mbmi->ext_txfrm =
+            vpx_read_tree(r, vp10_ext_tx_tree,
+                          cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode]);
+
+      if (counts) {
+        if (inter_block)
+          ++counts->inter_ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
+        else
+          ++counts->intra_ext_tx[mbmi->tx_size][mbmi->mode][mbmi->ext_txfrm];
+      }
     } else {
       mbmi->ext_txfrm = NORM;
     }
 #endif  // CONFIG_EXT_TX
-
-  if (inter_block)
-    read_inter_block_mode_info(pbi, xd, mi, mi_row, mi_col, r);
-  else
-    read_intra_block_mode_info(cm, xd, mi, r);
 }
 
 void vp10_read_mode_info(VP10Decoder *const pbi, MACROBLOCKD *xd,

diff --git a/vp10/encoder/bitstream.c b/vp10/encoder/bitstream.c
index 85057258f14b0c2404468ffb521de13773826283..f8c274e14d698ca549a3c9e5dc35090035ba4dde 100644 (file)
--- a/vp10/encoder/bitstream.c
+++ b/vp10/encoder/bitstream.c
@@ -164,22 +164,39 @@ static void update_switchable_interp_probs(VP10_COMMON *cm, vpx_writer *w,
 static void update_ext_tx_probs(VP10_COMMON *cm, vpx_writer *w) {
   const int savings_thresh = vp10_cost_one(GROUP_DIFF_UPDATE_PROB) -
                              vp10_cost_zero(GROUP_DIFF_UPDATE_PROB);
-  int i;
+  int i, j;
   int savings = 0;
   int do_update = 0;
   for (i = TX_4X4; i <= TX_16X16; ++i) {
     savings += prob_diff_update_savings(
-        vp10_ext_tx_tree, cm->fc->ext_tx_prob[i],
-        cm->counts.ext_tx[i], EXT_TX_TYPES);
+        vp10_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
+        cm->counts.inter_ext_tx[i], EXT_TX_TYPES);
   }
   do_update = savings > savings_thresh;
   vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
   if (do_update) {
     for (i = TX_4X4; i <= TX_16X16; ++i) {
-      prob_diff_update(vp10_ext_tx_tree, cm->fc->ext_tx_prob[i],
-                       cm->counts.ext_tx[i], EXT_TX_TYPES, w);
+      prob_diff_update(vp10_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
+                       cm->counts.inter_ext_tx[i], EXT_TX_TYPES, w);
     }
   }
+
+  savings = 0;
+  do_update = 0;
+
+  for (i = TX_4X4; i <= TX_16X16; ++i)
+    for (j = 0; j < INTRA_MODES; ++j)
+      savings += prob_diff_update_savings(
+          vp10_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
+          cm->counts.intra_ext_tx[i][j], EXT_TX_TYPES);
+  do_update = savings > savings_thresh;
+  vpx_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
+  if (do_update) {
+    for (i = TX_4X4; i <= TX_16X16; ++i)
+      for (j = 0; j < INTRA_MODES; ++j)
+        prob_diff_update(vp10_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
+                         cm->counts.intra_ext_tx[i][j], EXT_TX_TYPES, w);
+  }
 }
 #endif  // CONFIG_EXT_TX
 
@@ -337,18 +354,6 @@ static void pack_inter_mode_mvs(VP10_COMP *cpi, const MODE_INFO *mi,
     write_selected_tx_size(cm, xd, w);
   }
 
-#if CONFIG_EXT_TX
-  if (is_inter &&
-      mbmi->tx_size <= TX_16X16 &&
-      cm->base_qindex > 0 &&
-      bsize >= BLOCK_8X8 &&
-      !mbmi->skip &&
-      !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
-    vp10_write_token(w, vp10_ext_tx_tree, cm->fc->ext_tx_prob[mbmi->tx_size],
-                     &ext_tx_encodings[mbmi->ext_txfrm]);
-  }
-#endif  // CONFIG_EXT_TX
-
   if (!is_inter) {
     if (bsize >= BLOCK_8X8) {
       write_intra_mode(w, mode, cm->fc->y_mode_prob[size_group_lookup[bsize]]);
@@ -412,6 +417,21 @@ static void pack_inter_mode_mvs(VP10_COMP *cpi, const MODE_INFO *mi,
       }
     }
   }
+
+#if CONFIG_EXT_TX
+  if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+      bsize >= BLOCK_8X8 && !mbmi->skip &&
+      !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+    if (is_inter)
+      vp10_write_token(w, vp10_ext_tx_tree,
+                       cm->fc->inter_ext_tx_prob[mbmi->tx_size],
+                       &ext_tx_encodings[mbmi->ext_txfrm]);
+    else
+      vp10_write_token(w, vp10_ext_tx_tree,
+                       cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode],
+                       &ext_tx_encodings[mbmi->ext_txfrm]);
+  }
+#endif  // CONFIG_EXT_TX
 }
 
 static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd,
@@ -448,6 +468,16 @@ static void write_mb_modes_kf(const VP10_COMMON *cm, const MACROBLOCKD *xd,
   }
 
   write_intra_mode(w, mbmi->uv_mode, vp10_kf_uv_mode_prob[mbmi->mode]);
+
+#if CONFIG_EXT_TX
+  if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+      bsize >= BLOCK_8X8 && !mbmi->skip &&
+      !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+    vp10_write_token(w, vp10_ext_tx_tree,
+                     cm->fc->intra_ext_tx_prob[mbmi->tx_size][mbmi->mode],
+                     &ext_tx_encodings[mbmi->ext_txfrm]);
+  }
+#endif  // CONFIG_EXT_TX
 }
 
 static void write_modes_b(VP10_COMP *cpi, const TileInfo *const tile,

diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index ceb9eb471c1cd93787e6629ce523468456d862d0..356baf586c037c58035f72cee9733145b1a4407e 100644 (file)
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -2989,13 +2989,13 @@ static void encode_superblock(VP10_COMP *cpi, ThreadData *td,
     ++td->counts->tx.tx_totals[mbmi->tx_size];
     ++td->counts->tx.tx_totals[get_uv_tx_size(mbmi, &xd->plane[1])];
 #if CONFIG_EXT_TX
-    if (is_inter_block(mbmi) &&
-        mbmi->tx_size <= TX_16X16 &&
-        cm->base_qindex > 0 &&
-        bsize >= BLOCK_8X8 &&
-        !mbmi->skip &&
+    if (mbmi->tx_size <= TX_16X16 && cm->base_qindex > 0 &&
+        bsize >= BLOCK_8X8 && !mbmi->skip &&
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
-      ++td->counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
+      if (is_inter_block(mbmi))
+        ++td->counts->inter_ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
+      else
+        ++td->counts->intra_ext_tx[mbmi->tx_size][mbmi->mode][mbmi->ext_txfrm];
     }
 #endif  // CONFIG_EXT_TX
   }

diff --git a/vp10/encoder/encoder.h b/vp10/encoder/encoder.h
index e9779103bf62cbb6818ae0e0a4c6ff2170faf1ba..0f8076db0aa24847b89614630c0c87fdd9d3fe31 100644 (file)
--- a/vp10/encoder/encoder.h
+++ b/vp10/encoder/encoder.h
@@ -459,7 +459,8 @@ typedef struct VP10_COMP {
   int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
   int partition_cost[PARTITION_CONTEXTS][PARTITION_TYPES];
 #if CONFIG_EXT_TX
-  int ext_tx_costs[EXT_TX_SIZES][EXT_TX_TYPES];
+  int inter_ext_tx_costs[EXT_TX_SIZES][EXT_TX_TYPES];
+  int intra_ext_tx_costs[EXT_TX_SIZES][INTRA_MODES][EXT_TX_TYPES];
 #endif  // CONFIG_EXT_TX
 
   int multi_arf_allowed;

diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c
index 82dfd231086439b7da7aa51190a8329d5bbe3a97..2b549af9655a33cad583e858d6ac67e4d8892eb2 100644 (file)
--- a/vp10/encoder/rd.c
+++ b/vp10/encoder/rd.c
@@ -85,9 +85,13 @@ static void fill_mode_costs(VP10_COMP *cpi) {
     vp10_cost_tokens(cpi->switchable_interp_costs[i],
                     fc->switchable_interp_prob[i], vp10_switchable_interp_tree);
 #if CONFIG_EXT_TX
-  for (i = TX_4X4; i <= TX_16X16; ++i)
-    vp10_cost_tokens(cpi->ext_tx_costs[i], fc->ext_tx_prob[i],
+  for (i = TX_4X4; i <= TX_16X16; ++i) {
+    vp10_cost_tokens(cpi->inter_ext_tx_costs[i], fc->inter_ext_tx_prob[i],
                      vp10_ext_tx_tree);
+    for (j = 0; j < INTRA_MODES; ++j)
+      vp10_cost_tokens(cpi->intra_ext_tx_costs[i][j],
+                       fc->intra_ext_tx_prob[i][j], vp10_ext_tx_tree);
+  }
 #endif  // CONFIG_EXT_TX
 }
 

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 1f1405e8e1515dff079d26ba0ec43eba14f80e95..bec1103ebd826d0774f3cdcb2759d6ee6d125634 100644 (file)
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -603,6 +603,11 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
 #if CONFIG_EXT_TX
   if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && !xd->lossless) {
     for (tx_type = NORM; tx_type < EXT_TX_TYPES - 1; ++tx_type) {
+      if (mbmi->ext_txfrm >= ALT11 && mbmi->ext_txfrm < ALT16 &&
+          best_tx_type == NORM) {
+        tx_type = ALT16 - 1;
+        continue;
+      }
       if (tx_type >= GET_EXT_TX_TYPES(mbmi->tx_size))
         continue;
 
@@ -613,8 +618,13 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
 
       if (r == INT_MAX)
         continue;
-      if (mbmi->tx_size <= TX_16X16)
-        r += cpi->ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
+      if (mbmi->tx_size <= TX_16X16) {
+        if (is_inter_block(mbmi))
+          r += cpi->inter_ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
+        else
+          r += cpi->intra_ext_tx_costs[mbmi->tx_size]
+                                       [mbmi->mode][mbmi->ext_txfrm];
+      }
 
       if (s)
         this_rd = RDCOST(x->rdmult, x->rddiv, s1, psse);
@@ -638,10 +648,14 @@ static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
                    mbmi->tx_size, cpi->sf.use_fast_coef_costing);
 
 #if CONFIG_EXT_TX
-  if (is_inter_block(mbmi) && bs >= BLOCK_8X8 &&
-      mbmi->tx_size <= TX_16X16 &&
-      !xd->lossless && *rate != INT_MAX)
-    *rate += cpi->ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
+  if (bs >= BLOCK_8X8 && mbmi->tx_size <= TX_16X16 &&
+      !xd->lossless && *rate != INT_MAX) {
+    if (is_inter_block(mbmi))
+      *rate += cpi->inter_ext_tx_costs[mbmi->tx_size][mbmi->ext_txfrm];
+    else
+      *rate += cpi->intra_ext_tx_costs[mbmi->tx_size]
+                                       [mbmi->mode][mbmi->ext_txfrm];
+  }
 #endif  // CONFIG_EXT_TX
 }
 
@@ -693,7 +707,7 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
 
 #if CONFIG_EXT_TX
   start_tx_type = NORM;
-  if (is_inter_block(mbmi) && bs >= BLOCK_8X8 && !xd->lossless)
+  if (bs >= BLOCK_8X8 && !xd->lossless)
     end_tx_type = EXT_TX_TYPES - 1;
   else
     end_tx_type = NORM;
@@ -726,9 +740,12 @@ static void choose_tx_size_from_rd(VP10_COMP *cpi, MACROBLOCK *x,
                        &sse, ref_best_rd, 0, bs, n,
                        cpi->sf.use_fast_coef_costing);
 #if CONFIG_EXT_TX
-      if (is_inter_block(mbmi) && bs >= BLOCK_8X8 &&
-          !xd->lossless && r != INT_MAX && n < TX_32X32)
-        r += cpi->ext_tx_costs[n][mbmi->ext_txfrm];
+      if (bs >= BLOCK_8X8 && !xd->lossless && r != INT_MAX && n < TX_32X32) {
+        if (is_inter_block(mbmi))
+          r += cpi->inter_ext_tx_costs[n][mbmi->ext_txfrm];
+        else
+          r += cpi->intra_ext_tx_costs[n][mbmi->mode][mbmi->ext_txfrm];
+      }
 #endif  // CONFIG_EXT_TX
 
       if (r == INT_MAX)
@@ -1136,6 +1153,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x,
   int this_rate, this_rate_tokenonly, s;
   int64_t this_distortion, this_rd;
   TX_SIZE best_tx = TX_4X4;
+#if CONFIG_EXT_TX
+  EXT_TX_TYPE best_tx_type = NORM;
+#endif  // CONFIG_EXT_TX
   int *bmode_costs;
   const MODE_INFO *above_mi = xd->above_mi;
   const MODE_INFO *left_mi = xd->left_mi;
@@ -1161,6 +1181,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x,
       mode_selected   = mode;
       best_rd         = this_rd;
       best_tx         = mic->mbmi.tx_size;
+#if CONFIG_EXT_TX
+      best_tx_type    = mic->mbmi.ext_txfrm;
+#endif  // CONFIG_EXT_TX
       *rate           = this_rate;
       *rate_tokenonly = this_rate_tokenonly;
       *distortion     = this_distortion;
@@ -1170,6 +1193,9 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x,
 
   mic->mbmi.mode = mode_selected;
   mic->mbmi.tx_size = best_tx;
+#if CONFIG_EXT_TX
+  mic->mbmi.ext_txfrm = best_tx_type;
+#endif  // CONFIG_EXT_TX
 
   return best_rd;
 }
@@ -3295,9 +3321,6 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
     if (ref_frame == INTRA_FRAME) {
       TX_SIZE uv_tx;
       struct macroblockd_plane *const pd = &xd->plane[1];
-#if CONFIG_EXT_TX
-      mbmi->ext_txfrm = NORM;
-#endif  // CONFIG_EXT_TX
       memset(x->skip_txfm, 0, sizeof(x->skip_txfm));
       super_block_yrd(cpi, x, &rate_y, &distortion_y, &skippable,
                       NULL, bsize, best_rd);
@@ -3322,9 +3345,6 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
         rate2 += intra_cost_penalty;
       distortion2 = distortion_y + distortion_uv;
     } else {
-#if CONFIG_EXT_TX
-      mbmi->ext_txfrm = NORM;
-#endif
       this_rd = handle_inter_mode(cpi, x, bsize,
                                   &rate2, &distortion2, &skippable,
                                   &rate_y, &rate_uv,
@@ -3755,9 +3775,6 @@ void vp10_rd_pick_inter_mode_sub8x8(VP10_COMP *cpi,
   int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS];
   int internal_active_edge =
     vp10_active_edge_sb(cpi, mi_row, mi_col) && vp10_internal_image_edge(cpi);
-#if CONFIG_EXT_TX
-  mbmi->ext_txfrm = NORM;
-#endif
 
   memset(x->zcoeff_blk[TX_4X4], 0, 4);
   vp10_zero(best_mbmode);
@@ -3916,9 +3933,6 @@ void vp10_rd_pick_inter_mode_sub8x8(VP10_COMP *cpi,
 
     if (ref_frame == INTRA_FRAME) {
       int rate;
-#if CONFIG_EXT_TX
-      mbmi->ext_txfrm = NORM;
-#endif
       if (rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate, &rate_y,
                                        &distortion_y, best_rd) >= best_rd)
         continue;