Migrate EXT_TX from playground to nextgen.

Change-Id: I1cb0584104323fb3781e66bb65d44ebbe853c9c8

diff --git a/configure b/configure
index 0621b516491f7bf7b38fbba79cee3571fa6d678e..7a7946a7f8cc9ee31eb1f39df3cfb8e5cacb661a 100755 (executable)
--- a/configure
+++ b/configure
@@ -284,6 +284,7 @@ EXPERIMENT_LIST="
     emulate_hardware
     tx64x64
     filterintra
+    ext_tx
 "
 CONFIG_LIST="
     external_build

diff --git a/vp9/common/vp9_blockd.h b/vp9/common/vp9_blockd.h
index 008524e13e354a4fadd3c802b7569b50e8344cb8..95e71808a8c2964a0d32cb1e2982d61e30cdd5cb 100644 (file)
--- a/vp9/common/vp9_blockd.h
+++ b/vp9/common/vp9_blockd.h
@@ -129,6 +129,10 @@ typedef struct {
   int_mv ref_mvs[MAX_REF_FRAMES][MAX_MV_REF_CANDIDATES];
   uint8_t mode_context[MAX_REF_FRAMES];
   INTERP_FILTER interp_filter;
+
+#if CONFIG_EXT_TX
+  EXT_TX_TYPE ext_txfrm;
+#endif
 } MB_MODE_INFO;
 
 typedef struct MODE_INFO {
@@ -147,11 +151,12 @@ static INLINE PREDICTION_MODE get_y_mode(const MODE_INFO *mi, int block) {
 
 #if CONFIG_FILTERINTRA
 static INLINE int is_filter_allowed(PREDICTION_MODE mode) {
+  (void)mode;
   return 1;
 }
 
 static INLINE int is_filter_enabled(TX_SIZE txsize) {
-  return (txsize <= TX_32X32);
+  return (txsize < TX_SIZES);
 }
 #endif
 
@@ -256,8 +261,20 @@ static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type,
                                   const MACROBLOCKD *xd) {
   const MB_MODE_INFO *const mbmi = &xd->mi[0].src_mi->mbmi;
 
+#if CONFIG_EXT_TX
+  if (plane_type != PLANE_TYPE_Y)
+      return DCT_DCT;
+
+  if (is_inter_block(mbmi)) {
+    if (mbmi->ext_txfrm == NORM || mbmi->tx_size >= TX_32X32)
+      return DCT_DCT;
+    else
+      return ADST_ADST;
+  }
+#else
   if (plane_type != PLANE_TYPE_Y || is_inter_block(mbmi))
     return DCT_DCT;
+#endif
   return intra_mode_to_tx_type_lookup[mbmi->mode];
 }
 
@@ -265,8 +282,20 @@ static INLINE TX_TYPE get_tx_type_4x4(PLANE_TYPE plane_type,
                                       const MACROBLOCKD *xd, int ib) {
   const MODE_INFO *const mi = xd->mi[0].src_mi;
 
+#if CONFIG_EXT_TX
+  if (plane_type != PLANE_TYPE_Y || xd->lossless)
+      return DCT_DCT;
+
+  if (is_inter_block(&mi->mbmi)) {
+    if (mi->mbmi.ext_txfrm == NORM)
+      return DCT_DCT;
+    else
+      return ADST_ADST;
+  }
+#else
   if (plane_type != PLANE_TYPE_Y || xd->lossless || is_inter_block(&mi->mbmi))
     return DCT_DCT;
+#endif
 
   return intra_mode_to_tx_type_lookup[get_y_mode(mi, ib)];
 }

diff --git a/vp9/common/vp9_entropymode.c b/vp9/common/vp9_entropymode.c
index db62954bfc2e478b1779edcb7a88c30507daff6d..3314e38843f06ea6bf7e303f6dcc122455567cd3 100644 (file)
--- a/vp9/common/vp9_entropymode.c
+++ b/vp9/common/vp9_entropymode.c
@@ -13,6 +13,10 @@
 #include "vp9/common/vp9_onyxc_int.h"
 #include "vp9/common/vp9_seg_common.h"
 
+#if CONFIG_EXT_TX
+static const vp9_prob default_ext_tx_prob = 216;
+#endif
+
 const vp9_prob vp9_kf_y_mode_prob[INTRA_MODES][INTRA_MODES][INTRA_MODES - 1] = {
   {  // above = dc
     { 137,  30,  42, 148, 151, 207,  70,  52,  91 },  // left = dc
@@ -364,6 +368,9 @@ void vp9_init_mode_probs(FRAME_CONTEXT *fc) {
 #if CONFIG_FILTERINTRA
   vp9_copy(fc->filterintra_prob, default_filterintra_prob);
 #endif
+#if CONFIG_EXT_TX
+  fc->ext_tx_prob = default_ext_tx_prob;
+#endif
 }
 
 const vp9_tree_index vp9_switchable_interp_tree
@@ -470,6 +477,10 @@ void vp9_adapt_mode_probs(VP9_COMMON *cm) {
 
   for (i = 0; i < SKIP_CONTEXTS; ++i)
     fc->skip_probs[i] = adapt_prob(pre_fc->skip_probs[i], counts->skip[i]);
+
+#if CONFIG_EXT_TX
+  fc->ext_tx_prob = adapt_prob(pre_fc->ext_tx_prob, counts->ext_tx);
+#endif
 }
 
 static void set_default_lf_deltas(struct loopfilter *lf) {

diff --git a/vp9/common/vp9_entropymode.h b/vp9/common/vp9_entropymode.h
index 0f00234c7c88e14a1456812245b2d248ec4bcebf..65ac4165f19db357390e7d5db7f77600f1e7bd0c 100644 (file)
--- a/vp9/common/vp9_entropymode.h
+++ b/vp9/common/vp9_entropymode.h
@@ -59,6 +59,9 @@ typedef struct frame_contexts {
 #if CONFIG_FILTERINTRA
   vp9_prob filterintra_prob[TX_SIZES][INTRA_MODES];
 #endif
+#if CONFIG_EXT_TX
+  vp9_prob ext_tx_prob;
+#endif
 } FRAME_CONTEXT;
 
 typedef struct {
@@ -81,6 +84,9 @@ typedef struct {
 #if CONFIG_FILTERINTRA
   unsigned int filterintra[TX_SIZES][INTRA_MODES][2];
 #endif
+#if CONFIG_EXT_TX
+  unsigned int ext_tx[2];
+#endif
 } FRAME_COUNTS;
 
 extern const vp9_prob vp9_kf_uv_mode_prob[INTRA_MODES][INTRA_MODES - 1];

diff --git a/vp9/common/vp9_enums.h b/vp9/common/vp9_enums.h
index ca060217568f3df58a5a2ef7b93b9ec0ff849504..0b019a91f464a61fd4cdeaf3f374c664eb0aaeca 100644 (file)
--- a/vp9/common/vp9_enums.h
+++ b/vp9/common/vp9_enums.h
@@ -104,6 +104,14 @@ typedef enum {
   TX_TYPES = 4
 } TX_TYPE;
 
+#if CONFIG_EXT_TX
+typedef enum {
+  NORM = 0,
+  ALT = 1,
+  EXT_TX_TYPES = 2
+} EXT_TX_TYPE;
+#endif
+
 typedef enum {
   UNKNOWN    = 0,
   BT_601     = 1,  // YUV

diff --git a/vp9/decoder/vp9_decodeframe.c b/vp9/decoder/vp9_decodeframe.c
index bf20455af81f1687580996001ae7c8023dece739..dce5ff35ab3cfcab821b7c5c1d9b911a4d92cc44 100644 (file)
--- a/vp9/decoder/vp9_decodeframe.c
+++ b/vp9/decoder/vp9_decodeframe.c
@@ -1472,6 +1472,9 @@ static int read_compressed_header(VP9Decoder *pbi, const uint8_t *data,
         vp9_diff_update_prob(&r, &fc->partition_prob[j][i]);
 
     read_mv_probs(nmvc, cm->allow_high_precision_mv, &r);
+#if CONFIG_EXT_TX
+    vp9_diff_update_prob(&r, &fc->ext_tx_prob);
+#endif
   }
 
   return vp9_reader_has_error(&r);
@@ -1523,6 +1526,10 @@ static void debug_check_frame_counts(const VP9_COMMON *const cm) {
   assert(!memcmp(&cm->counts.tx, &zero_counts.tx, sizeof(cm->counts.tx)));
   assert(!memcmp(cm->counts.skip, zero_counts.skip, sizeof(cm->counts.skip)));
   assert(!memcmp(&cm->counts.mv, &zero_counts.mv, sizeof(cm->counts.mv)));
+#if CONFIG_EXT_TX
+  assert(!memcmp(cm->counts.ext_tx, zero_counts.ext_tx,
+                 sizeof(cm->counts.ext_tx)));
+#endif
 }
 #endif  // NDEBUG
 

diff --git a/vp9/decoder/vp9_decodemv.c b/vp9/decoder/vp9_decodemv.c
index 886c3161cd7a94801ff0961b5d5e933a735e66c8..8f2b3251dca54bc268c52858f423fd515d60e188 100644 (file)
--- a/vp9/decoder/vp9_decodemv.c
+++ b/vp9/decoder/vp9_decodemv.c
@@ -666,11 +666,25 @@ static void read_inter_frame_mode_info(VP9_COMMON *const cm,
 
   mbmi->mv[0].as_int = 0;
   mbmi->mv[1].as_int = 0;
+
   mbmi->segment_id = read_inter_segment_id(cm, xd, mi_row, mi_col, r);
   mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r);
   inter_block = read_is_inter_block(cm, xd, mbmi->segment_id, r);
   mbmi->tx_size = read_tx_size(cm, xd, cm->tx_mode, mbmi->sb_type,
                                !mbmi->skip || !inter_block, r);
+#if CONFIG_EXT_TX
+  if (inter_block &&
+      mbmi->tx_size <= TX_16X16 &&
+      mbmi->sb_type >= BLOCK_8X8 &&
+      !vp9_segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) &&
+      !mbmi->skip) {
+    mbmi->ext_txfrm = vp9_read(r, cm->fc.ext_tx_prob);
+    if (!cm->frame_parallel_decoding_mode)
+      ++cm->counts.ext_tx[mbmi->ext_txfrm];
+  } else {
+    mbmi->ext_txfrm = NORM;
+  }
+#endif
 
   if (inter_block)
     read_inter_block_mode_info(cm, xd, tile, mi, mi_row, mi_col, r);

diff --git a/vp9/encoder/vp9_bitstream.c b/vp9/encoder/vp9_bitstream.c
index 9f2ef44b2efc46fd09d5027e8d91ed076d4def99..8c3d1d57eb2533f09c5093d0f86ef343a68afc24 100644 (file)
--- a/vp9/encoder/vp9_bitstream.c
+++ b/vp9/encoder/vp9_bitstream.c
@@ -277,6 +277,15 @@ static void pack_inter_mode_mvs(VP9_COMP *cpi, const MODE_INFO *mi,
         (skip || vp9_segfeature_active(seg, segment_id, SEG_LVL_SKIP)))) {
     write_selected_tx_size(cm, xd, mbmi->tx_size, bsize, w);
   }
+#if CONFIG_EXT_TX
+    if (is_inter &&
+        mbmi->tx_size <= TX_16X16 &&
+        bsize >= BLOCK_8X8 &&
+        !mbmi->skip &&
+        !vp9_segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+      vp9_write(w, mbmi->ext_txfrm, cm->fc.ext_tx_prob);
+    }
+#endif
 
   if (!is_inter) {
     if (bsize >= BLOCK_8X8) {
@@ -1290,6 +1299,10 @@ static size_t write_compressed_header(VP9_COMP *cpi, uint8_t *data) {
                        cm->counts.partition[i], PARTITION_TYPES, &header_bc);
 
     vp9_write_nmv_probs(cm, cm->allow_high_precision_mv, &header_bc);
+
+#if CONFIG_EXT_TX
+    vp9_cond_prob_diff_update(&header_bc, &fc->ext_tx_prob, cm->counts.ext_tx);
+#endif
   }
 
   vp9_stop_encode(&header_bc);

diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 7bd92b50e77431e977b415d31ccbcbb4e65d7fe8..7fb2ca2c0ee0e02f57ac2f1ff68ba5458b903bec 100644 (file)
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -3883,5 +3883,14 @@ static void encode_superblock(VP9_COMP *cpi, TOKENEXTRA **t, int output_enabled,
           if (mi_col + x < cm->mi_cols && mi_row + y < cm->mi_rows)
             mi_8x8[mis * y + x].src_mi->mbmi.tx_size = tx_size;
     }
+#if CONFIG_EXT_TX
+    if (mbmi->tx_size < TX_32X32 &&
+        is_inter_block(mbmi) &&
+        bsize >= BLOCK_8X8 &&
+        !mbmi->skip &&
+        !vp9_segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+      ++cm->counts.ext_tx[mbmi->ext_txfrm];
+    }
+#endif
   }
 }

diff --git a/vp9/encoder/vp9_encodemb.c b/vp9/encoder/vp9_encodemb.c
index 8f91e6dd17b69150e7c80c88cc3c8b43626b2da2..c1b88473208894ce19170f709b701d0fc0f364ca 100644 (file)
--- a/vp9/encoder/vp9_encodemb.c
+++ b/vp9/encoder/vp9_encodemb.c
@@ -592,6 +592,9 @@ void vp9_xform_quant(MACROBLOCK *x, int plane, int block,
   const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
   int i, j;
   const int16_t *src_diff;
+#if CONFIG_EXT_TX
+  MB_MODE_INFO *mbmi = &xd->mi[0].src_mi->mbmi;
+#endif
   txfrm_block_to_raster_xy(plane_bsize, tx_size, block, &i, &j);
   src_diff = &p->src_diff[4 * (j * diff_stride + i)];
 
@@ -615,21 +618,45 @@ void vp9_xform_quant(MACROBLOCK *x, int plane, int block,
                                     scan_order->scan, scan_order->iscan);
         break;
       case TX_16X16:
+#if CONFIG_EXT_TX
+        if (plane != 0 || mbmi->ext_txfrm == NORM) {
+          vp9_highbd_fdct16x16(src_diff, coeff, diff_stride);
+        } else {
+          vp9_highbd_fht16x16(src_diff, coeff, diff_stride, ADST_ADST);
+        }
+#else
         vp9_highbd_fdct16x16(src_diff, coeff, diff_stride);
+#endif
         vp9_highbd_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round,
                               p->quant, p->quant_shift, qcoeff, dqcoeff,
                               pd->dequant, p->zbin_extra, eob,
                               scan_order->scan, scan_order->iscan);
         break;
       case TX_8X8:
+#if CONFIG_EXT_TX
+        if (plane != 0 || mbmi->ext_txfrm == NORM) {
+          vp9_highbd_fdct8x8(src_diff, coeff, diff_stride);
+        } else {
+          vp9_highbd_fht8x8(src_diff, coeff, diff_stride, ADST_ADST);
+        }
+#else
         vp9_highbd_fdct8x8(src_diff, coeff, diff_stride);
+#endif
         vp9_highbd_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round,
                               p->quant, p->quant_shift, qcoeff, dqcoeff,
                               pd->dequant, p->zbin_extra, eob,
                               scan_order->scan, scan_order->iscan);
         break;
       case TX_4X4:
+#if CONFIG_EXT_TX
+        if (plane != 0 || mbmi->ext_txfrm == NORM) {
+          x->fwd_txm4x4(src_diff, coeff, diff_stride);
+        } else {
+          vp9_highbd_fht4x4(src_diff, coeff, diff_stride, ADST_ADST);
+        }
+#else
         x->fwd_txm4x4(src_diff, coeff, diff_stride);
+#endif
         vp9_highbd_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round,
                               p->quant, p->quant_shift, qcoeff, dqcoeff,
                               pd->dequant, p->zbin_extra, eob,
@@ -660,21 +687,45 @@ void vp9_xform_quant(MACROBLOCK *x, int plane, int block,
                            scan_order->iscan);
       break;
     case TX_16X16:
+#if CONFIG_EXT_TX
+      if (plane != 0 || mbmi->ext_txfrm == NORM) {
+        vp9_fdct16x16(src_diff, coeff, diff_stride);
+      } else {
+        vp9_fht16x16(src_diff, coeff, diff_stride, ADST_ADST);
+      }
+#else
       vp9_fdct16x16(src_diff, coeff, diff_stride);
+#endif
       vp9_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round,
                      p->quant, p->quant_shift, qcoeff, dqcoeff,
                      pd->dequant, p->zbin_extra, eob,
                      scan_order->scan, scan_order->iscan);
       break;
     case TX_8X8:
+#if CONFIG_EXT_TX
+      if (plane != 0 || mbmi->ext_txfrm == NORM) {
+        vp9_fdct8x8(src_diff, coeff, diff_stride);
+      } else {
+        vp9_fht8x8(src_diff, coeff, diff_stride, ADST_ADST);
+      }
+#else
       vp9_fdct8x8(src_diff, coeff, diff_stride);
+#endif
       vp9_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round,
                      p->quant, p->quant_shift, qcoeff, dqcoeff,
                      pd->dequant, p->zbin_extra, eob,
                      scan_order->scan, scan_order->iscan);
       break;
     case TX_4X4:
+#if CONFIG_EXT_TX
+      if (plane != 0 || mbmi->ext_txfrm == NORM) {
+        x->fwd_txm4x4(src_diff, coeff, diff_stride);
+      } else {
+        vp9_fht4x4(src_diff, coeff, diff_stride, ADST_ADST);
+      }
+#else
       x->fwd_txm4x4(src_diff, coeff, diff_stride);
+#endif
       vp9_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round,
                      p->quant, p->quant_shift, qcoeff, dqcoeff,
                      pd->dequant, p->zbin_extra, eob,
@@ -698,6 +749,9 @@ static void encode_block(int plane, int block, BLOCK_SIZE plane_bsize,
   int i, j;
   uint8_t *dst;
   ENTROPY_CONTEXT *a, *l;
+#if CONFIG_EXT_TX
+  MB_MODE_INFO *mbmi = &xd->mi[0].src_mi->mbmi;
+#endif
   txfrm_block_to_raster_xy(plane_bsize, tx_size, block, &i, &j);
   dst = &pd->dst.buf[4 * j * pd->dst.stride + 4 * i];
   a = &ctx->ta[plane][i];
@@ -762,19 +816,52 @@ static void encode_block(int plane, int block, BLOCK_SIZE plane_bsize,
                                  p->eobs[block], xd->bd);
         break;
       case TX_16X16:
+#if CONFIG_EXT_TX
+        if (plane != 0 || mbmi->ext_txfrm == NORM) {
+          vp9_highbd_idct16x16_add(dqcoeff, dst, pd->dst.stride,
+                                   p->eobs[block], xd->bd);
+        } else {
+          vp9_highbd_iht16x16_add(ADST_ADST, dqcoeff, dst, pd->dst.stride,
+                                  p->eobs[block], xd->bd);
+        }
+#else
         vp9_highbd_idct16x16_add(dqcoeff, dst, pd->dst.stride,
                                  p->eobs[block], xd->bd);
+#endif
         break;
       case TX_8X8:
+#if CONFIG_EXT_TX
+        if (plane != 0 || mbmi->ext_txfrm == NORM) {
+          vp9_highbd_idct8x8_add(dqcoeff, dst, pd->dst.stride,
+                                 p->eobs[block], xd->bd);
+        } else {
+          vp9_highbd_iht8x8_add(ADST_ADST, dqcoeff, dst, pd->dst.stride,
+                                p->eobs[block], xd->bd);
+        }
+#else
         vp9_highbd_idct8x8_add(dqcoeff, dst, pd->dst.stride,
                                p->eobs[block], xd->bd);
+#endif
         break;
       case TX_4X4:
+#if CONFIG_EXT_TX
+        if (plane != 0 || mbmi->ext_txfrm == NORM) {
+          // this is like vp9_short_idct4x4 but has a special case around eob<=1
+          // which is significant (not just an optimization) for the lossless
+          // case.
+          x->highbd_itxm_add(dqcoeff, dst, pd->dst.stride,
+                             p->eobs[block], xd->bd);
+        } else {
+          vp9_highbd_iht4x4_add(ADST_ADST, dqcoeff, dst, pd->dst.stride,
+                                p->eobs[block], xd->bd);
+        }
+#else
         // this is like vp9_short_idct4x4 but has a special case around eob<=1
         // which is significant (not just an optimization) for the lossless
         // case.
         x->highbd_itxm_add(dqcoeff, dst, pd->dst.stride,
                            p->eobs[block], xd->bd);
+#endif
         break;
       default:
         assert(0 && "Invalid transform size");
@@ -793,16 +880,46 @@ static void encode_block(int plane, int block, BLOCK_SIZE plane_bsize,
       vp9_idct32x32_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
       break;
     case TX_16X16:
+#if CONFIG_EXT_TX
+      if (plane != 0 || mbmi->ext_txfrm == NORM) {
+        vp9_idct16x16_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
+      } else {
+        vp9_iht16x16_add(ADST_ADST, dqcoeff, dst, pd->dst.stride,
+                         p->eobs[block]);
+      }
+#else
       vp9_idct16x16_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
+#endif
       break;
     case TX_8X8:
+#if CONFIG_EXT_TX
+      if (plane != 0 || mbmi->ext_txfrm == NORM) {
+        vp9_idct8x8_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
+      } else {
+        vp9_iht8x8_add(ADST_ADST, dqcoeff, dst, pd->dst.stride,
+                       p->eobs[block]);
+      }
+#else
       vp9_idct8x8_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
+#endif
       break;
     case TX_4X4:
+#if CONFIG_EXT_TX
+      if (plane != 0 || mbmi->ext_txfrm == NORM) {
+        // this is like vp9_short_idct4x4 but has a special case around eob<=1
+        // which is significant (not just an optimization) for the lossless
+        // case.
+        x->itxm_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
+      } else {
+        vp9_iht4x4_add(ADST_ADST, dqcoeff, dst, pd->dst.stride,
+                       p->eobs[block]);
+      }
+#else
       // this is like vp9_short_idct4x4 but has a special case around eob<=1
       // which is significant (not just an optimization) for the lossless
       // case.
       x->itxm_add(dqcoeff, dst, pd->dst.stride, p->eobs[block]);
+#endif
       break;
     default:
       assert(0 && "Invalid transform size");
@@ -819,6 +936,10 @@ static void encode_block_pass1(int plane, int block, BLOCK_SIZE plane_bsize,
   tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
   int i, j;
   uint8_t *dst;
+#if CONFIG_EXT_TX
+  MB_MODE_INFO *mbmi = &xd->mi[0].src_mi->mbmi;
+  mbmi->ext_txfrm = NORM;
+#endif
   txfrm_block_to_raster_xy(plane_bsize, tx_size, block, &i, &j);
   dst = &pd->dst.buf[4 * j * pd->dst.stride + 4 * i];
 
@@ -958,7 +1079,7 @@ static void encode_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
         mode = plane == 0 ? mbmi->mode : mbmi->uv_mode;
         vp9_predict_intra_block(xd, block >> 4, bwl, TX_16X16, mode,
 #if CONFIG_FILTERINTRA
-                              fbit,
+                                fbit,
 #endif
                                 x->skip_encode ? src : dst,
                                 x->skip_encode ? src_stride : dst_stride,
@@ -983,7 +1104,7 @@ static void encode_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
         mode = plane == 0 ? mbmi->mode : mbmi->uv_mode;
         vp9_predict_intra_block(xd, block >> 2, bwl, TX_8X8, mode,
 #if CONFIG_FILTERINTRA
-                              fbit,
+                                fbit,
 #endif
                                 x->skip_encode ? src : dst,
                                 x->skip_encode ? src_stride : dst_stride,
@@ -1008,7 +1129,7 @@ static void encode_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
         mode = plane == 0 ? get_y_mode(xd->mi[0].src_mi, block) : mbmi->uv_mode;
         vp9_predict_intra_block(xd, block, bwl, TX_4X4, mode,
 #if CONFIG_FILTERINTRA
-                              fbit,
+                                fbit,
 #endif
                                 x->skip_encode ? src : dst,
                                 x->skip_encode ? src_stride : dst_stride,

diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 4520eb6fd1756ced99468d14e339494f3d5bc87e..9a26c253ee924362aba42b9e565e8927f9cea9df 100644 (file)
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -924,7 +924,22 @@ static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
   }
 #endif  // CONFIG_VP9_HIGHBITDEPTH
 
-#if !CONFIG_FILTERINTRA
+#if CONFIG_FILTERINTRA
+  for (mode_ext = 2 * DC_PRED; mode_ext <= 2 * TM_PRED + 1; ++mode_ext) {
+    int64_t this_rd;
+    int ratey = 0;
+    int64_t distortion = 0;
+    int rate;
+
+    fbit = mode_ext & 1;
+    mode = mode_ext >> 1;
+    if (fbit && !is_filter_allowed(mode))
+      continue;
+
+    rate = bmode_costs[mode];
+    if (is_filter_allowed(mode))
+      rate += vp9_cost_bit(cpi->common.fc.filterintra_prob[0][mode], fbit);
+#else
   for (mode = DC_PRED; mode <= TM_PRED; ++mode) {
     int64_t this_rd;
     int ratey = 0;
@@ -940,21 +955,6 @@ static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
       if (conditional_skipintra(mode, *best_mode))
           continue;
     }
-#else
-  for (mode_ext = 2 * DC_PRED; mode_ext <= 2 * TM_PRED + 1; ++mode_ext) {
-    int64_t this_rd;
-    int ratey = 0;
-    int64_t distortion = 0;
-    int rate;
-
-    fbit = mode_ext & 1;
-    mode = mode_ext >> 1;
-    if (fbit && !is_filter_allowed(mode))
-      continue;
-
-    rate = bmode_costs[mode];
-    if (is_filter_allowed(mode))
-      rate += vp9_cost_bit(cpi->common.fc.filterintra_prob[0][mode], fbit);
 #endif
 
     vpx_memcpy(tempa, ta, sizeof(ta));
@@ -1159,22 +1159,19 @@ static int64_t rd_pick_intra_sby_mode(VP9_COMP *cpi, MACROBLOCK *x,
 
   vpx_memset(x->skip_txfm, 0, sizeof(x->skip_txfm));
   /* Y Search for intra prediction mode */
-#if !CONFIG_FILTERINTRA
-  for (mode = DC_PRED; mode <= TM_PRED; mode++) {
-    int64_t local_tx_cache[TX_MODES];
-    mic->mbmi.mode = mode;
-#else
+#if CONFIG_FILTERINTRA
   for (mode_ext = 2 * DC_PRED; mode_ext <= 2 * TM_PRED + 1; mode_ext++) {
     int64_t local_tx_cache[TX_MODES];
-    mic->mbmi.mode = mode;
     fbit = mode_ext & 1;
     mode = mode_ext >> 1;
     if (fbit && !is_filter_allowed(mode))
       continue;
-#endif
-    mic->mbmi.mode = mode;
-#if CONFIG_FILTERINTRA
     mic->mbmi.filterbit = fbit;
+    mic->mbmi.mode = mode;
+#else
+  for (mode = DC_PRED; mode <= TM_PRED; mode++) {
+    int64_t local_tx_cache[TX_MODES];
+    mic->mbmi.mode = mode;
 #endif
 
     super_block_yrd(cpi, x, &this_rate_tokenonly, &this_distortion,
@@ -1291,10 +1288,13 @@ static int64_t rd_pick_intra_sbuv_mode(VP9_COMP *cpi, MACROBLOCK *x,
   int64_t best_rd = INT64_MAX, this_rd;
   int this_rate_tokenonly, this_rate, s;
   int64_t this_distortion, this_sse;
+#if CONFIG_FILTERINTRA
+  int mode_ext, fbit = 0, fbit_selected = 0;
+  (void)max_tx_size;
+#endif
 
   vpx_memset(x->skip_txfm, 0, sizeof(x->skip_txfm));
 #if CONFIG_FILTERINTRA
-  int mode_ext, fbit = 0, fbit_selected = 0;
   for (mode_ext = 2 * DC_PRED; mode_ext <= 2 * TM_PRED + 1; mode_ext++) {
     mode = mode_ext >> 1;
     fbit = mode_ext & 1;
@@ -2808,8 +2808,41 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
     int64_t sseuv = INT64_MAX;
     int64_t rdcosty = INT64_MAX;
 
-    // Y cost and distortion
+#if CONFIG_EXT_TX
+    int64_t rdcost_tx;
+    int rate_y_tx;
+    int64_t distortion_y_tx;
+    int dummy;
+    int64_t best_rdcost_tx = INT64_MAX;
+    int best_ext_tx = NORM;
+    double th = 0.99;
+
+    vp9_subtract_plane(x, bsize, 0);
+    for (i = 0; i < EXT_TX_TYPES; i++) {
+      mbmi->ext_txfrm = i;
+      super_block_yrd(cpi, x, &rate_y_tx, &distortion_y_tx, &dummy, psse,
+                      bsize, txfm_cache, INT64_MAX);
+      assert(rate_y_tx != INT_MAX);
+      if (mbmi->tx_size < TX_32X32)
+        rate_y_tx += vp9_cost_bit(cm->fc.ext_tx_prob, i);
+      assert(rate_y_tx >= 0);
+      rdcost_tx = RDCOST(x->rdmult, x->rddiv, rate_y_tx, distortion_y_tx);
+      rdcost_tx = MIN(rdcost_tx, RDCOST(x->rdmult, x->rddiv, 0, *psse));
+      assert(rdcost_tx >= 0);
+      if (rdcost_tx < best_rdcost_tx * th) {
+        best_ext_tx = i;
+        best_rdcost_tx = rdcost_tx;
+      }
+    }
+    if (mbmi->tx_size >= TX_32X32)
+      mbmi->ext_txfrm = NORM;
+    else
+      mbmi->ext_txfrm = best_ext_tx;
+#else
     vp9_subtract_plane(x, bsize, 0);
+#endif
+
+    // Y cost and distortion
     super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
                     bsize, txfm_cache, ref_best_rd);
 
@@ -2821,6 +2854,10 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
     }
 
     *rate2 += *rate_y;
+#if CONFIG_EXT_TX
+    if (mbmi->tx_size < TX_32X32)
+      *rate2 += vp9_cost_bit(cm->fc.ext_tx_prob, mbmi->ext_txfrm);
+#endif
     *distortion += distortion_y;
 
     rdcosty = RDCOST(x->rdmult, x->rddiv, *rate2, *distortion);
@@ -3304,10 +3341,13 @@ void vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
 
     if (ref_frame == INTRA_FRAME) {
       TX_SIZE uv_tx;
+      struct macroblockd_plane *const pd = &xd->plane[1];
 #if CONFIG_FILTERINTRA
       mbmi->filterbit = 0;
 #endif
-      struct macroblockd_plane *const pd = &xd->plane[1];
+#if CONFIG_EXT_TX
+      mbmi->ext_txfrm = NORM;
+#endif
       vpx_memset(x->skip_txfm, 0, sizeof(x->skip_txfm));
       super_block_yrd(cpi, x, &rate_y, &distortion_y, &skippable,
                       NULL, bsize, tx_cache, best_rd);
@@ -3319,8 +3359,8 @@ void vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
         mbmi->filterbit = 0;
 
         super_block_yrd(cpi, x, &rate_y_tmp, &distortion_y_tmp,
-                              &skippable_tmp, NULL, bsize, tx_cache_tmp,
-                              best_rd);
+                        &skippable_tmp, NULL, bsize, tx_cache_tmp,
+                        best_rd);
 
         if (rate_y == INT_MAX && rate_y_tmp == INT_MAX)
           continue;
@@ -3378,6 +3418,9 @@ void vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
         rate2 += intra_cost_penalty;
       distortion2 = distortion_y + distortion_uv;
     } else {
+#if CONFIG_EXT_TX
+      mbmi->ext_txfrm = NORM;
+#endif
       this_rd = handle_inter_mode(cpi, x, bsize,
                                   tx_cache,
                                   &rate2, &distortion2, &skippable,
@@ -3831,6 +3874,9 @@ void vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
   b_mode_info best_bmodes[4];
   int best_skip2 = 0;
   int ref_frame_skip_mask[2] = { 0 };
+#if CONFIG_EXT_TX
+  mbmi->ext_txfrm = NORM;
+#endif
 
   x->skip_encode = sf->skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
   vpx_memset(x->zcoeff_blk[TX_4X4], 0, 4);
@@ -3987,6 +4033,9 @@ void vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x,
 
     if (ref_frame == INTRA_FRAME) {
       int rate;
+#if CONFIG_EXT_TX
+      mbmi->ext_txfrm = NORM;
+#endif
       if (rd_pick_intra_sub_8x8_y_mode(cpi, x, &rate, &rate_y,
                                        &distortion_y, best_rd) >= best_rd)
         continue;