Merge remote-tracking branch 'origin/master' into nextgenv2

Periodic merge to get master changes into nextgenv2.

Change-Id: I6f0e4b470f193da03f1a8cb8e6a93ae39395699a

diff --cc configure
index 6bb90e386eeca2e927fc0d9da03cbcfdec28d259,0e01a2e49ac403b2c2db512f77a2c55c6233903e..508afd1a1ff92af8537d5197f12244fa314b0094
--- 1/configure
--- 2/configure
+++ b/configure
@@@ -264,7 -264,7 +264,8 @@@ EXPERIMENT_LIST=
      spatial_svc
      fp_mb_stats
      emulate_hardware
 +    ext_tx
+     misc_fixes
  "
  CONFIG_LIST="
      dependency_tracking

diff --cc vp10/common/blockd.h
Simple merge

diff --cc vp10/common/entropymode.c
Simple merge

diff --cc vp10/common/scan.c
index 57095d966ab0a156217ed894d02d3cac3b0e0b22,7217f6d045926ae85c87bccf785506078e5363eb..e5c044a1f4fd33c42b5f39a745ddcaebdd78de6e
--- 1/vp10/common/scan.c
--- 2/vp10/common/scan.c
+++ b/vp10/common/scan.c
@@@ -695,154 -695,14 +695,161 @@@ DECLARE_ALIGNED(16, static const int16_
    1023,
  };
  
 -const scan_order vp10_scan_orders[TX_SIZES][TX_TYPES] = {
+ const scan_order vp10_default_scan_orders[TX_SIZES] = {
+   {default_scan_4x4,   vp10_default_iscan_4x4,   default_scan_4x4_neighbors},
+   {default_scan_8x8,   vp10_default_iscan_8x8,   default_scan_8x8_neighbors},
+   {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
+   {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
+ };
+ 
 +#if CONFIG_EXT_TX
 +const scan_order vp10_intra_scan_orders[TX_SIZES][TX_TYPES] = {
 +  {  // TX_4X4
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {row_scan_4x4,     vp10_row_iscan_4x4,     row_scan_4x4_neighbors},
 +    {col_scan_4x4,     vp10_col_iscan_4x4,     col_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +  }, {  // TX_8X8
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {row_scan_8x8,     vp10_row_iscan_8x8,     row_scan_8x8_neighbors},
 +    {col_scan_8x8,     vp10_col_iscan_8x8,     col_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +  }, {  // TX_16X16
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {row_scan_16x16,     vp10_row_iscan_16x16,     row_scan_16x16_neighbors},
 +    {col_scan_16x16,     vp10_col_iscan_16x16,     col_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +  }, {  // TX_32X32
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +  }
 +};
 +
 +const scan_order vp10_inter_scan_orders[TX_SIZES][TX_TYPES] = {
 +  {  // TX_4X4
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +    {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
 +  }, {  // TX_8X8
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +    {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors},
 +  }, {  // TX_16X16
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +    {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors},
 +  }, {  // TX_32X32
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +    {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors},
 +  }
 +};
 +
 +#else   // CONFIG_EXT_TX
 +
 +const scan_order vp10_intra_scan_orders[TX_SIZES][TX_TYPES] = {
    {  // TX_4X4
      {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors},
      {row_scan_4x4,     vp10_row_iscan_4x4,     row_scan_4x4_neighbors},

diff --cc vp10/common/scan.h
index 2f1b658487a012a9d8a1c6f36181d1c26eda3479,f5a020f1e744bae35be5ffe1b4dfff7305190763..aadae4024e06ea6ac1ffa55abfe75ab661b2e29f
--- 1/vp10/common/scan.h
--- 2/vp10/common/scan.h
+++ b/vp10/common/scan.h
@@@ -29,7 -29,8 +29,8 @@@ typedef struct 
    const int16_t *neighbors;
  } scan_order;
  
 -extern const scan_order vp10_scan_orders[TX_SIZES][TX_TYPES];
+ extern const scan_order vp10_default_scan_orders[TX_SIZES];
 +extern const scan_order vp10_intra_scan_orders[TX_SIZES][TX_TYPES];
  
  static INLINE int get_coef_context(const int16_t *neighbors,
                                     const uint8_t *token_cache, int c) {

diff --cc vp10/decoder/decodeframe.c
Simple merge

diff --cc vp10/decoder/decodemv.c
index 71a5e28177b0f589c1b460ad7a80bc8ea575575b,f157ada20e19116b8f1a3f37b18e492d7b790798..4d8a1eba36f961be0b8759b4e761f4da9e568b93
--- 1/vp10/decoder/decodemv.c
--- 2/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@@ -597,22 -595,6 +595,23 @@@ static void read_inter_frame_mode_info(
    mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r);
    inter_block = read_is_inter_block(cm, xd, mbmi->segment_id, r);
    mbmi->tx_size = read_tx_size(cm, xd, !mbmi->skip || !inter_block, r);
-       if (!cm->frame_parallel_decoding_mode)
-         ++cm->counts.ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
 +#if CONFIG_EXT_TX
 +    if (inter_block &&
 +        mbmi->tx_size <= TX_16X16 &&
 +        cm->base_qindex > 0 &&
 +        mbmi->sb_type >= BLOCK_8X8 &&
 +        !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) &&
 +        !mbmi->skip) {
++      FRAME_COUNTS *counts = xd->counts;
 +      mbmi->ext_txfrm = vpx_read_tree(r,
 +                                      vp10_ext_tx_tree,
 +                                      cm->fc->ext_tx_prob[mbmi->tx_size]);
++      if (counts)
++        ++counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm];
 +    } else {
 +      mbmi->ext_txfrm = NORM;
 +    }
 +#endif  // CONFIG_EXT_TX
  
    if (inter_block)
      read_inter_block_mode_info(pbi, xd, mi, mi_row, mi_col, r);

diff --cc vp10/encoder/bitstream.c
Simple merge

diff --cc vp10/encoder/bitstream.h
index b8b2354d50e50753d7e021218558985e01673fec,fc2517b1564d9854233c0b4d6fd0d21d3f5786d8..dda000fb23386d1d922bd336b7d48da81484287a
--- 1/vp10/encoder/bitstream.h
--- 2/vp10/encoder/bitstream.h
+++ b/vp10/encoder/bitstream.h
@@@ -20,16 -20,9 +20,11 @@@ extern "C" 
  
  void vp10_pack_bitstream(VP10_COMP *cpi, uint8_t *dest, size_t *size);
  
 +void vp10_encode_token_init();
 +
  static INLINE int vp10_preserve_existing_gf(VP10_COMP *cpi) {
    return !cpi->multi_arf_allowed && cpi->refresh_golden_frame &&
-          cpi->rc.is_src_frame_alt_ref &&
-          (!cpi->use_svc ||      // Add spatial svc base layer case here
-           (is_two_pass_svc(cpi) &&
-            cpi->svc.spatial_layer_id == 0 &&
-            cpi->svc.layer_context[0].gold_ref_idx >=0 &&
-            cpi->oxcf.ss_enable_auto_arf[0]));
+          cpi->rc.is_src_frame_alt_ref;
  }
  
  #ifdef __cplusplus

diff --cc vp10/encoder/dct.c
index f183167819168bc50036e23ec56c6e665ceb70ce,80ace06cd8c2e1f6bf37014eaa167c295ea95eb5..cdbe655e9cd6c7191b2ee936804ec97dd8ab415a
--- 1/vp10/encoder/dct.c
--- 2/vp10/encoder/dct.c
+++ b/vp10/encoder/dct.c
@@@ -20,101 -20,54 +20,132 @@@
  #include "vpx_dsp/fwd_txfm.h"
  #include "vpx_ports/mem.h"
  
 +#if CONFIG_EXT_TX
 +void fdst4(const tran_low_t *input, tran_low_t *output) {
 +  static const int N = 4;
 +  // {sin(pi/5), sin(pi*2/5)} * sqrt(2/5) * sqrt(2)
 +  static const int32_t sinvalue_lookup[] = {
 +    141124871, 228344838,
 +  };
 +  int i, j;
 +  for (i = 0; i < N; i++) {
 +    int64_t sum = 0;
 +    for (j = 0; j < N; j++) {
 +      int idx = (i + 1) * (j + 1);
 +      int sign = 0;
 +      if (idx > N + 1) {
 +        sign = (idx / (N + 1)) & 1;
 +        idx %= (N + 1);
 +      }
 +      idx = idx > N + 1 - idx ? N + 1 - idx : idx;
 +      if (idx == 0) continue;
 +      idx--;
 +      sum += (int64_t)input[j] * sinvalue_lookup[idx] * (sign ? -1 : 1);
 +    }
 +    output[i] = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
 +  }
 +}
 +
 +void fdst8(const tran_low_t *input, tran_low_t *output) {
 +  static const int N = 8;
 +  // {sin(pi/9), sin(pi*2/9), ..., sin(pi*4/9)} * sqrt(2/9) * 2
 +  static const int sinvalue_lookup[] = {
 +    86559612, 162678858, 219176632, 249238470
 +  };
 +  int i, j;
 +  for (i = 0; i < N; i++) {
 +    int64_t sum = 0;
 +    for (j = 0; j < N; j++) {
 +      int idx = (i + 1) * (j + 1);
 +      int sign = 0;
 +      if (idx > N + 1) {
 +        sign = (idx / (N + 1)) & 1;
 +        idx %= (N + 1);
 +      }
 +      idx = idx > N + 1 - idx ? N + 1 - idx : idx;
 +      if (idx == 0) continue;
 +      idx--;
 +      sum += (int64_t)input[j] * sinvalue_lookup[idx] * (sign ? -1 : 1);
 +    }
 +    output[i] = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
 +  }
 +}
 +
 +void fdst16(const tran_low_t *input, tran_low_t *output) {
 +  static const int N = 16;
 +  // {sin(pi/17), sin(pi*2/17, ..., sin(pi*8/17)} * sqrt(2/17) * 2 * sqrt(2)
 +  static const int sinvalue_lookup[] = {
 +    47852167, 94074787, 137093803, 175444254,
 +    207820161, 233119001, 250479254, 259309736
 +  };
 +  int i, j;
 +  for (i = 0; i < N; i++) {
 +    int64_t sum = 0;
 +    for (j = 0; j < N; j++) {
 +      int idx = (i + 1) * (j + 1);
 +      int sign = 0;
 +      if (idx > N + 1) {
 +        sign = (idx / (N + 1)) & 1;
 +        idx %= (N + 1);
 +      }
 +      idx = idx > N + 1 - idx ? N + 1 - idx : idx;
 +      if (idx == 0) continue;
 +      idx--;
 +      sum += (int64_t)input[j] * sinvalue_lookup[idx] * (sign ? -1 : 1);
 +    }
 +    output[i] = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS));
 +  }
 +}
 +#endif  // CONFIG_EXT_TX
 +
+ static INLINE void range_check(const tran_low_t *input, const int size,
+                                const int bit) {
+ #if CONFIG_COEFFICIENT_RANGE_CHECKING
+   int i;
+   for (i = 0; i < size; ++i) {
+     assert(abs(input[i]) < (1 << bit));
+   }
+ #else
+   (void)input;
+   (void)size;
+   (void)bit;
+ #endif
+ }
+ 
  static void fdct4(const tran_low_t *input, tran_low_t *output) {
-   tran_high_t step[4];
-   tran_high_t temp1, temp2;
- 
-   step[0] = input[0] + input[3];
-   step[1] = input[1] + input[2];
-   step[2] = input[1] - input[2];
-   step[3] = input[0] - input[3];
- 
-   temp1 = (step[0] + step[1]) * cospi_16_64;
-   temp2 = (step[0] - step[1]) * cospi_16_64;
-   output[0] = (tran_low_t)fdct_round_shift(temp1);
-   output[2] = (tran_low_t)fdct_round_shift(temp2);
-   temp1 = step[2] * cospi_24_64 + step[3] * cospi_8_64;
-   temp2 = -step[2] * cospi_8_64 + step[3] * cospi_24_64;
-   output[1] = (tran_low_t)fdct_round_shift(temp1);
-   output[3] = (tran_low_t)fdct_round_shift(temp2);
+   tran_high_t temp;
+   tran_low_t step[4];
+ 
+   // stage 0
+   range_check(input, 4, 11);
+ 
+   // stage 1
+   output[0] = input[0] + input[3];
+   output[1] = input[1] + input[2];
+   output[2] = input[1] - input[2];
+   output[3] = input[0] - input[3];
+ 
+   range_check(output, 4, 12);
+ 
+   // stage 2
+   temp = output[0] * cospi_16_64 + output[1] * cospi_16_64;
+   step[0] = (tran_low_t)fdct_round_shift(temp);
+   temp = output[1] * -cospi_16_64 + output[0] * cospi_16_64;
+   step[1] = (tran_low_t)fdct_round_shift(temp);
+   temp = output[2] * cospi_24_64 + output[3] * cospi_8_64;
+   step[2] = (tran_low_t)fdct_round_shift(temp);
+   temp = output[3] * cospi_24_64 + output[2] * -cospi_8_64;
+   step[3] = (tran_low_t)fdct_round_shift(temp);
+ 
+   range_check(step, 4, 13);
+ 
+   // stage 3
+   output[0] = step[0];
+   output[1] = step[2];
+   output[2] = step[1];
+   output[3] = step[3];
+ 
+   range_check(output, 4, 13);
  }
  
  static void fdct8(const tran_low_t *input, tran_low_t *output) {

diff --cc vp10/encoder/encodeframe.c
Simple merge

diff --cc vp10/encoder/encodemb.c
Simple merge

diff --cc vp10/encoder/encoder.c
Simple merge

diff --cc vp10/encoder/encoder.h
Simple merge

diff --cc vp10/encoder/rd.c
Simple merge

diff --cc vp10/encoder/rdopt.c
index 0c2f58f43f6c9d78e0b1e51743fde4efae59f4bd,a3fbc65c6291842e962229059c1efb1c1244a216..e955e5f4abcf8c9824fa6b96626e9d0e0955c237
--- 1/vp10/encoder/rdopt.c
--- 2/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@@ -603,16 -587,7 +591,16 @@@ static void choose_largest_tx_size(VP10
    MACROBLOCKD *const xd = &x->e_mbd;
    MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  
-   mbmi->tx_size = MIN(max_tx_size, largest_tx_size);
+   mbmi->tx_size = VPXMIN(max_tx_size, largest_tx_size);
 +#if CONFIG_EXT_TX
 +  if (mbmi->ext_txfrm >= GET_EXT_TX_TYPES(mbmi->tx_size)) {
 +    *rate = INT_MAX;
 +    *distortion = INT64_MAX;
 +    *sse = INT64_MAX;
 +    *skip = 0;
 +    return;
 +  }
 +#endif  // CONFIG_EXT_TX
  
    txfm_rd_in_plane(x, rate, distortion, skip,
                     sse, ref_best_rd, 0, bs,
@@@ -2727,38 -2667,6 +2703,38 @@@ static int64_t handle_inter_mode(VP10_C
  
      // Y cost and distortion
      vp10_subtract_plane(x, bsize, 0);
-         rdcost_tx = MIN(rdcost_tx, RDCOST(x->rdmult, x->rddiv, 0, *psse));
 +#if CONFIG_EXT_TX
 +    if (xd->lossless) {
 +      mbmi->ext_txfrm = NORM;
 +    } else {
 +      int64_t rdcost_tx;
 +      int rate_y_tx;
 +      int64_t distortion_y_tx;
 +      int dummy;
 +      int64_t best_rdcost_tx = INT64_MAX;
 +      int best_ext_tx = -1;
 +
 +      for (i = NORM; i < EXT_TX_TYPES; i++) {
 +        mbmi->ext_txfrm = i;
 +        super_block_yrd(cpi, x, &rate_y_tx, &distortion_y_tx, &dummy, psse,
 +                        bsize, INT64_MAX);
 +        assert(rate_y_tx != INT_MAX);
 +        assert(rate_y_tx >= 0);
 +        rdcost_tx = RDCOST(x->rdmult, x->rddiv, rate_y_tx, distortion_y_tx);
++        rdcost_tx = VPXMIN(rdcost_tx, RDCOST(x->rdmult, x->rddiv, 0, *psse));
 +        assert(rdcost_tx >= 0);
 +        if (rdcost_tx <
 +            (best_ext_tx == NORM ? ext_tx_th : 1) * best_rdcost_tx) {
 +          best_ext_tx = i;
 +          best_rdcost_tx = rdcost_tx;
 +        }
 +      }
 +      if (mbmi->tx_size > TX_16X16)
 +        assert(best_ext_tx == NORM);
 +      mbmi->ext_txfrm = best_ext_tx;
 +    }
 +#endif  // CONFIG_EXT_TX
 +
      super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
                      bsize, ref_best_rd);
  
@@@ -3818,11 -3665,7 +3739,10 @@@ void vp10_rd_pick_inter_mode_sub8x8(VP1
    int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS];
    int internal_active_edge =
      vp10_active_edge_sb(cpi, mi_row, mi_col) && vp10_internal_image_edge(cpi);
 +#if CONFIG_EXT_TX
 +  mbmi->ext_txfrm = NORM;
 +#endif
  
-   x->skip_encode = sf->skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
    memset(x->zcoeff_blk[TX_4X4], 0, 4);
    vp10_zero(best_mbmode);