From: Debargha Mukherjee Date: Thu, 17 Sep 2015 18:20:03 +0000 (-0700) Subject: Merge remote-tracking branch 'origin/master' into nextgenv2 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=09ff5f2792576c912f4a0e315a14db0f0005bfeb;p=libvpx Merge remote-tracking branch 'origin/master' into nextgenv2 Periodic merge to get master changes into nextgenv2. Change-Id: I6f0e4b470f193da03f1a8cb8e6a93ae39395699a --- 09ff5f2792576c912f4a0e315a14db0f0005bfeb diff --cc configure index 6bb90e386,0e01a2e49..508afd1a1 --- a/configure +++ b/configure @@@ -264,7 -264,7 +264,8 @@@ EXPERIMENT_LIST= spatial_svc fp_mb_stats emulate_hardware + ext_tx + misc_fixes " CONFIG_LIST=" dependency_tracking diff --cc vp10/common/scan.c index 57095d966,7217f6d04..e5c044a1f --- a/vp10/common/scan.c +++ b/vp10/common/scan.c @@@ -695,154 -695,14 +695,161 @@@ DECLARE_ALIGNED(16, static const int16_ 1023, }; + const scan_order vp10_default_scan_orders[TX_SIZES] = { + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + }; + -const scan_order vp10_scan_orders[TX_SIZES][TX_TYPES] = { +#if CONFIG_EXT_TX +const scan_order vp10_intra_scan_orders[TX_SIZES][TX_TYPES] = { + { // TX_4X4 + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {row_scan_4x4, vp10_row_iscan_4x4, row_scan_4x4_neighbors}, + {col_scan_4x4, vp10_col_iscan_4x4, col_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + }, { // TX_8X8 + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {row_scan_8x8, vp10_row_iscan_8x8, row_scan_8x8_neighbors}, + {col_scan_8x8, vp10_col_iscan_8x8, col_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + }, { // TX_16X16 + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {row_scan_16x16, vp10_row_iscan_16x16, row_scan_16x16_neighbors}, + {col_scan_16x16, vp10_col_iscan_16x16, col_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + }, { // TX_32X32 + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + } +}; + +const scan_order vp10_inter_scan_orders[TX_SIZES][TX_TYPES] = { + { // TX_4X4 + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, + }, { // TX_8X8 + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + {default_scan_8x8, vp10_default_iscan_8x8, default_scan_8x8_neighbors}, + }, { // TX_16X16 + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + {default_scan_16x16, vp10_default_iscan_16x16, default_scan_16x16_neighbors}, + }, { // TX_32X32 + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + {default_scan_32x32, vp10_default_iscan_32x32, default_scan_32x32_neighbors}, + } +}; + +#else // CONFIG_EXT_TX + +const scan_order vp10_intra_scan_orders[TX_SIZES][TX_TYPES] = { { // TX_4X4 {default_scan_4x4, vp10_default_iscan_4x4, default_scan_4x4_neighbors}, {row_scan_4x4, vp10_row_iscan_4x4, row_scan_4x4_neighbors}, diff --cc vp10/common/scan.h index 2f1b65848,f5a020f1e..aadae4024 --- a/vp10/common/scan.h +++ b/vp10/common/scan.h @@@ -29,7 -29,8 +29,8 @@@ typedef struct const int16_t *neighbors; } scan_order; + extern const scan_order vp10_default_scan_orders[TX_SIZES]; -extern const scan_order vp10_scan_orders[TX_SIZES][TX_TYPES]; +extern const scan_order vp10_intra_scan_orders[TX_SIZES][TX_TYPES]; static INLINE int get_coef_context(const int16_t *neighbors, const uint8_t *token_cache, int c) { diff --cc vp10/decoder/decodemv.c index 71a5e2817,f157ada20..4d8a1eba3 --- a/vp10/decoder/decodemv.c +++ b/vp10/decoder/decodemv.c @@@ -597,22 -595,6 +595,23 @@@ static void read_inter_frame_mode_info( mbmi->skip = read_skip(cm, xd, mbmi->segment_id, r); inter_block = read_is_inter_block(cm, xd, mbmi->segment_id, r); mbmi->tx_size = read_tx_size(cm, xd, !mbmi->skip || !inter_block, r); +#if CONFIG_EXT_TX + if (inter_block && + mbmi->tx_size <= TX_16X16 && + cm->base_qindex > 0 && + mbmi->sb_type >= BLOCK_8X8 && + !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP) && + !mbmi->skip) { ++ FRAME_COUNTS *counts = xd->counts; + mbmi->ext_txfrm = vpx_read_tree(r, + vp10_ext_tx_tree, + cm->fc->ext_tx_prob[mbmi->tx_size]); - if (!cm->frame_parallel_decoding_mode) - ++cm->counts.ext_tx[mbmi->tx_size][mbmi->ext_txfrm]; ++ if (counts) ++ ++counts->ext_tx[mbmi->tx_size][mbmi->ext_txfrm]; + } else { + mbmi->ext_txfrm = NORM; + } +#endif // CONFIG_EXT_TX if (inter_block) read_inter_block_mode_info(pbi, xd, mi, mi_row, mi_col, r); diff --cc vp10/encoder/bitstream.h index b8b2354d5,fc2517b15..dda000fb2 --- a/vp10/encoder/bitstream.h +++ b/vp10/encoder/bitstream.h @@@ -20,16 -20,9 +20,11 @@@ extern "C" void vp10_pack_bitstream(VP10_COMP *cpi, uint8_t *dest, size_t *size); +void vp10_encode_token_init(); + static INLINE int vp10_preserve_existing_gf(VP10_COMP *cpi) { return !cpi->multi_arf_allowed && cpi->refresh_golden_frame && - cpi->rc.is_src_frame_alt_ref && - (!cpi->use_svc || // Add spatial svc base layer case here - (is_two_pass_svc(cpi) && - cpi->svc.spatial_layer_id == 0 && - cpi->svc.layer_context[0].gold_ref_idx >=0 && - cpi->oxcf.ss_enable_auto_arf[0])); + cpi->rc.is_src_frame_alt_ref; } #ifdef __cplusplus diff --cc vp10/encoder/dct.c index f18316781,80ace06cd..cdbe655e9 --- a/vp10/encoder/dct.c +++ b/vp10/encoder/dct.c @@@ -20,101 -20,54 +20,132 @@@ #include "vpx_dsp/fwd_txfm.h" #include "vpx_ports/mem.h" +#if CONFIG_EXT_TX +void fdst4(const tran_low_t *input, tran_low_t *output) { + static const int N = 4; + // {sin(pi/5), sin(pi*2/5)} * sqrt(2/5) * sqrt(2) + static const int32_t sinvalue_lookup[] = { + 141124871, 228344838, + }; + int i, j; + for (i = 0; i < N; i++) { + int64_t sum = 0; + for (j = 0; j < N; j++) { + int idx = (i + 1) * (j + 1); + int sign = 0; + if (idx > N + 1) { + sign = (idx / (N + 1)) & 1; + idx %= (N + 1); + } + idx = idx > N + 1 - idx ? N + 1 - idx : idx; + if (idx == 0) continue; + idx--; + sum += (int64_t)input[j] * sinvalue_lookup[idx] * (sign ? -1 : 1); + } + output[i] = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS)); + } +} + +void fdst8(const tran_low_t *input, tran_low_t *output) { + static const int N = 8; + // {sin(pi/9), sin(pi*2/9), ..., sin(pi*4/9)} * sqrt(2/9) * 2 + static const int sinvalue_lookup[] = { + 86559612, 162678858, 219176632, 249238470 + }; + int i, j; + for (i = 0; i < N; i++) { + int64_t sum = 0; + for (j = 0; j < N; j++) { + int idx = (i + 1) * (j + 1); + int sign = 0; + if (idx > N + 1) { + sign = (idx / (N + 1)) & 1; + idx %= (N + 1); + } + idx = idx > N + 1 - idx ? N + 1 - idx : idx; + if (idx == 0) continue; + idx--; + sum += (int64_t)input[j] * sinvalue_lookup[idx] * (sign ? -1 : 1); + } + output[i] = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS)); + } +} + +void fdst16(const tran_low_t *input, tran_low_t *output) { + static const int N = 16; + // {sin(pi/17), sin(pi*2/17, ..., sin(pi*8/17)} * sqrt(2/17) * 2 * sqrt(2) + static const int sinvalue_lookup[] = { + 47852167, 94074787, 137093803, 175444254, + 207820161, 233119001, 250479254, 259309736 + }; + int i, j; + for (i = 0; i < N; i++) { + int64_t sum = 0; + for (j = 0; j < N; j++) { + int idx = (i + 1) * (j + 1); + int sign = 0; + if (idx > N + 1) { + sign = (idx / (N + 1)) & 1; + idx %= (N + 1); + } + idx = idx > N + 1 - idx ? N + 1 - idx : idx; + if (idx == 0) continue; + idx--; + sum += (int64_t)input[j] * sinvalue_lookup[idx] * (sign ? -1 : 1); + } + output[i] = ROUND_POWER_OF_TWO(sum, (2 * DCT_CONST_BITS)); + } +} +#endif // CONFIG_EXT_TX + + static INLINE void range_check(const tran_low_t *input, const int size, + const int bit) { + #if CONFIG_COEFFICIENT_RANGE_CHECKING + int i; + for (i = 0; i < size; ++i) { + assert(abs(input[i]) < (1 << bit)); + } + #else + (void)input; + (void)size; + (void)bit; + #endif + } + static void fdct4(const tran_low_t *input, tran_low_t *output) { - tran_high_t step[4]; - tran_high_t temp1, temp2; - - step[0] = input[0] + input[3]; - step[1] = input[1] + input[2]; - step[2] = input[1] - input[2]; - step[3] = input[0] - input[3]; - - temp1 = (step[0] + step[1]) * cospi_16_64; - temp2 = (step[0] - step[1]) * cospi_16_64; - output[0] = (tran_low_t)fdct_round_shift(temp1); - output[2] = (tran_low_t)fdct_round_shift(temp2); - temp1 = step[2] * cospi_24_64 + step[3] * cospi_8_64; - temp2 = -step[2] * cospi_8_64 + step[3] * cospi_24_64; - output[1] = (tran_low_t)fdct_round_shift(temp1); - output[3] = (tran_low_t)fdct_round_shift(temp2); + tran_high_t temp; + tran_low_t step[4]; + + // stage 0 + range_check(input, 4, 11); + + // stage 1 + output[0] = input[0] + input[3]; + output[1] = input[1] + input[2]; + output[2] = input[1] - input[2]; + output[3] = input[0] - input[3]; + + range_check(output, 4, 12); + + // stage 2 + temp = output[0] * cospi_16_64 + output[1] * cospi_16_64; + step[0] = (tran_low_t)fdct_round_shift(temp); + temp = output[1] * -cospi_16_64 + output[0] * cospi_16_64; + step[1] = (tran_low_t)fdct_round_shift(temp); + temp = output[2] * cospi_24_64 + output[3] * cospi_8_64; + step[2] = (tran_low_t)fdct_round_shift(temp); + temp = output[3] * cospi_24_64 + output[2] * -cospi_8_64; + step[3] = (tran_low_t)fdct_round_shift(temp); + + range_check(step, 4, 13); + + // stage 3 + output[0] = step[0]; + output[1] = step[2]; + output[2] = step[1]; + output[3] = step[3]; + + range_check(output, 4, 13); } static void fdct8(const tran_low_t *input, tran_low_t *output) { diff --cc vp10/encoder/rdopt.c index 0c2f58f43,a3fbc65c6..e955e5f4a --- a/vp10/encoder/rdopt.c +++ b/vp10/encoder/rdopt.c @@@ -603,16 -587,7 +591,16 @@@ static void choose_largest_tx_size(VP10 MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; - mbmi->tx_size = MIN(max_tx_size, largest_tx_size); + mbmi->tx_size = VPXMIN(max_tx_size, largest_tx_size); +#if CONFIG_EXT_TX + if (mbmi->ext_txfrm >= GET_EXT_TX_TYPES(mbmi->tx_size)) { + *rate = INT_MAX; + *distortion = INT64_MAX; + *sse = INT64_MAX; + *skip = 0; + return; + } +#endif // CONFIG_EXT_TX txfm_rd_in_plane(x, rate, distortion, skip, sse, ref_best_rd, 0, bs, @@@ -2727,38 -2667,6 +2703,38 @@@ static int64_t handle_inter_mode(VP10_C // Y cost and distortion vp10_subtract_plane(x, bsize, 0); +#if CONFIG_EXT_TX + if (xd->lossless) { + mbmi->ext_txfrm = NORM; + } else { + int64_t rdcost_tx; + int rate_y_tx; + int64_t distortion_y_tx; + int dummy; + int64_t best_rdcost_tx = INT64_MAX; + int best_ext_tx = -1; + + for (i = NORM; i < EXT_TX_TYPES; i++) { + mbmi->ext_txfrm = i; + super_block_yrd(cpi, x, &rate_y_tx, &distortion_y_tx, &dummy, psse, + bsize, INT64_MAX); + assert(rate_y_tx != INT_MAX); + assert(rate_y_tx >= 0); + rdcost_tx = RDCOST(x->rdmult, x->rddiv, rate_y_tx, distortion_y_tx); - rdcost_tx = MIN(rdcost_tx, RDCOST(x->rdmult, x->rddiv, 0, *psse)); ++ rdcost_tx = VPXMIN(rdcost_tx, RDCOST(x->rdmult, x->rddiv, 0, *psse)); + assert(rdcost_tx >= 0); + if (rdcost_tx < + (best_ext_tx == NORM ? ext_tx_th : 1) * best_rdcost_tx) { + best_ext_tx = i; + best_rdcost_tx = rdcost_tx; + } + } + if (mbmi->tx_size > TX_16X16) + assert(best_ext_tx == NORM); + mbmi->ext_txfrm = best_ext_tx; + } +#endif // CONFIG_EXT_TX + super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse, bsize, ref_best_rd); @@@ -3818,11 -3665,7 +3739,10 @@@ void vp10_rd_pick_inter_mode_sub8x8(VP1 int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS]; int internal_active_edge = vp10_active_edge_sb(cpi, mi_row, mi_col) && vp10_internal_image_edge(cpi); +#if CONFIG_EXT_TX + mbmi->ext_txfrm = NORM; +#endif - x->skip_encode = sf->skip_encode_frame && x->q_index < QIDX_SKIP_THRESH; memset(x->zcoeff_blk[TX_4X4], 0, 4); vp10_zero(best_mbmode);