From 8ee5ab9f1348bf66912b8c46f1e76a501ee08f7b Mon Sep 17 00:00:00 2001 From: Debargha Mukherjee Date: Mon, 29 Aug 2016 11:04:17 -0700 Subject: [PATCH] Fix for supertx with rect-tx Change-Id: I0cc3523a8992f889f8dd203449ceb55f2a422324 --- vp10/common/blockd.h | 9 +++++---- vp10/encoder/rd.c | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h index 98fbcf62d..3c0fdef67 100644 --- a/vp10/common/blockd.h +++ b/vp10/common/blockd.h @@ -418,8 +418,9 @@ static const TX_TYPE intra_mode_to_tx_type_context[INTRA_MODES] = { #if CONFIG_SUPERTX static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) { - return (int)mbmi->tx_size > VPXMIN(b_width_log2_lookup[mbmi->sb_type], - b_height_log2_lookup[mbmi->sb_type]); + return (int)txsize_sqr_map[mbmi->tx_size] > + VPXMIN(b_width_log2_lookup[mbmi->sb_type], + b_height_log2_lookup[mbmi->sb_type]); } #endif // CONFIG_SUPERTX @@ -733,8 +734,8 @@ static INLINE TX_SIZE get_uv_tx_size(const MB_MODE_INFO *mbmi, const struct macroblockd_plane *pd) { #if CONFIG_SUPERTX if (supertx_enabled(mbmi)) - return uvsupertx_size_lookup[mbmi->tx_size][pd->subsampling_x] - [pd->subsampling_y]; + return uvsupertx_size_lookup[txsize_sqr_map[mbmi->tx_size]] + [pd->subsampling_x][pd->subsampling_y]; #endif // CONFIG_SUPERTX return get_uv_tx_size_impl(mbmi->tx_size, mbmi->sb_type, pd->subsampling_x, pd->subsampling_y); diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c index 0075d2122..77c939f94 100644 --- a/vp10/encoder/rd.c +++ b/vp10/encoder/rd.c @@ -593,7 +593,7 @@ static void get_entropy_contexts_plane( for (i = 0; i < num_4x4_h; i += 8) t_left[i] = !!*(const uint64_t *)&left[i]; break; -#if CONFIG_EXT_TX +#if CONFIG_EXT_TX && CONFIG_RECT_TX case TX_4X8: memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w); for (i = 0; i < num_4x4_h; i += 2) @@ -628,7 +628,7 @@ static void get_entropy_contexts_plane( for (i = 0; i < num_4x4_h; i += 4) t_left[i] = !!*(const uint32_t *)&left[i]; break; -#endif // CONFIG_EXT_TX +#endif // CONFIG_EXT_TX && CONFIG_RECT_TX default: assert(0 && "Invalid transform size."); break; } } -- 2.50.1