Fix for supertx with rect-tx

Change-Id: I0cc3523a8992f889f8dd203449ceb55f2a422324

diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index 98fbcf62d880f75fdc711c06ffcf431eac72276a..3c0fdef679e82affd78e447be99434cbba651a53 100644 (file)
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -418,8 +418,9 @@ static const TX_TYPE intra_mode_to_tx_type_context[INTRA_MODES] = {
 
 #if CONFIG_SUPERTX
 static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
-  return (int)mbmi->tx_size > VPXMIN(b_width_log2_lookup[mbmi->sb_type],
-                                     b_height_log2_lookup[mbmi->sb_type]);
+  return (int)txsize_sqr_map[mbmi->tx_size] >
+         VPXMIN(b_width_log2_lookup[mbmi->sb_type],
+                b_height_log2_lookup[mbmi->sb_type]);
 }
 #endif  // CONFIG_SUPERTX
 
@@ -733,8 +734,8 @@ static INLINE TX_SIZE get_uv_tx_size(const MB_MODE_INFO *mbmi,
                                      const struct macroblockd_plane *pd) {
 #if CONFIG_SUPERTX
   if (supertx_enabled(mbmi))
-    return uvsupertx_size_lookup[mbmi->tx_size][pd->subsampling_x]
-                                [pd->subsampling_y];
+    return uvsupertx_size_lookup[txsize_sqr_map[mbmi->tx_size]]
+                                [pd->subsampling_x][pd->subsampling_y];
 #endif  // CONFIG_SUPERTX
   return get_uv_tx_size_impl(mbmi->tx_size, mbmi->sb_type, pd->subsampling_x,
                              pd->subsampling_y);

diff --git a/vp10/encoder/rd.c b/vp10/encoder/rd.c
index 0075d21221a25a5cb3b963787a41872117e26618..77c939f94ce1aa54087123fd7e08170642727f3a 100644 (file)
--- a/vp10/encoder/rd.c
+++ b/vp10/encoder/rd.c
@@ -593,7 +593,7 @@ static void get_entropy_contexts_plane(
       for (i = 0; i < num_4x4_h; i += 8)
         t_left[i] = !!*(const uint64_t *)&left[i];
       break;
-#if CONFIG_EXT_TX
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
     case TX_4X8:
       memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
       for (i = 0; i < num_4x4_h; i += 2)
@@ -628,7 +628,7 @@ static void get_entropy_contexts_plane(
       for (i = 0; i < num_4x4_h; i += 4)
         t_left[i] = !!*(const uint32_t *)&left[i];
       break;
-#endif  // CONFIG_EXT_TX
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     default: assert(0 && "Invalid transform size."); break;
   }
 }