From e7c0e157d273b73f39d8545c1400d67b04385a66 Mon Sep 17 00:00:00 2001
From: Geza Lore <gezalore@gmail.com>
Date: Thu, 21 Jan 2016 10:46:33 +0000
Subject: [PATCH] Set inter_tx_size for supertx coded blocks.

The loop filter relies on inter_tx_size in MB_MODE_INFO being set
properly when VAR_TX is enabled. Supertx coded blocks did not set this
previously at all, and the differing garbage values eventually resulted
in in a YUV mismatch between encoder and decoder after loop filtering.

This patch fixes this by setting inter_tx_size to the proper supertx
size in both the encoder and the decoder. This should also mean that
loop filtering is done at the proper transform boundaries, even when
supertx or vartx is being used.

Change-Id: I41a564cd6d34ce4a8313ad4efa89d905f5ead731
---
 vp10/common/blockd.h       |  3 +++
 vp10/decoder/decodeframe.c |  3 +++
 vp10/decoder/decodemv.c    | 11 +++++++++++
 vp10/encoder/encodeframe.c | 11 +++++++++++
 4 files changed, 28 insertions(+)

diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index 8011456a3..27e33bafa 100644
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -281,6 +281,9 @@ typedef struct macroblockd {
   TXFM_CONTEXT left_txfm_context_buffer[8];
 
   TX_SIZE max_tx_size;
+#if CONFIG_SUPERTX
+  TX_SIZE supertx_size;
+#endif
 #endif
 
   // dimension in the unit of 8x8 block of the current block
diff --git a/vp10/decoder/decodeframe.c b/vp10/decoder/decodeframe.c
index 2237bef28..c2fbc121f 100644
--- a/vp10/decoder/decodeframe.c
+++ b/vp10/decoder/decodeframe.c
@@ -1829,6 +1829,9 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
       }
 #endif  // CONFIG_EXT_TX
     }
+#if CONFIG_VAR_TX
+    xd->supertx_size = supertx_size;
+#endif
   }
 #endif  // CONFIG_SUPERTX
   if (!hbs) {
diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c
index 321b7e32f..44b0fc7ec 100644
--- a/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@ -1283,6 +1283,17 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi,
 #endif  // CONFIG_VAR_TX
 #if CONFIG_SUPERTX
   }
+#if CONFIG_VAR_TX
+  else if (inter_block) {
+    const int width  = num_4x4_blocks_wide_lookup[bsize];
+    const int height = num_4x4_blocks_high_lookup[bsize];
+    int idx, idy;
+    xd->mi[0]->mbmi.tx_size = xd->supertx_size;
+    for (idy = 0; idy < height; ++idy)
+      for (idx = 0; idx < width; ++idx)
+        xd->mi[0]->mbmi.inter_tx_size[(idy >> 1) * 8 + (idx >> 1)] = xd->supertx_size;
+  }
+#endif  // CONFIG_VAR_TX
 #endif  // CONFIG_SUPERTX
 
   if (inter_block)
diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 75f75d250..11c1bdd1b 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -1236,6 +1236,7 @@ static void update_state_supertx(VP10_COMP *cpi, ThreadData *td,
   *mi_addr = *mi;
   *x->mbmi_ext = ctx->mbmi_ext;
   assert(is_inter_block(mbmi));
+  assert(mbmi->tx_size == ctx->mic.mbmi.tx_size);
 
   // If segmentation in use
   if (seg->enabled && output_enabled) {
@@ -1309,6 +1310,16 @@ static void update_state_supertx(VP10_COMP *cpi, ThreadData *td,
       mv->mv[1].as_int = mi->mbmi.mv[1].as_int;
     }
   }
+
+#if CONFIG_VAR_TX
+  {
+    const TX_SIZE mtx = mbmi->tx_size;
+    int idy, idx;
+    for (idy = 0; idy < (1 << mtx) / 2; ++idy)
+      for (idx = 0; idx < (1 << mtx) / 2; ++idx)
+        mbmi->inter_tx_size[(idy << 3) + idx] = mbmi->tx_size;
+  }
+#endif
 }
 
 static void update_state_sb_supertx(VP10_COMP *cpi, ThreadData *td,
-- 
2.40.0