Set inter_tx_size for supertx coded blocks.

The loop filter relies on inter_tx_size in MB_MODE_INFO being set
properly when VAR_TX is enabled. Supertx coded blocks did not set this
previously at all, and the differing garbage values eventually resulted
in in a YUV mismatch between encoder and decoder after loop filtering.

This patch fixes this by setting inter_tx_size to the proper supertx
size in both the encoder and the decoder. This should also mean that
loop filtering is done at the proper transform boundaries, even when
supertx or vartx is being used.

Change-Id: I41a564cd6d34ce4a8313ad4efa89d905f5ead731

diff --git a/vp10/common/blockd.h b/vp10/common/blockd.h
index 8011456a390ee0f8b0ebaf355942b9188d1924d1..27e33bafa6a1fc46ed7db1342f8ac0ee800284ca 100644 (file)
--- a/vp10/common/blockd.h
+++ b/vp10/common/blockd.h
@@ -281,6 +281,9 @@ typedef struct macroblockd {
   TXFM_CONTEXT left_txfm_context_buffer[8];
 
   TX_SIZE max_tx_size;
+#if CONFIG_SUPERTX
+  TX_SIZE supertx_size;
+#endif
 #endif
 
   // dimension in the unit of 8x8 block of the current block

diff --git a/vp10/decoder/decodeframe.c b/vp10/decoder/decodeframe.c
index 2237bef287f7dfa45c7319413dcf2c2912822b04..c2fbc121f81a7287b57b15f512a99e9ff10d50bf 100644 (file)
--- a/vp10/decoder/decodeframe.c
+++ b/vp10/decoder/decodeframe.c
@@ -1829,6 +1829,9 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
       }
 #endif  // CONFIG_EXT_TX
     }
+#if CONFIG_VAR_TX
+    xd->supertx_size = supertx_size;
+#endif
   }
 #endif  // CONFIG_SUPERTX
   if (!hbs) {

diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c
index 321b7e32f626a899fc42aec596f6a311ac4c840f..44b0fc7eca1db5b7c233141b7c488b66e99725d6 100644 (file)
--- a/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@ -1283,6 +1283,17 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi,
 #endif  // CONFIG_VAR_TX
 #if CONFIG_SUPERTX
   }
+#if CONFIG_VAR_TX
+  else if (inter_block) {
+    const int width  = num_4x4_blocks_wide_lookup[bsize];
+    const int height = num_4x4_blocks_high_lookup[bsize];
+    int idx, idy;
+    xd->mi[0]->mbmi.tx_size = xd->supertx_size;
+    for (idy = 0; idy < height; ++idy)
+      for (idx = 0; idx < width; ++idx)
+        xd->mi[0]->mbmi.inter_tx_size[(idy >> 1) * 8 + (idx >> 1)] = xd->supertx_size;
+  }
+#endif  // CONFIG_VAR_TX
 #endif  // CONFIG_SUPERTX
 
   if (inter_block)

diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 75f75d250be6522ad5694c39f39713400283234c..11c1bdd1b420961b7a6236163826191e6cfa3fa2 100644 (file)
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -1236,6 +1236,7 @@ static void update_state_supertx(VP10_COMP *cpi, ThreadData *td,
   *mi_addr = *mi;
   *x->mbmi_ext = ctx->mbmi_ext;
   assert(is_inter_block(mbmi));
+  assert(mbmi->tx_size == ctx->mic.mbmi.tx_size);
 
   // If segmentation in use
   if (seg->enabled && output_enabled) {
@@ -1309,6 +1310,16 @@ static void update_state_supertx(VP10_COMP *cpi, ThreadData *td,
       mv->mv[1].as_int = mi->mbmi.mv[1].as_int;
     }
   }
+
+#if CONFIG_VAR_TX
+  {
+    const TX_SIZE mtx = mbmi->tx_size;
+    int idy, idx;
+    for (idy = 0; idy < (1 << mtx) / 2; ++idy)
+      for (idx = 0; idx < (1 << mtx) / 2; ++idx)
+        mbmi->inter_tx_size[(idy << 3) + idx] = mbmi->tx_size;
+  }
+#endif
 }
 
 static void update_state_sb_supertx(VP10_COMP *cpi, ThreadData *td,