Properly handle non-420 color format in recursive transform scheme

This commit makes the recursive transform block partitioning properly
handle the non-420 color format. It resolves an enc/dec mismatch
issue in that setting when var-tx experiment is turned on.

Change-Id: I48a91de02c11b3153f897d1cca0ae948eec15605

diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c
index e178a4d137168179beed9602bb272d5be4829380..874f9f85cbf331bb9fc8f06bdeaf50bbc23c8d02 100644 (file)
--- a/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@ -115,7 +115,11 @@ static void read_tx_size_inter(VP10_COMMON *cm, MACROBLOCKD *xd,
       read_tx_size_inter(cm, xd, mbmi, tx_size - 1, offsetr, offsetc, r);
     }
   } else {
+    int idx, idy;
     mbmi->inter_tx_size[tx_idx] = tx_size;
+    for (idy = 0; idy < (1 << tx_size) / 2; ++idy)
+      for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
+        mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
     mbmi->tx_size = mbmi->inter_tx_size[tx_idx];
   }
 }

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 777139055e5db82a1ca648aa757fc874446a5197..738408454f0e7152c4b79687d629991dcd282ddf 100644 (file)
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1635,9 +1635,14 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
   }
 
   if (this_rd < sum_rd) {
+    int idx, idy;
     for (i = 0; i < (1 << tx_size); ++i)
       pta[i] = ptl[i] = !(tmp_eob == 0);
     mbmi->inter_tx_size[tx_idx] = tx_size;
+
+    for (idy = 0; idy < (1 << tx_size) / 2; ++idy)
+      for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
+        mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
     mbmi->tx_size = tx_size;
   } else {
     *rate = sum_rate;