From: Jingning Han <jingning@google.com>
Date: Fri, 23 Oct 2015 21:27:21 +0000 (-0700)
Subject: Properly handle non-420 color format in recursive transform scheme
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=79fe7246c1dde199fed399ae3468fe828a5a7fb1;p=libvpx

Properly handle non-420 color format in recursive transform scheme

This commit makes the recursive transform block partitioning properly
handle the non-420 color format. It resolves an enc/dec mismatch
issue in that setting when var-tx experiment is turned on.

Change-Id: I48a91de02c11b3153f897d1cca0ae948eec15605
---

diff --git a/vp10/decoder/decodemv.c b/vp10/decoder/decodemv.c
index e178a4d13..874f9f85c 100644
--- a/vp10/decoder/decodemv.c
+++ b/vp10/decoder/decodemv.c
@@ -115,7 +115,11 @@ static void read_tx_size_inter(VP10_COMMON *cm, MACROBLOCKD *xd,
       read_tx_size_inter(cm, xd, mbmi, tx_size - 1, offsetr, offsetc, r);
     }
   } else {
+    int idx, idy;
     mbmi->inter_tx_size[tx_idx] = tx_size;
+    for (idy = 0; idy < (1 << tx_size) / 2; ++idy)
+      for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
+        mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
     mbmi->tx_size = mbmi->inter_tx_size[tx_idx];
   }
 }
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 777139055..738408454 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1635,9 +1635,14 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
   }
 
   if (this_rd < sum_rd) {
+    int idx, idy;
     for (i = 0; i < (1 << tx_size); ++i)
       pta[i] = ptl[i] = !(tmp_eob == 0);
     mbmi->inter_tx_size[tx_idx] = tx_size;
+
+    for (idy = 0; idy < (1 << tx_size) / 2; ++idy)
+      for (idx = 0; idx < (1 << tx_size) / 2; ++idx)
+        mbmi->inter_tx_size[tx_idx + (idy << 3) + idx] = tx_size;
     mbmi->tx_size = tx_size;
   } else {
     *rate = sum_rate;