From: Ronald S. Bultje <rbultje@google.com>
Date: Fri, 12 Oct 2012 01:19:20 +0000 (-0700)
Subject: Remove reverting of tx-select if only a single txfm-size is used.
X-Git-Tag: v1.3.0~1217^2~246^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=5a4ea8bea6e3a056cd077fb39a913e0690ec8fee;p=libvpx

Remove reverting of tx-select if only a single txfm-size is used.

Entropy coding takes care of this anyway, and this causes changes to
the txfm size assigned to skip blocks, which can affect the loopfilter
output, thus causing encoder/decoding mismatches.

Change-Id: I591a8d8a4758a507986b751a9f83e6d76e406998
---

diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c
index 1606a19ba..4f2437fac 100644
--- a/vp8/encoder/encodeframe.c
+++ b/vp8/encoder/encodeframe.c
@@ -1458,6 +1458,32 @@ static int check_dual_ref_flags(VP8_COMP *cpi) {
   }
 }
 
+#if CONFIG_TX_SELECT
+static void reset_skip_txfm_size(VP8_COMP *cpi, TX_SIZE txfm_max) {
+  VP8_COMMON *cm = &cpi->common;
+  int mb_row, mb_col, mis = cm->mode_info_stride;
+  MODE_INFO *mi, *mi_ptr = cm->mi;
+  MB_MODE_INFO *mbmi;
+  MACROBLOCK *x = &cpi->mb;
+  MACROBLOCKD *xd = &x->e_mbd;
+
+  for (mb_row = 0; mb_row < cm->mb_rows; mb_row++, mi_ptr += mis) {
+    mi = mi_ptr;
+    for (mb_col = 0; mb_col < cm->mb_cols; mb_col++, mi++) {
+      mbmi = &mi->mbmi;
+      if (mbmi->txfm_size > txfm_max) {
+        int segment_id = mbmi->segment_id;
+        xd->mode_info_context = mi;
+        assert((segfeature_active(xd, segment_id, SEG_LVL_EOB) &&
+                get_segdata(xd, segment_id, SEG_LVL_EOB) == 0) ||
+               (cm->mb_no_coeff_skip && mbmi->mb_skip_coeff));
+        mbmi->txfm_size = txfm_max;
+      }
+    }
+  }
+}
+#endif
+
 void vp8_encode_frame(VP8_COMP *cpi) {
   if (cpi->sf.RD) {
     int i, frame_type, pred_type;
@@ -1606,8 +1632,10 @@ void vp8_encode_frame(VP8_COMP *cpi) {
 
       if (count4x4 == 0 && count16x16 == 0) {
         cpi->common.txfm_mode = ALLOW_8X8;
+        reset_skip_txfm_size(cpi, TX_8X8);
       } else if (count8x8 == 0 && count16x16 == 0 && count8x8_8x8p == 0) {
         cpi->common.txfm_mode = ONLY_4X4;
+        reset_skip_txfm_size(cpi, TX_4X4);
       } else if (count8x8 == 0 && count4x4 == 0) {
         cpi->common.txfm_mode = ALLOW_16X16;
       }