From c741a4fe037b6d003f93c96071c54f650da4dcad Mon Sep 17 00:00:00 2001
From: Yue Chen <chenyue1212@gmail.com>
Date: Mon, 20 Oct 2014 14:17:33 -0700
Subject: [PATCH] Fixing skip flag bugs in recent experiments

Bugs were in vp9_rdopt.c
Also did minor clean-ups in vp9_encodeframe.c

Change-Id: I6fec18e349cd0b810b0772e506927b423db077b6
---
 vp9/encoder/vp9_encodeframe.c | 10 ++++++----
 vp9/encoder/vp9_rdopt.c       |  6 +++---
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index ff315f3fd..0650b79fa 100644
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -4844,11 +4844,12 @@ static void rd_supertx_sb(VP9_COMP *cpi, const TileInfo *const tile,
     vp9_subtract_plane(x, bsize, plane);
 #if CONFIG_EXT_TX
     if (bsize <= BLOCK_16X16 && plane == 0) {
-      int txfm, this_rate_tx, pnskip_tx,
+      EXT_TX_TYPE txfm;
+      int this_rate_tx, pnskip_tx,
           best_tx_nostx = xd->mi[0]->mbmi.ext_txfrm;
       int64_t this_dist_tx, pnsse_tx, rd, bestrd_tx = INT64_MAX;
 
-      for (txfm = 0; txfm < EXT_TX_TYPES; txfm++) {
+      for (txfm = NORM; txfm < EXT_TX_TYPES; txfm++) {
         xd->mi[0]->mbmi.ext_txfrm = txfm;
         txfm_rd_in_plane_supertx(x, &this_rate_tx, &this_dist_tx,
                                  &pnskip_tx, &pnsse_tx,
@@ -4864,9 +4865,10 @@ static void rd_supertx_sb(VP9_COMP *cpi, const TileInfo *const tile,
           pnsse = pnsse_tx;
         }
       }
-
       xd->mi[0]->mbmi.ext_txfrm = best_tx_nostx;
     } else {
+      xd->mi[0]->mbmi.ext_txfrm = NORM;
+
 #endif
       txfm_rd_in_plane_supertx(x, &this_rate, &this_dist, &pnskip, &pnsse,
                                INT64_MAX, plane, bsize, tx_size, 0);
@@ -4898,7 +4900,7 @@ static void rd_supertx_sb(VP9_COMP *cpi, const TileInfo *const tile,
 #if CONFIG_EXT_TX
       if (bsize <= BLOCK_16X16)
         *tmp_rate -= vp9_cost_bit(cm->fc.ext_tx_prob, *best_tx);
-      *best_tx = 0;
+      *best_tx = NORM;
 #endif
       x->skip = 1;
     }
diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index 4721e30cf..7ff0575c2 100644
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -4583,6 +4583,7 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
           int prob_skip_cost = vp9_cost_bit(skip_prob, 1);
           rate2 += prob_skip_cost;
         }
+        this_skip2 = 1;
       } else if (!xd->lossless) {
         if (RDCOST(x->rdmult, x->rddiv, rate_y + rate_uv, distortion2) <
             RDCOST(x->rdmult, x->rddiv, 0, total_sse)) {
@@ -4598,6 +4599,7 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
         }
       } else {
         rate2 += vp9_cost_bit(vp9_get_skip_prob(cm, xd), 0);
+        this_skip2 = 0;
       }
 #if CONFIG_EXT_TX
       if (mbmi->tx_size < TX_32X32 && !this_skip2)
@@ -4641,8 +4643,6 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
     this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
 
     if (this_rd < best_rd) {
-      // int max_plane = MAX_MB_PLANE;
-
       *returnrate = rate2;
       *returndistortion = distortion2;
 #if CONFIG_SUPERTX
@@ -4658,7 +4658,7 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
       best_rd = this_rd;
       best_mbmode = *mbmi;
       best_skip2 = this_skip2;
-      // if (!x->select_tx_size) swap_block_ptr(x, ctx, 1, 0, 0, max_plane);
+      // if (!x->select_tx_size) swap_block_ptr(x, ctx, 1, 0, 0, MAX_MB_PLANE);
       vpx_memcpy(ctx->zcoeff_blk, x->zcoeff_blk[mbmi->tx_size],
                  sizeof(uint8_t) * ctx->num_4x4_blk);
     }
-- 
2.50.1