From: Jingning Han <jingning@google.com>
Date: Wed, 10 Feb 2016 01:51:49 +0000 (-0800)
Subject: Resolve conflict between var-tx and super-tx
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=4c6c82a2e856df8c3740b7d27c9b11d633716be2;p=libvpx

Resolve conflict between var-tx and super-tx

This commit aligns the rate-distortion metric for the recursive
transform block partitioning and the super transform. It resolves
the conflicts between these two experiments. The coding performance
gains of the combined experiments (var-tx + super-tx) has been
improved:

derf   0.89%  ->  1.9%
hevcmr 1.06%  ->  1.8%
stdhd  0.29%  ->  1.4%
hevchr 0.80%  ->  2.3%

Change-Id: I7e33994ad70c1b2751435620815f867d82172f41
---

diff --git a/vp10/encoder/encodeframe.c b/vp10/encoder/encodeframe.c
index 18a49809b..9e20000ba 100644
--- a/vp10/encoder/encodeframe.c
+++ b/vp10/encoder/encodeframe.c
@@ -3164,7 +3164,7 @@ static void rd_pick_partition(VP10_COMP *cpi, ThreadData *td,
 #if CONFIG_VAR_TX
           xd->above_txfm_context = cm->above_txfm_context + mi_col;
           xd->left_txfm_context =
-              xd->left_txfm_context_buffer + (mi_row & 0x07);
+              xd->left_txfm_context_buffer + (mi_row & MI_MASK);
           restore_context(x, mi_row, mi_col, a, l, sa, sl, ta, tl, bsize);
 #else
           restore_context(x, mi_row, mi_col, a, l, sa, sl, bsize);
@@ -5188,6 +5188,11 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
   ext_tx_set = get_ext_tx_set(tx_size, bsize, 1);
 #endif  // CONFIG_EXT_TX
   for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
+#if CONFIG_VAR_TX
+    ENTROPY_CONTEXT ctxa[16], ctxl[16];
+    const struct macroblockd_plane *const pd = &xd->plane[0];
+    int coeff_ctx = 1;
+#endif  // CONFIG_VAR_TX
 #if CONFIG_EXT_TX
     if (!ext_tx_used_inter[ext_tx_set][tx_type])
       continue;
@@ -5201,12 +5206,23 @@ static void rd_supertx_sb(VP10_COMP *cpi, ThreadData *td,
       continue;
 #endif  // CONFIG_EXT_TX
     mbmi->tx_type = tx_type;
-    vp10_txfm_rd_in_plane_supertx(x,
+
 #if CONFIG_VAR_TX
-                                  cpi,
-#endif
-                                  &this_rate, &this_dist, &pnskip,
+    this_rate = 0;
+    this_dist = 0;
+    pnsse = 0;
+    pnskip = 1;
+
+    vp10_get_entropy_contexts(bsize, tx_size, pd, ctxa, ctxl);
+    coeff_ctx = combine_entropy_contexts(ctxa[0], ctxl[0]);
+    vp10_tx_block_rd_b(cpi, x, tx_size,
+                       0, 0, 0, 0,
+                       bsize, coeff_ctx,
+                       &this_rate, &this_dist, &pnsse, &pnskip);
+#else
+    vp10_txfm_rd_in_plane_supertx(x, &this_rate, &this_dist, &pnskip,
                                   &pnsse, INT64_MAX, 0, bsize, tx_size, 0);
+#endif  // CONFIG_VAR_TX
 
 #if CONFIG_EXT_TX
     if (get_ext_tx_types(tx_size, bsize, 1) > 1 &&
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index f22727220..9f21c5d34 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -2317,10 +2317,10 @@ static int64_t rd_pick_intra_sby_mode(VP10_COMP *cpi, MACROBLOCK *x,
 }
 
 #if CONFIG_VAR_TX
-static void tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
-                          int blk_row, int blk_col, int plane, int block,
-                          int plane_bsize, int coeff_ctx,
-                          int *rate, int64_t *dist, int64_t *bsse, int *skip) {
+void vp10_tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
+                        int blk_row, int blk_col, int plane, int block,
+                        int plane_bsize, int coeff_ctx,
+                        int *rate, int64_t *dist, int64_t *bsse, int *skip) {
   MACROBLOCKD *xd = &x->e_mbd;
   const struct macroblock_plane *const p = &x->plane[plane];
   struct macroblockd_plane *const pd = &xd->plane[plane];
@@ -2531,8 +2531,8 @@ static void select_tx_block(const VP10_COMP *cpi, MACROBLOCK *x,
 
   if (cpi->common.tx_mode == TX_MODE_SELECT || tx_size == TX_4X4) {
     mbmi->inter_tx_size[tx_idx] = tx_size;
-    tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
-                  plane_bsize, coeff_ctx, rate, dist, bsse, skip);
+    vp10_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
+                       plane_bsize, coeff_ctx, rate, dist, bsse, skip);
 
     if ((RDCOST(x->rdmult, x->rddiv, *rate, *dist) >=
          RDCOST(x->rdmult, x->rddiv, zero_blk_rate, *bsse) || *skip == 1) &&
@@ -2863,8 +2863,8 @@ static void tx_block_rd(const VP10_COMP *cpi, MACROBLOCK *x,
         break;
     }
     coeff_ctx = combine_entropy_contexts(ta[0], tl[0]);
-    tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
-                  plane_bsize, coeff_ctx, rate, dist, bsse, skip);
+    vp10_tx_block_rd_b(cpi, x, tx_size, blk_row, blk_col, plane, block,
+                       plane_bsize, coeff_ctx, rate, dist, bsse, skip);
     for (i = 0; i < (1 << tx_size); ++i) {
       ta[i] = !(p->eobs[block] == 0);
       tl[i] = !(p->eobs[block] == 0);
diff --git a/vp10/encoder/rdopt.h b/vp10/encoder/rdopt.h
index 62b0aeaeb..a6394fa45 100644
--- a/vp10/encoder/rdopt.h
+++ b/vp10/encoder/rdopt.h
@@ -74,6 +74,13 @@ void vp10_rd_pick_inter_mode_sub8x8(struct VP10_COMP *cpi,
                                     int64_t best_rd_so_far);
 
 #if CONFIG_SUPERTX
+#if CONFIG_VAR_TX
+void vp10_tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
+                        int blk_row, int blk_col, int plane, int block,
+                        int plane_bsize, int coeff_ctx,
+                        int *rate, int64_t *dist, int64_t *bsse, int *skip);
+#endif
+
 void vp10_txfm_rd_in_plane_supertx(MACROBLOCK *x,
 #if CONFIG_VAR_TX
                                    const VP10_COMP *cpi,