From: Jingning Han <jingning@google.com>
Date: Wed, 8 Jun 2016 00:54:20 +0000 (-0700)
Subject: Trellis based adaptive quantization
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=25ca32295772f84f91ec73c77fc78784ddb9a15a;p=libvpx

Trellis based adaptive quantization

This commit combines uniform quantizer with trellis based coefficient
level optimization. It improves the codebase compression performance:

lowres 0.8%
midres 1.0%
hdres  1.6%

Note that the current trellis optimization unit is using C code. This
will make the cost of the overall quantization process slower. A number
of optimizations will come up next.

Change-Id: Id441dd238e4844409d0f08f82604be777f3f5282
---

diff --git a/vp10/encoder/encodemb.c b/vp10/encoder/encodemb.c
index dfb72ea94..9d0eb6674 100644
--- a/vp10/encoder/encodemb.c
+++ b/vp10/encoder/encodemb.c
@@ -52,10 +52,6 @@ void vp10_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane) {
                      pd->dst.buf, pd->dst.stride);
 }
 
-#define RDTRUNC(RM, DM, R, D)                        \
-  (((1 << (VP9_PROB_COST_SHIFT - 1)) + (R) * (RM)) & \
-   ((1 << VP9_PROB_COST_SHIFT) - 1))
-
 typedef struct vp10_token_state {
   int           rate;
   int           error;
@@ -65,16 +61,12 @@ typedef struct vp10_token_state {
 } vp10_token_state;
 
 // TODO(jimbankoski): experiment to find optimal RD numbers.
-static const int plane_rd_mult[REF_TYPES][PLANE_TYPES] ={ {9, 7}, {7, 5}, };
+static const int plane_rd_mult[REF_TYPES][PLANE_TYPES] ={ {9, 7}, {8, 5}, };
 
 #define UPDATE_RD_COST()\
 {\
   rd_cost0 = RDCOST(rdmult, rddiv, rate0, error0);\
   rd_cost1 = RDCOST(rdmult, rddiv, rate1, error1);\
-  if (rd_cost0 == rd_cost1) {\
-    rd_cost0 = RDTRUNC(rdmult, rddiv, rate0, error0);\
-    rd_cost1 = RDTRUNC(rdmult, rddiv, rate1, error1);\
-  }\
 }
 
 // This function is a place holder for now but may ultimately need
@@ -90,8 +82,8 @@ static int trellis_get_coeff_context(const int16_t *scan,
   return pt;
 }
 
-static int optimize_b(MACROBLOCK *mb, int plane, int block,
-                      TX_SIZE tx_size, int ctx) {
+int vp10_optimize_b(MACROBLOCK *mb, int plane, int block,
+                    TX_SIZE tx_size, int ctx) {
   MACROBLOCKD *const xd = &mb->e_mbd;
   struct macroblock_plane *const p = &mb->plane[plane];
   struct macroblockd_plane *const pd = &xd->plane[plane];
@@ -961,7 +953,7 @@ static void encode_block(int plane, int block, int blk_row, int blk_col,
                                tx_size);
 #else
           vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
-                           tx_size, VP10_XFORM_QUANT_B);
+                           tx_size, VP10_XFORM_QUANT_FP);
 #endif  // CONFIG_NEW_QUANT
         } else if (x->skip_txfm[plane][blk_index] == SKIP_TXFM_AC_ONLY) {
           // fast path forward transform and quantization
@@ -986,7 +978,7 @@ static void encode_block(int plane, int block, int blk_row, int blk_col,
                              tx_size);
 #else
         vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
-                         tx_size, VP10_XFORM_QUANT_B);
+                         tx_size, VP10_XFORM_QUANT_FP);
 #endif  // CONFIG_NEW_QUANT
       }
     }
@@ -1021,7 +1013,7 @@ static void encode_block(int plane, int block, int blk_row, int blk_col,
     }
 #endif
     ctx = combine_entropy_contexts(*a, *l);
-    *a = *l = optimize_b(x, plane, block, tx_size, ctx) > 0;
+    *a = *l = vp10_optimize_b(x, plane, block, tx_size, ctx) > 0;
   } else {
     *a = *l = p->eobs[block] > 0;
   }
@@ -1293,7 +1285,7 @@ void vp10_encode_block_intra(int plane, int block, int blk_row, int blk_col,
     if (x->optimize) {
       int ctx;
       ctx = combine_entropy_contexts(*a, *l);
-      *a = *l = optimize_b(x, plane, block, tx_size, ctx) > 0;
+      *a = *l = vp10_optimize_b(x, plane, block, tx_size, ctx) > 0;
     } else {
       *a = *l = p->eobs[block] > 0;
     }
diff --git a/vp10/encoder/encodemb.h b/vp10/encoder/encodemb.h
index eae1db799..cef6cccbd 100644
--- a/vp10/encoder/encodemb.h
+++ b/vp10/encoder/encodemb.h
@@ -56,6 +56,9 @@ void vp10_xform_quant_dc_fp_nuq(MACROBLOCK *x, int plane, int block,
                                 BLOCK_SIZE plane_bsize, TX_SIZE tx_size);
 #endif
 
+int vp10_optimize_b(MACROBLOCK *mb, int plane, int block,
+                    TX_SIZE tx_size, int ctx);
+
 void vp10_subtract_plane(MACROBLOCK *x, BLOCK_SIZE bsize, int plane);
 
 void vp10_encode_block_intra(int plane, int block, int blk_row, int blk_col,
diff --git a/vp10/encoder/quantize.c b/vp10/encoder/quantize.c
index 2a8b33f6c..8789bc501 100644
--- a/vp10/encoder/quantize.c
+++ b/vp10/encoder/quantize.c
@@ -1223,10 +1223,7 @@ void vp10_init_quantizer(VP10_COMP *cpi) {
     const int qrounding_factor = q == 0 ? 64 : 48;
 
     for (i = 0; i < 2; ++i) {
-      int qrounding_factor_fp = i == 0 ? 48 : 42;
-      if (q == 0)
-        qrounding_factor_fp = 64;
-
+      int qrounding_factor_fp = 64;
       // y
       quant = i == 0 ? vp10_dc_quant(q, cm->y_dc_delta_q, cm->bit_depth)
                      : vp10_ac_quant(q, 0, cm->bit_depth);
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 640a409c9..1cf43b5b3 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1213,6 +1213,10 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
   int rate;
   int64_t dist;
   int64_t sse;
+#if !CONFIG_NEW_QUANT
+  ENTROPY_CONTEXT coeff_ctx = combine_entropy_contexts(
+      *(args->t_above + blk_col), *(args->t_left + blk_row));
+#endif
 
   if (args->exit_early)
     return;
@@ -1264,9 +1268,10 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
                            plane_bsize, tx_size);
 #else
       vp10_xform_quant(x, plane, block, blk_row, blk_col,
-                       plane_bsize, tx_size, VP10_XFORM_QUANT_B);
+                       plane_bsize, tx_size, VP10_XFORM_QUANT_FP);
+      vp10_optimize_b(x, plane, block, tx_size, coeff_ctx);
 #endif  // CONFIG_NEW_QUANT
-      dist_block(args->cpi, x, plane, block, blk_row, blk_col,
+     dist_block(args->cpi, x, plane, block, blk_row, blk_col,
                  tx_size, &dist, &sse);
     } else if (x->skip_txfm[plane][block >> (tx_size << 1)] ==
                SKIP_TXFM_AC_ONLY) {
@@ -1318,7 +1323,8 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
                            tx_size);
 #else
     vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-                     VP10_XFORM_QUANT_B);
+                     VP10_XFORM_QUANT_FP);
+    vp10_optimize_b(x, plane, block, tx_size, coeff_ctx);
 #endif  // CONFIG_NEW_QUANT
     dist_block(args->cpi, x, plane, block, blk_row, blk_col,
                tx_size, &dist, &sse);
@@ -3076,6 +3082,8 @@ void vp10_tx_block_rd_b(const VP10_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
   vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
                    VP10_XFORM_QUANT_B);
 
+  vp10_optimize_b(x, plane, block, tx_size, coeff_ctx);
+
   // TODO(any): Use dist_block to compute distortion
 #if CONFIG_VP9_HIGHBITDEPTH
   if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {