Improve skip_txfm thresholds in the non-rd mode selection

Modified the thresholds of deciding whether or not to skip
the transforms in model_rd_for_sb_y(). Used zbin[] instead
of dequant[] to be more precise. Also, modified the checking
coditions.

Rtc set borg test results (at speed 6) showed:
average PSNR gain: 0.138%, overall PSNR gain: 0.158%,
and SSIM gain: 0.177%.

The data rate test was modified slightly as suggested by
Marco.

Change-Id: Ieaf633ab77f4838cb3c45cf69065b29d55f8ae6c

diff --git a/test/datarate_test.cc b/test/datarate_test.cc
index e529347714cf10b63abc67d3b0fba010947f3726..94efeae97ac6e99096bd770a18ac06eff4402108 100644 (file)
--- a/test/datarate_test.cc
+++ b/test/datarate_test.cc
@@ -540,7 +540,7 @@ TEST_P(DatarateTestVP9Large, ChangingDropFrameThresh) {
         << " The first dropped frame for drop_thresh " << i
         << " > first dropped frame for drop_thresh "
         << i - kDropFrameThreshTestStep;
-    ASSERT_GE(num_drops_, last_num_drops)
+    ASSERT_GE(num_drops_, last_num_drops * 0.90)
         << " The number of dropped frames for drop_thresh " << i
         << " < number of dropped frames for drop_thresh "
         << i - kDropFrameThreshTestStep;

diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index 46a354700ab5c1bc1de2251775fddbd3124d61be..71cea0e45e0be4e01c0181603f50c39b0cd7369d 100644 (file)
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -216,6 +216,8 @@ static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
   int64_t dist;
   struct macroblock_plane *const p = &x->plane[0];
   struct macroblockd_plane *const pd = &xd->plane[0];
+  const int64_t dc_thr = p->quant_thred[0] >> 6;
+  const int64_t ac_thr = p->quant_thred[1] >> 6;
   const uint32_t dc_quant = pd->dequant[0];
   const uint32_t ac_quant = pd->dequant[1];
   unsigned int var = cpi->fn_ptr[bsize].vf(p->src.buf, p->src.stride,
@@ -223,12 +225,14 @@ static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
   *var_y = var;
   *sse_y = sse;
 
-  if (sse < dc_quant * dc_quant >> 6)
-    x->skip_txfm[0] = 1;
-  else if (var < ac_quant * ac_quant >> 6)
+  x->skip_txfm[0] = 0;
+  // Check if all ac coefficients can be quantized to zero.
+  if (var < ac_thr || var == 0) {
     x->skip_txfm[0] = 2;
-  else
-    x->skip_txfm[0] = 0;
+    // Check if dc coefficient can be quantized to zero.
+    if (sse - var < dc_thr || sse == var)
+      x->skip_txfm[0] = 1;
+  }
 
   if (cpi->common.tx_mode == TX_MODE_SELECT) {
     if (sse > (var << 2))