Implements several heuristics to prune mode search

Skips mode searches for intra and compound inter modes depending
on the best mode so far and the reference frames. The various
heuristics to be used are selected by bits from a flag. The
previous direction based intra mode search pruning is also absorbed
in this framework.

Specifically the flags and their impact are:

1) FLAG_SKIP_INTRA_BESTINTER (skip intra mode search for oblique
directional modes and TM_PRED if the best so far is
an inter mode)
derfraw300: -0.15%, 10% speedup

2) FLAG_SKIP_INTRA_DIRMISMATCH (skip D27, D63, D117 and D153
mode search if the best so far is not one of the closest
hor/vert/diagonal directions.
derfraw300: -0.05%, about 9% speedup

3) FLAG_SKIP_COMP_BESTINTRA (skip compound prediction mode
search if the best so far is an intra mode)
derfraw300: -0.06%, about 7-8% speedup

4) FLAG_SKIP_COMP_REFMISMATCH (skip compound prediction search
if the best single ref inter mode does not have the same ref
as one of the two references being tested in the compound mode)
derfraw300: -0.56%, about 10% speedup

Change-Id: I1a736cd29b36325489e7af9f32698d6394b2c495

diff --git a/vp9/encoder/vp9_encodeframe.c b/vp9/encoder/vp9_encodeframe.c
index 1a42961b5b6cda990e5e3a87e1c28a55845fd3da..a33dbe7c0f98ed49839fbf2b0720f42a3688ab20 100644 (file)
--- a/vp9/encoder/vp9_encodeframe.c
+++ b/vp9/encoder/vp9_encodeframe.c
@@ -2018,14 +2018,14 @@ static void select_txfm_mode(VP9_COMP *cpi) {
   } else if (cpi->common.current_video_frame == 0) {
     cpi->common.txfm_mode = TX_MODE_SELECT;
   } else {
-    if (cpi->sf.tx_size_search_method == USE_FULL_RD) {
+    if (cpi->sf.tx_size_search_method == USE_LARGESTALL) {
+      cpi->common.txfm_mode = ALLOW_32X32;
+    } else if (cpi->sf.tx_size_search_method == USE_FULL_RD) {
       int frame_type = get_frame_type(cpi);
       cpi->common.txfm_mode =
           cpi->rd_tx_select_threshes[frame_type][ALLOW_32X32]
           > cpi->rd_tx_select_threshes[frame_type][TX_MODE_SELECT] ?
           ALLOW_32X32 : TX_MODE_SELECT;
-    } else if (cpi->sf.tx_size_search_method == USE_LARGESTALL) {
-      cpi->common.txfm_mode = ALLOW_32X32;
     } else {
       unsigned int total = 0;
       int i;

diff --git a/vp9/encoder/vp9_onyx_if.c b/vp9/encoder/vp9_onyx_if.c
index 66c9142c7a182956fb85c5e81608bc95a8ff42b3..34645c66c567b3d1c3e33233de695adab2ad9fa2 100644 (file)
--- a/vp9/encoder/vp9_onyx_if.c
+++ b/vp9/encoder/vp9_onyx_if.c
@@ -718,7 +718,7 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
   sf->adjust_partitioning_from_last_frame = 0;
   sf->last_partitioning_redo_frequency = 4;
   sf->disable_splitmv = 0;
-  sf->conditional_oblique_intramodes = 0;
+  sf->mode_search_skip_flags = 0;
 
   // Skip any mode not chosen at size < X for all sizes > X
   // Hence BLOCK_SIZE_SB64X64 (skip is off)
@@ -755,13 +755,15 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
                                       cpi->common.show_frame == 0) ?
                                      USE_FULL_RD :
                                      USE_LARGESTALL);
-        sf->conditional_oblique_intramodes = 1;
-        sf->disable_splitmv =
-            (MIN(cpi->common.width, cpi->common.height) >= 720)? 1 : 0;
         sf->use_square_partition_only = !(cpi->common.frame_type == KEY_FRAME ||
                                    cpi->common.intra_only ||
                                    cpi->common.show_frame == 0);
+        sf->disable_splitmv =
+            (MIN(cpi->common.width, cpi->common.height) >= 720)? 1 : 0;
         sf->unused_mode_skip_lvl = BLOCK_SIZE_SB32X32;
+        sf->mode_search_skip_flags = FLAG_SKIP_INTRA_DIRMISMATCH |
+                                     FLAG_SKIP_INTRA_BESTINTER |
+                                     FLAG_SKIP_COMP_BESTINTRA;
       }
       if (speed == 2) {
         sf->adjust_thresholds_by_speed = 1;
@@ -771,12 +773,19 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
         sf->use_lastframe_partitioning = 1;
         sf->adjust_partitioning_from_last_frame = 1;
         sf->last_partitioning_redo_frequency = 3;
-        sf->tx_size_search_method = USE_LARGESTALL;
-        sf->conditional_oblique_intramodes = 1;
         sf->unused_mode_skip_lvl = BLOCK_SIZE_SB32X32;
         sf->reduce_first_step_size = 1;
         sf->optimize_coefficients = 0;
         // sf->reference_masking = 1;
+        sf->tx_size_search_method = ((cpi->common.frame_type == KEY_FRAME ||
+                                      cpi->common.intra_only ||
+                                      cpi->common.show_frame == 0) ?
+                                     USE_FULL_RD :
+                                     USE_LARGESTALL);
+        sf->mode_search_skip_flags = FLAG_SKIP_INTRA_DIRMISMATCH |
+                                     FLAG_SKIP_INTRA_BESTINTER |
+                                     FLAG_SKIP_COMP_BESTINTRA |
+                                     FLAG_SKIP_COMP_REFMISMATCH;
       }
       if (speed == 3) {
         sf->comp_inter_joint_search_thresh = BLOCK_SIZE_TYPES;
@@ -787,7 +796,10 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
                                      USE_FULL_RD :
                                      USE_LARGESTALL);
         sf->reduce_first_step_size = 1;
-        sf->conditional_oblique_intramodes = 1;
+        sf->mode_search_skip_flags = FLAG_SKIP_INTRA_DIRMISMATCH |
+                                     FLAG_SKIP_INTRA_BESTINTER |
+                                     FLAG_SKIP_COMP_BESTINTRA |
+                                     FLAG_SKIP_COMP_REFMISMATCH;
       }
       if (speed == 4) {
         sf->comp_inter_joint_search_thresh = BLOCK_SIZE_TYPES;
@@ -798,7 +810,10 @@ void vp9_set_speed_features(VP9_COMP *cpi) {
                                       cpi->common.show_frame == 0) ?
                                      USE_FULL_RD :
                                      USE_LARGESTALL);
-        sf->conditional_oblique_intramodes = 1;
+        sf->mode_search_skip_flags = FLAG_SKIP_INTRA_DIRMISMATCH |
+                                     FLAG_SKIP_INTRA_BESTINTER |
+                                     FLAG_SKIP_COMP_BESTINTRA |
+                                     FLAG_SKIP_COMP_REFMISMATCH;
       }
       /*
       if (speed == 2) {

diff --git a/vp9/encoder/vp9_onyx_int.h b/vp9/encoder/vp9_onyx_int.h
index 96a55b9483ec1502ede12f3add8f97f854788540..d90ada9fdf978c7c9870726f5f414ed1d9399940 100644 (file)
--- a/vp9/encoder/vp9_onyx_int.h
+++ b/vp9/encoder/vp9_onyx_int.h
@@ -207,6 +207,30 @@ typedef enum {
   USE_LARGESTALL
 } TX_SIZE_SEARCH_METHOD;
 
+typedef enum {
+  // Values should be powers of 2 so that they can be selected as bits of
+  // an integer flags field
+
+  // terminate search early based on distortion so far compared to
+  // qp step, distortion in the neighborhood of the frame, etc.
+  FLAG_EARLY_TERMINATE = 1,
+
+  // skips comp inter modes if the best so far is an intra mode
+  FLAG_SKIP_COMP_BESTINTRA = 2,
+
+  // skips comp inter modes if the best single intermode so far does
+  // not have the same reference as one of the two references being
+  // tested
+  FLAG_SKIP_COMP_REFMISMATCH = 4,
+
+  // skips oblique intra modes if the best so far is an inter mode
+  FLAG_SKIP_INTRA_BESTINTER = 8,
+
+  // skips oblique intra modes  at angles 27, 63, 117, 153 if the best
+  // intra so far is not one of the neighboring directions
+  FLAG_SKIP_INTRA_DIRMISMATCH = 16,
+} MODE_SEARCH_SKIP_LOGIC;
+
 typedef struct {
   int RD;
   SEARCH_METHODS search_method;
@@ -245,10 +269,11 @@ typedef struct {
   int adjust_partitioning_from_last_frame;
   int last_partitioning_redo_frequency;
   int disable_splitmv;
-  // Search the D27, D63, D117 and D153 modes
-  // only if the best intra mode so far is one
-  // of the two directional modes nearest to each.
-  int conditional_oblique_intramodes;
+
+  // Implements various heuristics to skip searching modes
+  // The heuristics selected are based on  flags
+  // defined in the MODE_SEARCH_SKIP_HEURISTICS enum
+  unsigned int mode_search_skip_flags;
 } SPEED_FEATURES;
 
 enum BlockSize {

diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c
index e3e84e8594dfb573d2713b6c5d03795008760486..d72025868cccdaa56fad4d6f13c6a5f04db796c3 100644 (file)
--- a/vp9/encoder/vp9_rdopt.c
+++ b/vp9/encoder/vp9_rdopt.c
@@ -1146,8 +1146,8 @@ static void super_block_yrd(VP9_COMP *cpi,
     *psse = sse[mbmi->txfm_size];
 }
 
-static int conditional_skip(MB_PREDICTION_MODE mode,
-                            MB_PREDICTION_MODE best_intra_mode) {
+static int conditional_skipintra(MB_PREDICTION_MODE mode,
+                                 MB_PREDICTION_MODE best_intra_mode) {
   if (mode == D117_PRED &&
       best_intra_mode != V_PRED &&
       best_intra_mode != D135_PRED)
@@ -1206,8 +1206,8 @@ static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
     int ratey = 0;
     // Only do the oblique modes if the best so far is
     // one of the neighboring directional modes
-    if (cpi->sf.conditional_oblique_intramodes) {
-      if (conditional_skip(mode, *best_mode))
+    if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
+      if (conditional_skipintra(mode, *best_mode))
           continue;
     }
 
@@ -2916,7 +2916,10 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
   vp9_prob comp_mode_p;
   int64_t best_overall_rd = INT64_MAX;
   int64_t best_intra_rd = INT64_MAX;
+  int64_t best_inter_rd = INT64_MAX;
   MB_PREDICTION_MODE best_intra_mode = DC_PRED;
+  // MB_PREDICTION_MODE best_inter_mode = ZEROMV;
+  MV_REFERENCE_FRAME best_inter_ref_frame = LAST_FRAME;
   INTERPOLATIONFILTERTYPE best_filter = SWITCHABLE;
   INTERPOLATIONFILTERTYPE tmp_best_filter = SWITCHABLE;
   int rate_uv_intra[TX_SIZE_MAX_SB], rate_uv_tokenonly[TX_SIZE_MAX_SB];
@@ -3031,6 +3034,7 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
     int i;
     int this_skip2 = 0;
     int64_t total_sse = INT_MAX;
+    int early_term = 0;
 
     for (i = 0; i < NB_TXFM_MODES; ++i)
       txfm_cache[i] = INT64_MAX;
@@ -3088,6 +3092,16 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
       continue;
     }
 
+    comp_pred = mbmi->ref_frame[1] > INTRA_FRAME;
+    if (comp_pred) {
+      if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA)
+        if (vp9_mode_order[best_mode_index].ref_frame == INTRA_FRAME)
+          continue;
+      if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_COMP_REFMISMATCH)
+        if (vp9_mode_order[mode_index].ref_frame != best_inter_ref_frame &&
+            vp9_mode_order[mode_index].second_ref_frame != best_inter_ref_frame)
+          continue;
+    }
     // TODO(jingning, jkoleszar): scaling reference frame not supported for
     // SPLITMV.
     if (mbmi->ref_frame[0] > 0 &&
@@ -3108,7 +3122,6 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
 
     set_scale_factors(xd, mbmi->ref_frame[0], mbmi->ref_frame[1],
                       scale_factor);
-    comp_pred = mbmi->ref_frame[1] > INTRA_FRAME;
     mbmi->mode = this_mode;
     mbmi->uv_mode = DC_PRED;
 
@@ -3186,6 +3199,12 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
     if (this_mode == I4X4_PRED) {
       int rate;
 
+      /*
+      if ((cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_BESTINTER) &&
+          (vp9_mode_order[best_mode_index].ref_frame > INTRA_FRAME))
+        continue;
+        */
+
       mbmi->txfm_size = TX_4X4;
       rd_pick_intra4x4mby_modes(cpi, x, &rate, &rate_y,
                                 &distortion_y, INT64_MAX);
@@ -3205,8 +3224,13 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
       TX_SIZE uv_tx;
       // Only search the oblique modes if the best so far is
       // one of the neighboring directional modes
-      if (cpi->sf.conditional_oblique_intramodes) {
-        if (conditional_skip(mbmi->mode, best_intra_mode))
+      if ((cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_BESTINTER) &&
+          (this_mode >= D45_PRED && this_mode <= TM_PRED)) {
+        if (vp9_mode_order[best_mode_index].ref_frame > INTRA_FRAME)
+          continue;
+      }
+      if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_INTRA_DIRMISMATCH) {
+        if (conditional_skipintra(mbmi->mode, best_intra_mode))
             continue;
       }
       super_block_yrd(cpi, x, &rate_y, &distortion_y, &skippable, NULL,
@@ -3249,6 +3273,16 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
       PARTITION_INFO tmp_best_partition;
       int pred_exists = 0;
       int uv_skippable;
+      if (is_comp_pred) {
+        if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA)
+          if (vp9_mode_order[best_mode_index].ref_frame == INTRA_FRAME)
+            continue;
+        if (cpi->sf.mode_search_skip_flags & FLAG_SKIP_COMP_REFMISMATCH)
+          if (vp9_mode_order[mode_index].ref_frame != best_inter_ref_frame &&
+              vp9_mode_order[mode_index].second_ref_frame !=
+              best_inter_ref_frame)
+            continue;
+      }
 
       this_rd_thresh = (mbmi->ref_frame[0] == LAST_FRAME) ?
           cpi->rd_threshes[bsize][THR_NEWMV] :
@@ -3441,13 +3475,22 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
       this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
     }
 
-    // Keep record of best intra distortion
+    // Keep record of best intra rd
     if (xd->mode_info_context->mbmi.ref_frame[0] == INTRA_FRAME &&
         xd->mode_info_context->mbmi.mode <= TM_PRED &&
         this_rd < best_intra_rd) {
       best_intra_rd = this_rd;
       best_intra_mode = xd->mode_info_context->mbmi.mode;
     }
+    // Keep record of best inter rd with single reference
+    if (xd->mode_info_context->mbmi.ref_frame[0] > INTRA_FRAME &&
+        xd->mode_info_context->mbmi.ref_frame[1] == NONE &&
+        !mode_excluded &&
+        this_rd < best_inter_rd) {
+      best_inter_rd = this_rd;
+      best_inter_ref_frame = ref_frame;
+      // best_inter_mode = xd->mode_info_context->mbmi.mode;
+    }
 
     if (!disable_skip && mbmi->ref_frame[0] == INTRA_FRAME)
       for (i = 0; i < NB_PREDICTION_TYPES; ++i)
@@ -3475,6 +3518,8 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
     if (this_rd < best_rd || x->skip) {
       if (!mode_excluded) {
         // Note index of best mode so far
+        const int qstep = xd->plane[0].dequant[1];
+
         best_mode_index = mode_index;
 
         if (ref_frame == INTRA_FRAME) {
@@ -3492,6 +3537,12 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
         if (this_mode == I4X4_PRED || this_mode == SPLITMV)
           for (i = 0; i < 4; i++)
             best_bmodes[i] = xd->mode_info_context->bmi[i];
+
+        // TODO(debargha): enhance this test with a better distortion prediction
+        // based on qp, activity mask and history
+        if (cpi->sf.mode_search_skip_flags & FLAG_EARLY_TERMINATE)
+          if (ref_frame > INTRA_FRAME && distortion2 * 4 < qstep * qstep)
+            early_term = 1;
       }
 #if 0
       // Testing this mode gave rise to an improvement in best error score.
@@ -3567,6 +3618,9 @@ int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
       }
     }
 
+    if (early_term)
+      break;
+
     if (x->skip && !mode_excluded)
       break;
   }