Make non-RD intra mode search txfm size dependent

This commit fixes the potential issue in the non-RD mode decision
flow that only checks part of the block to estimate the cost. It
was due to the use of fixed transform size, in replacing the
largest transform block size. This commit enables per transform
block cost estimation of the intra prediction mode in the non-RD
mode decision.

Change-Id: I14ff92065e193e3e731c2bbf7ec89db676f1e132

diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index e33d52b0c73d6792d29cf9b0633b5e9e1d129015..f24fcf8580df12137c40320a37b8cb468887f721 100644 (file)
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -566,26 +566,48 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
   // threshold.
   if (!x->skip && best_rd > inter_mode_thresh &&
       bsize <= cpi->sf.max_intra_bsize) {
+    int i, j;
+    int step   = 1 << mbmi->tx_size;
+    int width  = num_4x4_blocks_wide_lookup[bsize];
+    int height = num_4x4_blocks_high_lookup[bsize];
+
+    int rate2 = 0;
+    int64_t dist2 = 0;
+    int dst_stride = pd->dst.stride;
+    int src_stride = p->src.stride;
+    int block_idx = 0;
+
     for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) {
       if (cpi->sf.reuse_inter_pred_sby) {
         pd->dst.buf = tmp[0].data;
         pd->dst.stride = bw;
       }
 
-      vp9_predict_intra_block(xd, 0, b_width_log2(bsize),
-                              mbmi->tx_size, this_mode,
-                              &p->src.buf[0], p->src.stride,
-                              &pd->dst.buf[0], pd->dst.stride, 0, 0, 0);
-
-      model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist, &var_y, &sse_y);
+      for (j = 0; j < height; j += step) {
+        for (i = 0; i < width; i += step) {
+          vp9_predict_intra_block(xd, block_idx, b_width_log2(bsize),
+                                  mbmi->tx_size, this_mode,
+                                  &p->src.buf[4 * (j * dst_stride + i)],
+                                  src_stride,
+                                  &pd->dst.buf[4 * (j * dst_stride + i)],
+                                  dst_stride, i, j, 0);
+          model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist, &var_y, &sse_y);
+          rate2 += rate;
+          dist2 += dist;
+          ++block_idx;
+        }
+      }
 
-      if (cpi->sf.reuse_inter_pred_sby)
-        pd->dst = orig_dst;
+      rate = rate2;
+      dist = dist2;
 
       rate += cpi->mbmode_cost[this_mode];
       rate += intra_cost_penalty;
       this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
 
+      if (cpi->sf.reuse_inter_pred_sby)
+        pd->dst = orig_dst;
+
       if (this_rd + intra_mode_cost < best_rd) {
         best_rd = this_rd;
         *returnrate = rate;

diff --git a/vp9/encoder/vp9_speed_features.c b/vp9/encoder/vp9_speed_features.c
index d7017f269014fa0851b26c4af8ef7991f1caad56..1a14da3c8c5f7033f1a2535a3d3bddc281bb52a5 100644 (file)
--- a/vp9/encoder/vp9_speed_features.c
+++ b/vp9/encoder/vp9_speed_features.c
@@ -273,9 +273,6 @@ static void set_rt_speed_feature(VP9_COMP *cpi, SPEED_FEATURES *sf,
     sf->source_var_thresh = 360;
 
     sf->tx_size_search_method = USE_TX_8X8;
-    // TODO(yunqingwang): max_intra_bsize is used to decide if DC_PRED mode
-    // is checked for a partition block. Later, we can try to allow large
-    // partitions to do intra mode checking.
     sf->max_intra_bsize = BLOCK_8X8;
 
     // This feature is only enabled when partition search is disabled.