From: Jingning Han <jingning@google.com>
Date: Wed, 25 Jun 2014 18:41:49 +0000 (-0700)
Subject: Make non-RD intra mode search txfm size dependent
X-Git-Tag: v1.4.0~1332^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=2aa50eafb2fb3bb10bdfeac9b7ccf749d192dc93;p=libvpx

Make non-RD intra mode search txfm size dependent

This commit fixes the potential issue in the non-RD mode decision
flow that only checks part of the block to estimate the cost. It
was due to the use of fixed transform size, in replacing the
largest transform block size. This commit enables per transform
block cost estimation of the intra prediction mode in the non-RD
mode decision.

Change-Id: I14ff92065e193e3e731c2bbf7ec89db676f1e132
---

diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c
index e33d52b0c..f24fcf858 100644
--- a/vp9/encoder/vp9_pickmode.c
+++ b/vp9/encoder/vp9_pickmode.c
@@ -566,26 +566,48 @@ int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
   // threshold.
   if (!x->skip && best_rd > inter_mode_thresh &&
       bsize <= cpi->sf.max_intra_bsize) {
+    int i, j;
+    int step   = 1 << mbmi->tx_size;
+    int width  = num_4x4_blocks_wide_lookup[bsize];
+    int height = num_4x4_blocks_high_lookup[bsize];
+
+    int rate2 = 0;
+    int64_t dist2 = 0;
+    int dst_stride = pd->dst.stride;
+    int src_stride = p->src.stride;
+    int block_idx = 0;
+
     for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) {
       if (cpi->sf.reuse_inter_pred_sby) {
         pd->dst.buf = tmp[0].data;
         pd->dst.stride = bw;
       }
 
-      vp9_predict_intra_block(xd, 0, b_width_log2(bsize),
-                              mbmi->tx_size, this_mode,
-                              &p->src.buf[0], p->src.stride,
-                              &pd->dst.buf[0], pd->dst.stride, 0, 0, 0);
-
-      model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist, &var_y, &sse_y);
+      for (j = 0; j < height; j += step) {
+        for (i = 0; i < width; i += step) {
+          vp9_predict_intra_block(xd, block_idx, b_width_log2(bsize),
+                                  mbmi->tx_size, this_mode,
+                                  &p->src.buf[4 * (j * dst_stride + i)],
+                                  src_stride,
+                                  &pd->dst.buf[4 * (j * dst_stride + i)],
+                                  dst_stride, i, j, 0);
+          model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist, &var_y, &sse_y);
+          rate2 += rate;
+          dist2 += dist;
+          ++block_idx;
+        }
+      }
 
-      if (cpi->sf.reuse_inter_pred_sby)
-        pd->dst = orig_dst;
+      rate = rate2;
+      dist = dist2;
 
       rate += cpi->mbmode_cost[this_mode];
       rate += intra_cost_penalty;
       this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
 
+      if (cpi->sf.reuse_inter_pred_sby)
+        pd->dst = orig_dst;
+
       if (this_rd + intra_mode_cost < best_rd) {
         best_rd = this_rd;
         *returnrate = rate;
diff --git a/vp9/encoder/vp9_speed_features.c b/vp9/encoder/vp9_speed_features.c
index d7017f269..1a14da3c8 100644
--- a/vp9/encoder/vp9_speed_features.c
+++ b/vp9/encoder/vp9_speed_features.c
@@ -273,9 +273,6 @@ static void set_rt_speed_feature(VP9_COMP *cpi, SPEED_FEATURES *sf,
     sf->source_var_thresh = 360;
 
     sf->tx_size_search_method = USE_TX_8X8;
-    // TODO(yunqingwang): max_intra_bsize is used to decide if DC_PRED mode
-    // is checked for a partition block. Later, we can try to allow large
-    // partitions to do intra mode checking.
     sf->max_intra_bsize = BLOCK_8X8;
 
     // This feature is only enabled when partition search is disabled.