Add feature score for each block

The feature score is used to indicate whether a block's mv is reliable
or not.
Now we use Harris Corner Detector method to compute the score.

Change-Id: Ibbe7a1c1f3391d0bf4b03307eaabb5cc3cfb1360

diff --git a/tools/non_greedy_mv/non_greedy_mv.py b/tools/non_greedy_mv/non_greedy_mv.py
index 87b46e065fe53e25f8fa501b8dc73c7f43fb0dbe..c49c789f38f81c027adace93561ea4f969df3ba7 100644 (file)
--- a/tools/non_greedy_mv/non_greedy_mv.py
+++ b/tools/non_greedy_mv/non_greedy_mv.py
@@ -84,6 +84,14 @@ def yuv_to_rgb(yuv):
   return rgb / 255.
 
 
+def read_feature_score(fp, mv_rows, mv_cols):
+  line = fp.readline()
+  word_ls = line.split()
+  feature_score = np.array([float(v) for v in word_ls])
+  feature_score = feature_score.reshape(mv_rows, mv_cols)
+  return feature_score
+
+
 def read_frame_dpl_stats(fp):
   line = fp.readline()
   word_ls = line.split()
@@ -105,12 +113,13 @@ def read_frame_dpl_stats(fp):
     mv_ls.append([col, row, mv_col, mv_row])
   mv_ls = np.array(mv_ls)
   img = yuv_to_rgb(read_frame(fp))
+  feature_score = read_feature_score(fp, mv_rows, mv_cols)
   ref = None
   line = fp.readline()
   word_ls = line.split()
   if int(word_ls[1]):
     ref = yuv_to_rgb(read_frame(fp))
-  return frame_idx, mv_ls, img, ref, bs
+  return frame_idx, mv_ls, img, ref, bs, feature_score
 
 
 def read_dpl_stats_file(filename, frame_num=0):
@@ -131,8 +140,8 @@ def read_dpl_stats_file(filename, frame_num=0):
 if __name__ == '__main__':
   filename = sys.argv[1]
   data_ls = read_dpl_stats_file(filename, frame_num=5)
-  for frame_idx, mv_ls, img, ref, bs in data_ls:
-    fig, axes = plt.subplots(1, 2)
+  for frame_idx, mv_ls, img, ref, bs, feature_score in data_ls:
+    fig, axes = plt.subplots(1, 3)
 
     axes[0].imshow(img)
     draw_mv_ls(axes[0], mv_ls)
@@ -149,5 +158,7 @@ if __name__ == '__main__':
       axes[1].set_ylim(ref.shape[0], 0)
       axes[1].set_xlim(0, ref.shape[1])
 
+    axes[2].imshow(feature_score)
+
     plt.show()
     print frame_idx, len(mv_ls)

diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index 4b3f4162742406c0b47057c433cbc4509475781c..1355af984f7cf9727034e4f35bb4a89d662fe072 100644 (file)
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -5730,6 +5730,7 @@ void tpl_model_store(TplDepStats *tpl_stats, int mi_row, int mi_col,
         tpl_ptr->sse_arr[rf_idx] = src_stats->sse_arr[rf_idx];
         tpl_ptr->mv_arr[rf_idx].as_int = src_stats->mv_arr[rf_idx].as_int;
       }
+      tpl_ptr->feature_score = src_stats->feature_score;
 #endif
       tpl_ptr->intra_cost = intra_cost;
       tpl_ptr->inter_cost = inter_cost;
@@ -5845,6 +5846,31 @@ void wht_fwd_txfm(int16_t *src_diff, int bw, tran_low_t *coeff,
   }
 }
 
+#if CONFIG_NON_GREEDY_MV
+double get_feature_score(uint8_t *buf, ptrdiff_t stride, int rows, int cols) {
+  double IxIx = 0;
+  double IxIy = 0;
+  double IyIy = 0;
+  double score;
+  int r, c;
+  vpx_clear_system_state();
+  for (r = 0; r + 1 < rows; ++r) {
+    for (c = 0; c + 1 < cols; ++c) {
+      int diff_x = buf[r * stride + c] - buf[r * stride + c + 1];
+      int diff_y = buf[r * stride + c] - buf[(r + 1) * stride + c];
+      IxIx += diff_x * diff_x;
+      IxIy += diff_x * diff_y;
+      IyIy += diff_y * diff_y;
+    }
+  }
+  IxIx /= (rows - 1) * (cols - 1);
+  IxIy /= (rows - 1) * (cols - 1);
+  IyIy /= (rows - 1) * (cols - 1);
+  score = IxIx * IyIy - IxIy * IxIy - 0.04 * (IxIx + IyIy) * (IxIx + IyIy);
+  return score;
+}
+#endif
+
 void mode_estimation(VP9_COMP *cpi, MACROBLOCK *x, MACROBLOCKD *xd,
                      struct scale_factors *sf, GF_PICTURE *gf_picture,
                      int frame_idx, int16_t *src_diff, tran_low_t *coeff,
@@ -5916,6 +5942,11 @@ void mode_estimation(VP9_COMP *cpi, MACROBLOCK *x, MACROBLOCKD *xd,
   x->mv_limits.col_max =
       ((cm->mi_cols - 1 - mi_col) * MI_SIZE) + (17 - 2 * VP9_INTERP_EXTEND);
 
+#if CONFIG_NON_GREEDY_MV
+  tpl_stats->feature_score = get_feature_score(
+      xd->cur_buf->y_buffer + mb_y_offset, xd->cur_buf->y_stride, bw, bh);
+#endif
+
   for (rf_idx = 0; rf_idx < 3; ++rf_idx) {
     int_mv mv;
     if (ref_frame[rf_idx] == NULL) {
@@ -6134,6 +6165,17 @@ static void dump_tpl_stats(const VP9_COMP *cpi, int tpl_group_frames,
 
     dump_frame_buf(gf_picture[frame_idx].frame);
 
+    for (mi_row = 0; mi_row < cm->mi_rows; ++mi_row) {
+      for (mi_col = 0; mi_col < cm->mi_cols; ++mi_col) {
+        if ((mi_row % mi_height) == 0 && (mi_col % mi_width) == 0) {
+          const TplDepStats *tpl_ptr =
+              &tpl_frame->tpl_stats_ptr[mi_row * tpl_frame->stride + mi_col];
+          printf("%f ", tpl_ptr->feature_score);
+        }
+      }
+    }
+    printf("\n");
+
     rf_idx = gf_picture[frame_idx].ref_frame[idx];
     printf("has_ref %d\n", rf_idx != -1);
     if (rf_idx != -1) {

diff --git a/vp9/encoder/vp9_encoder.h b/vp9/encoder/vp9_encoder.h
index 365dde743bbfd31d840159d9bbc2b24416ea4021..5335937ed99bfa40b4ddeedaa933861fedcde329 100644 (file)
--- a/vp9/encoder/vp9_encoder.h
+++ b/vp9/encoder/vp9_encoder.h
@@ -295,6 +295,7 @@ typedef struct TplDepStats {
   int64_t recon_error_arr[3];
   int64_t sse_arr[3];
   int_mv mv_arr[3];
+  double feature_score;
 #endif
 } TplDepStats;