Fix range checks in motion search

There were some situations that the start motion vectors were
out of range. This fix adjusted range checks to make sure they
are checked and clamped.

Change-Id: Ife83b7fed0882bba6d1fa559b6e63c054fd5065d

diff --git a/vp8/encoder/mcomp.c b/vp8/encoder/mcomp.c
index 58b524f82b4f1ce50b07fc9f8946656d66434e81..282e5c91bdbd6ee44d389a1e2da802cb95daff7d 100644 (file)
--- a/vp8/encoder/mcomp.c
+++ b/vp8/encoder/mcomp.c
@@ -15,6 +15,7 @@
 #include <stdio.h>
 #include <limits.h>
 #include <math.h>
+#include "vp8/common/findnearmv.h"
 
 #ifdef ENTROPY_STATS
 static int mv_ref_ct [31] [4] [2];
@@ -866,7 +867,7 @@ int vp8_hex_search
     unsigned char *what = (*(b->base_src) + b->src);
     int what_stride = b->src_stride;
     int in_what_stride = d->pre_stride;
-    int br = ref_mv->as_mv.row, bc = ref_mv->as_mv.col;
+    int br, bc;
     int_mv this_mv;
     unsigned int bestsad = 0x7fffffff;
     unsigned int thissad;
@@ -880,6 +881,11 @@ int vp8_hex_search
     fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3;
     fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3;
 
+    // adjust ref_mv to make sure it is within MV range
+    vp8_clamp_mv(ref_mv, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max);
+    br = ref_mv->as_mv.row;
+    bc = ref_mv->as_mv.col;
+
     // Work out the start point for the search
     base_offset = (unsigned char *)(*(d->base_pre) + d->pre);
     this_offset = base_offset + (br * (d->pre_stride)) + bc;
@@ -1043,8 +1049,8 @@ int vp8_diamond_search_sad
     int best_site = 0;
     int last_site = 0;
 
-    int ref_row = ref_mv->as_mv.row;
-    int ref_col = ref_mv->as_mv.col;
+    int ref_row;
+    int ref_col;
     int this_row_offset;
     int this_col_offset;
     search_site *ss;
@@ -1057,8 +1063,10 @@ int vp8_diamond_search_sad
     fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3;
     fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3;
 
+    vp8_clamp_mv(ref_mv, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max);
+    ref_row = ref_mv->as_mv.row;
+    ref_col = ref_mv->as_mv.col;
     *num00 = 0;
-
     best_mv->as_mv.row = ref_row;
     best_mv->as_mv.col = ref_col;
 
@@ -1162,8 +1170,8 @@ int vp8_diamond_search_sadx4
     int best_site = 0;
     int last_site = 0;
 
-    int ref_row = ref_mv->as_mv.row;
-    int ref_col = ref_mv->as_mv.col;
+    int ref_row;
+    int ref_col;
     int this_row_offset;
     int this_col_offset;
     search_site *ss;
@@ -1176,6 +1184,9 @@ int vp8_diamond_search_sadx4
     fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3;
     fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3;
 
+    vp8_clamp_mv(ref_mv, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max);
+    ref_row = ref_mv->as_mv.row;
+    ref_col = ref_mv->as_mv.col;
     *num00 = 0;
     best_mv->as_mv.row = ref_row;
     best_mv->as_mv.col = ref_col;

diff --git a/vp8/encoder/pickinter.c b/vp8/encoder/pickinter.c
index 725e44e6209145ff1eca3c7ef3a991879fbb520f..9cdaf7d53315124931b31297414a58be8f89c646 100644 (file)
--- a/vp8/encoder/pickinter.c
+++ b/vp8/encoder/pickinter.c
@@ -669,8 +669,6 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset,
                 mvp_full.as_mv.col = mvp.as_mv.col>>3;
                 mvp_full.as_mv.row = mvp.as_mv.row>>3;
 
-                /* adjust mvp to make sure it is within MV range */
-                vp8_clamp_mv(&mvp_full, col_min, col_max, row_min, row_max);
             }else
             {
                 mvp.as_int = best_ref_mv.as_int;

diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c
index 8b18541615752ab0c7c690b42427202a8983d1de..356c03daccd4ffb859768ef5e21bc1e551639624 100644 (file)
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -2016,9 +2016,6 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
             mvp_full.as_mv.col = mvp.as_mv.col>>3;
             mvp_full.as_mv.row = mvp.as_mv.row>>3;
 
-            /* adjust mvp to make sure it is within MV range */
-            vp8_clamp_mv(&mvp_full, col_min, col_max, row_min, row_max);
-
             // Get intersection of UMV window and valid MV window to reduce # of checks in diamond search.
             if (x->mv_col_min < col_min )
                 x->mv_col_min = col_min;