From: Ronald S. Bultje <rbultje@google.com>
Date: Fri, 7 Jun 2013 18:40:42 +0000 (-0700)
Subject: Don't crash if motion vector ref points to out-of-bounds area.
X-Git-Tag: v1.3.0~1104^2~47
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=363dc6ceda3e2d2e11083eb29161b56b9b11057e;p=libvpx

Don't crash if motion vector ref points to out-of-bounds area.

This can only happen if partition is partly out-of-frame, in which
case the referenced mv is either out-of-frame also (and thus has the
same value as an already-read one), or it is actually uninitialized,
in which case we don't want to use it.

Change-Id: Icf39fa4d987c7abcbebb9bbdcdd6311e8fb9d3c9
---

diff --git a/vp9/common/vp9_mvref_common.c b/vp9/common/vp9_mvref_common.c
index 0a829d250..4878cbef6 100644
--- a/vp9/common/vp9_mvref_common.c
+++ b/vp9/common/vp9_mvref_common.c
@@ -161,42 +161,10 @@ void vp9_find_mv_refs_idx(VP9_COMMON *cm, MACROBLOCKD *xd, MODE_INFO *here,
   vpx_memset(mv_ref_list, 0, sizeof(int_mv) * MAX_MV_REF_CANDIDATES);
   vpx_memset(candidate_scores, 0, sizeof(candidate_scores));
 
-  if (xd->mb_to_right_edge < 0 || xd->mb_to_bottom_edge < 0) {
-    int pixels_wide = 4 * b_width_log2(mbmi->sb_type);
-    int pixels_high = 4 * b_height_log2(mbmi->sb_type);
-    int pixels_square = 0;
-
-    if (xd->mb_to_right_edge < 0)
-      pixels_wide += (xd->mb_to_right_edge >> 3);
-
-    if (xd->mb_to_bottom_edge < 0)
-      pixels_high += (xd->mb_to_bottom_edge >> 3);
-
-    if ( pixels_wide < pixels_high )
-      pixels_square = pixels_wide;
-    else
-      pixels_square = pixels_high;
-
-    if (pixels_square == 64) {
-      mv_ref_search = mv_ref_blocks[BLOCK_SIZE_SB64X64];
-    } else if (pixels_square == 32) {
-      mv_ref_search = mv_ref_blocks[BLOCK_SIZE_SB32X32];
-    } else if (pixels_square == 16) {
-      mv_ref_search = mv_ref_blocks[BLOCK_SIZE_MB16X16];
-    } else {
-      mv_ref_search = mv_ref_blocks[BLOCK_SIZE_SB8X8];
-      if (mbmi->sb_type < BLOCK_SIZE_SB8X8) {
-        x_idx = block_idx & 1;
-        y_idx = block_idx >> 1;
-      }
-    }
-  }
-  else {
   mv_ref_search = mv_ref_blocks[mbmi->sb_type];
-    if (mbmi->sb_type < BLOCK_SIZE_SB8X8) {
-      x_idx = block_idx & 1;
-      y_idx = block_idx >> 1;
-      }
+  if (mbmi->sb_type < BLOCK_SIZE_SB8X8) {
+    x_idx = block_idx & 1;
+    y_idx = block_idx >> 1;
   }
 
   // We first scan for candidate vectors that match the current reference frame
@@ -205,7 +173,8 @@ void vp9_find_mv_refs_idx(VP9_COMMON *cm, MACROBLOCKD *xd, MODE_INFO *here,
     const int mi_search_col = mi_col + mv_ref_search[i][0];
     if ((mi_search_col >= cm->cur_tile_mi_col_start) &&
         (mi_search_col < cm->cur_tile_mi_col_end) &&
-        ((mv_ref_search[i][1] << 6) >= xd->mb_to_top_edge)) {
+        ((mv_ref_search[i][1] << 6) >= xd->mb_to_top_edge) &&
+        ((-mv_ref_search[i][1] << 6) <= xd->mb_to_bottom_edge)) {
       int b;
 
       candidate_mi = here + mv_ref_search[i][0] +
@@ -240,7 +209,8 @@ void vp9_find_mv_refs_idx(VP9_COMMON *cm, MACROBLOCKD *xd, MODE_INFO *here,
 
     if ((mi_search_col >= cm->cur_tile_mi_col_start) &&
         (mi_search_col < cm->cur_tile_mi_col_end) &&
-        ((mv_ref_search[i][1] << 6) >= xd->mb_to_top_edge)) {
+        ((mv_ref_search[i][1] << 6) >= xd->mb_to_top_edge) &&
+        ((-mv_ref_search[i][1] << 6) <= xd->mb_to_bottom_edge)) {
       candidate_mi = here + mv_ref_search[i][0] +
                      (mv_ref_search[i][1] * xd->mode_info_stride);
 
@@ -270,7 +240,8 @@ void vp9_find_mv_refs_idx(VP9_COMMON *cm, MACROBLOCKD *xd, MODE_INFO *here,
 
     if ((mi_search_col >= cm->cur_tile_mi_col_start) &&
         (mi_search_col < cm->cur_tile_mi_col_end) &&
-        ((mv_ref_search[i][1] << 6) >= xd->mb_to_top_edge)) {
+        ((mv_ref_search[i][1] << 6) >= xd->mb_to_top_edge) &&
+        ((-mv_ref_search[i][1] << 6) <= xd->mb_to_bottom_edge)) {
       candidate_mi = here + mv_ref_search[i][0] +
                      (mv_ref_search[i][1] * xd->mode_info_stride);