From: Geza Lore <gezalore@gmail.com>
Date: Thu, 14 Apr 2016 13:26:03 +0000 (+0100)
Subject: Fix segfault with --cpu-used >= 3 and ext-refs.
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=77d197e635f52d0eaf4a9f2d6aecfd38b065265a;p=libvpx

Fix segfault with --cpu-used >= 3 and ext-refs.

With ext-ref enabled, it is possible that when trying to encode the
first true ALTREF frame after a keyframe, the previous ALTREF frame
(alias for the keyframe) is the same as one of the new LAST{2,3,4}
reference frames, and hence cpi->ref_frame_flags will have the ALTREF
bit clear, as computed by get_ref_frame_flags in encoder.c.

sf->alt_ref_search_fp forces the previous ALTREF frame to
be used as the only possible  reference when encoding a new ALTREF
frame, but due to cpi->ref_frame_flags, some buffers will not be
initialized (see rdopt.c:7689 yv12_mb), leading to a segfault.

get_ref_frame_flags in encoder.c has been changed to prefer to keep
the  LAST frame, then the ALTREF frame, then any of the LAST{2,3,4}
frames and then the GOLDEN frame in that order of preference in case
any of them are the same. This avoids the segfault and behaves the
same for the baseline.

Change-Id: I4da1991667614009da5d3061a6316c0d5dbc6c0c
---

diff --git a/vp10/encoder/encoder.c b/vp10/encoder/encoder.c
index b34b15eb9..2cff98ce5 100644
--- a/vp10/encoder/encoder.c
+++ b/vp10/encoder/encoder.c
@@ -4098,14 +4098,14 @@ static int get_ref_frame_flags(const VP10_COMP *cpi) {
   const int last2_is_last =
       map[cpi->lst_fb_idxes[1]] == map[cpi->lst_fb_idxes[0]];
   const int gld_is_last2 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[1]];
-  const int alt_is_last2 = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[1]];
+  const int last2_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[1]];
 
   const int last3_is_last =
       map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[0]];
   const int last3_is_last2 =
       map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[1]];
   const int gld_is_last3 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[2]];
-  const int alt_is_last3 = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[2]];
+  const int last3_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[2]];
 
   const int last4_is_last =
       map[cpi->lst_fb_idxes[3]] == map[cpi->lst_fb_idxes[0]];
@@ -4114,21 +4114,16 @@ static int get_ref_frame_flags(const VP10_COMP *cpi) {
   const int last4_is_last3 =
       map[cpi->lst_fb_idxes[3]] == map[cpi->lst_fb_idxes[2]];
   const int gld_is_last4 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[3]];
-  const int alt_is_last4 = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[3]];
+  const int last4_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[3]];
 #else
   const int gld_is_last = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idx];
   const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idx];
 #endif  // CONFIG_EXT_REFS
   const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx];
 
-  int flags = VP9_ALT_FLAG | VP9_GOLD_FLAG | VP9_LAST_FLAG;
-#if CONFIG_EXT_REFS
-  flags |= VP9_LAST2_FLAG;
-  flags |= VP9_LAST3_FLAG;
-  flags |= VP9_LAST4_FLAG;
-#endif  // CONFIG_EXT_REFS
+  int flags = VP9_REFFRAME_ALL;
 
-  if (gld_is_last)
+  if (gld_is_alt || gld_is_last)
     flags &= ~VP9_GOLD_FLAG;
 
   if (cpi->rc.frames_till_gf_update_due == INT_MAX)
@@ -4137,24 +4132,18 @@ static int get_ref_frame_flags(const VP10_COMP *cpi) {
   if (alt_is_last)
     flags &= ~VP9_ALT_FLAG;
 
-  if (gld_is_alt)
-    flags &= ~VP9_ALT_FLAG;
-
 #if CONFIG_EXT_REFS
-  if (last4_is_last || last4_is_last2 || last4_is_last3)
+  if (last4_is_alt || last4_is_last || last4_is_last2 || last4_is_last3)
     flags &= ~VP9_LAST4_FLAG;
 
-  if (last3_is_last || last3_is_last2)
+  if (last3_is_alt || last3_is_last || last3_is_last2)
     flags &= ~VP9_LAST3_FLAG;
 
-  if (last2_is_last)
+  if (last2_is_alt || last2_is_last)
     flags &= ~VP9_LAST2_FLAG;
 
   if (gld_is_last4 || gld_is_last3 || gld_is_last2)
     flags &= ~VP9_GOLD_FLAG;
-
-  if (alt_is_last4 || alt_is_last3 || alt_is_last2)
-    flags &= ~VP9_ALT_FLAG;
 #endif  // CONFIG_EXT_REFS
 
   return flags;
diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 0fde965b5..9deb0f4e5 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -7785,6 +7785,7 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
 
   if (cpi->rc.is_src_frame_alt_ref) {
     if (sf->alt_ref_search_fp) {
+      assert(cpi->ref_frame_flags & flag_list[ALTREF_FRAME]);
       mode_skip_mask[ALTREF_FRAME] = 0;
       ref_frame_skip_mask[0] = ~(1 << ALTREF_FRAME);
       ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK;