Fix segfault with --cpu-used >= 3 and ext-refs.

With ext-ref enabled, it is possible that when trying to encode the
first true ALTREF frame after a keyframe, the previous ALTREF frame
(alias for the keyframe) is the same as one of the new LAST{2,3,4}
reference frames, and hence cpi->ref_frame_flags will have the ALTREF
bit clear, as computed by get_ref_frame_flags in encoder.c.

sf->alt_ref_search_fp forces the previous ALTREF frame to
be used as the only possible  reference when encoding a new ALTREF
frame, but due to cpi->ref_frame_flags, some buffers will not be
initialized (see rdopt.c:7689 yv12_mb), leading to a segfault.

get_ref_frame_flags in encoder.c has been changed to prefer to keep
the  LAST frame, then the ALTREF frame, then any of the LAST{2,3,4}
frames and then the GOLDEN frame in that order of preference in case
any of them are the same. This avoids the segfault and behaves the
same for the baseline.

Change-Id: I4da1991667614009da5d3061a6316c0d5dbc6c0c

diff --git a/vp10/encoder/encoder.c b/vp10/encoder/encoder.c
index b34b15eb9467ac5b52dd6e7c5d833f2d25697b2d..2cff98ce5a78056b41eebf2d8845a42ef5e5b988 100644 (file)
--- a/vp10/encoder/encoder.c
+++ b/vp10/encoder/encoder.c
@@ -4098,14 +4098,14 @@ static int get_ref_frame_flags(const VP10_COMP *cpi) {
   const int last2_is_last =
       map[cpi->lst_fb_idxes[1]] == map[cpi->lst_fb_idxes[0]];
   const int gld_is_last2 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[1]];
-  const int alt_is_last2 = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[1]];
+  const int last2_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[1]];
 
   const int last3_is_last =
       map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[0]];
   const int last3_is_last2 =
       map[cpi->lst_fb_idxes[2]] == map[cpi->lst_fb_idxes[1]];
   const int gld_is_last3 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[2]];
-  const int alt_is_last3 = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[2]];
+  const int last3_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[2]];
 
   const int last4_is_last =
       map[cpi->lst_fb_idxes[3]] == map[cpi->lst_fb_idxes[0]];
@@ -4114,21 +4114,16 @@ static int get_ref_frame_flags(const VP10_COMP *cpi) {
   const int last4_is_last3 =
       map[cpi->lst_fb_idxes[3]] == map[cpi->lst_fb_idxes[2]];
   const int gld_is_last4 = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idxes[3]];
-  const int alt_is_last4 = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[3]];
+  const int last4_is_alt = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idxes[3]];
 #else
   const int gld_is_last = map[cpi->gld_fb_idx] == map[cpi->lst_fb_idx];
   const int alt_is_last = map[cpi->alt_fb_idx] == map[cpi->lst_fb_idx];
 #endif  // CONFIG_EXT_REFS
   const int gld_is_alt = map[cpi->gld_fb_idx] == map[cpi->alt_fb_idx];
 
-  int flags = VP9_ALT_FLAG | VP9_GOLD_FLAG | VP9_LAST_FLAG;
-#if CONFIG_EXT_REFS
-  flags |= VP9_LAST2_FLAG;
-  flags |= VP9_LAST3_FLAG;
-  flags |= VP9_LAST4_FLAG;
-#endif  // CONFIG_EXT_REFS
+  int flags = VP9_REFFRAME_ALL;
 
-  if (gld_is_last)
+  if (gld_is_alt || gld_is_last)
     flags &= ~VP9_GOLD_FLAG;
 
   if (cpi->rc.frames_till_gf_update_due == INT_MAX)
@@ -4137,24 +4132,18 @@ static int get_ref_frame_flags(const VP10_COMP *cpi) {
   if (alt_is_last)
     flags &= ~VP9_ALT_FLAG;
 
-  if (gld_is_alt)
-    flags &= ~VP9_ALT_FLAG;
-
 #if CONFIG_EXT_REFS
-  if (last4_is_last || last4_is_last2 || last4_is_last3)
+  if (last4_is_alt || last4_is_last || last4_is_last2 || last4_is_last3)
     flags &= ~VP9_LAST4_FLAG;
 
-  if (last3_is_last || last3_is_last2)
+  if (last3_is_alt || last3_is_last || last3_is_last2)
     flags &= ~VP9_LAST3_FLAG;
 
-  if (last2_is_last)
+  if (last2_is_alt || last2_is_last)
     flags &= ~VP9_LAST2_FLAG;
 
   if (gld_is_last4 || gld_is_last3 || gld_is_last2)
     flags &= ~VP9_GOLD_FLAG;
-
-  if (alt_is_last4 || alt_is_last3 || alt_is_last2)
-    flags &= ~VP9_ALT_FLAG;
 #endif  // CONFIG_EXT_REFS
 
   return flags;

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index 0fde965b5beb206a0963c5539e882e2d9b058c34..9deb0f4e550d584c70f4448cba66730e070d0554 100644 (file)
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -7785,6 +7785,7 @@ void vp10_rd_pick_inter_mode_sb(VP10_COMP *cpi,
 
   if (cpi->rc.is_src_frame_alt_ref) {
     if (sf->alt_ref_search_fp) {
+      assert(cpi->ref_frame_flags & flag_list[ALTREF_FRAME]);
       mode_skip_mask[ALTREF_FRAME] = 0;
       ref_frame_skip_mask[0] = ~(1 << ALTREF_FRAME);
       ref_frame_skip_mask[1] = SECOND_REF_FRAME_MASK;