From: Steven Walters <kemuri9@gmail.com>
Date: Sun, 8 Nov 2009 19:53:48 +0000 (-0800)
Subject: Fix assert failure in the case of forced i-frames
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=b06734129a221be0c7a9a66c91ad042338abcd7c;p=libx264

Fix assert failure in the case of forced i-frames
Note that this applies to non-IDR i-frames, not IDR-frames.
This fix is also required for future open-gop.
---

diff --git a/encoder/slicetype.c b/encoder/slicetype.c
index 00f89c3f..c1040a6d 100644
--- a/encoder/slicetype.c
+++ b/encoder/slicetype.c
@@ -996,25 +996,27 @@ void x264_slicetype_decide( x264_t *h )
     if( h->param.rc.i_rc_method != X264_RC_CQP )
     {
         x264_mb_analysis_t a;
-        int p0=0, p1, b;
+        int p0, p1, b;
+        p1 = b = bframes + 1;
 
         x264_lowres_context_init( h, &a );
 
         frames[0] = h->lookahead->last_nonb;
         memcpy( &frames[1], h->lookahead->next.list, (bframes+1) * sizeof(x264_frame_t*) );
         if( IS_X264_TYPE_I( h->lookahead->next.list[bframes]->i_type ) )
-            p0 = p1 = b = 1;
+            p0 = bframes + 1;
         else // P
-            p1 = b = bframes + 1;
+            p0 = 0;
 
         x264_slicetype_frame_cost( h, &a, frames, p0, p1, b, 0 );
 
-        if( p0 != p1 && h->param.rc.i_vbv_buffer_size )
+        if( (p0 != p1 || bframes) && h->param.rc.i_vbv_buffer_size )
         {
             /* We need the intra costs for row SATDs. */
             x264_slicetype_frame_cost( h, &a, frames, b, b, b, 0 );
 
             /* We need B-frame costs for row SATDs. */
+            p0 = 0;
             for( b = 1; b <= bframes; b++ )
             {
                 if( frames[b]->i_type == X264_TYPE_B )