decavcodec: fix avcodec_open failure upon bad extradata

If extradata is improperly parsed, avcodec_open may fail.  Keep trying
avcodec_open as long as there is more data to parse.

Fixes https://github.com/HandBrake/HandBrake/issues/542

(cherry picked from commit ca1dcccd09b5dfac93829d46fd6c7c9711898b77)

diff --git a/libhb/decavcodec.c b/libhb/decavcodec.c
index c8b12d7707be1a757d4f2a26aa0799c8f2338c3e..953fa2582906a86c2ab8e6348b0b3d82e848124f 100644 (file)
--- a/libhb/decavcodec.c
+++ b/libhb/decavcodec.c
@@ -1665,8 +1665,12 @@ static int decavcodecvWork( hb_work_object_t * w, hb_buffer_t ** buf_in,
         {
             av_dict_free( &av_opts );
             hb_log( "decavcodecvWork: avcodec_open failed" );
-            *buf_out = hb_buffer_eof_init();
-            return HB_WORK_DONE;
+            // avcodec_open can fail due to incorrectly parsed extradata
+            // so try again when this fails
+            av_freep( &pv->context->extradata );
+            pv->context->extradata_size = 0;
+            hb_buffer_close( &in );
+            return HB_WORK_OK;
         }
         av_dict_free( &av_opts );
         pv->video_codec_opened = 1;