From: John Stebbins <jstebbins.hb@gmail.com> Date: Tue, 12 Mar 2019 15:06:24 +0000 (-0600) Subject: encavcodec: fix bitrate ceiling overflow for VP8/9 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=b2da61acec2b987a9f58559f8afef606ebda3ff5;p=handbrake encavcodec: fix bitrate ceiling overflow for VP8/9 When using constant quality encoding with VP8/9, we set a bitrate ceiling to prevent bitrate spikes. The calculation of this ceiling was duplicated at some point and the second copy was not transcribed properly and resulted in integer overflow. Fixes https://github.com/HandBrake/HandBrake/issues/1966 --- diff --git a/libhb/encavcodec.c b/libhb/encavcodec.c index 3a163a2c3..d0bafe610 100644 --- a/libhb/encavcodec.c +++ b/libhb/encavcodec.c @@ -105,7 +105,6 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job ) AVCodec * codec = NULL; AVCodecContext * context; AVRational fps; - int64_t bit_rate_ceiling = -1; hb_work_private_t * pv = calloc( 1, sizeof( hb_work_private_t ) ); w->private_data = pv; @@ -268,20 +267,6 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job ) lavc_opts = hb_encopts_to_dict(job->encoder_options, job->vcodec); } - bit_rate_ceiling = (int64_t)job->width * (int64_t)job->height * (int64_t)fps.num / (int64_t)fps.den; - - if (job->vquality != HB_INVALID_VIDEO_QUALITY) - { - if ( w->codec_param == AV_CODEC_ID_VP8 || - w->codec_param == AV_CODEC_ID_VP9 ) - { - //This value was chosen to make the bitrate high enough - //for libvpx to "turn off" the maximum bitrate feature - //that is normally applied to constant quality. - context->bit_rate = bit_rate_ceiling; - } - } - AVDictionary * av_opts = NULL; if (apply_encoder_preset(job->vcodec, &av_opts, job->encoder_preset)) { @@ -342,7 +327,8 @@ int encavcodecInit( hb_work_object_t * w, hb_job_t * job ) //This value was chosen to make the bitrate high enough //for libvpx to "turn off" the maximum bitrate feature //that is normally applied to constant quality. - context->bit_rate = job->width * job->height * fps.num / fps.den; + context->bit_rate = (int64_t)job->width * job->height * + fps.num / fps.den; hb_log( "encavcodec: encoding at CQ %.2f", job->vquality ); } //Set constant quality for nvenc