From b9577e07fc2649ab1ec1e58d7d8c4450f45d1f40 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 30 Jun 2017 08:51:31 -0700 Subject: [PATCH] vp8: Drop due to overshoot for non-screen content. For 1 pass CBR mode: Apply the logic for dropping (and re-adjusting rate control) due to large overshoot to the case of non-screen content when drop_frames_allowed is enabled. For the non-screen content case: add additional condition that rate correction factor is close to minimum state, and flag to constrain the frequency of the dropping. Also handle the case of temporal layers and multi-res encoding. Add some flags/counters to the layer context for temporal layers. For multi-res: drop due to overshoot is checked on lowest stream, and if overshoot is detected we force drops on all upper streams for that frame. This feature is to avoid large frame sizes on big content changes following low content period. No change in behavior for screen_content_mode = 2. Change-Id: I797ab236cbbf3b15cad439e9a227fbebced632e6 --- vp8/common/blockd.h | 5 +++ vp8/encoder/onyx_if.c | 10 ++++-- vp8/encoder/onyx_int.h | 5 +++ vp8/encoder/ratectrl.c | 73 ++++++++++++++++++++++++++++++++++++------ 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/vp8/common/blockd.h b/vp8/common/blockd.h index 74fc5d6db..1a3aad16a 100644 --- a/vp8/common/blockd.h +++ b/vp8/common/blockd.h @@ -169,6 +169,11 @@ typedef struct { typedef struct { FRAME_TYPE frame_type; int is_frame_dropped; + // If frame is dropped due to overshoot after encode_frame. This triggers a + // drop and resets rate control with Q forced to max for following frame. + // The check for this dropping due to overshoot is only done on lowest stream, + // and if set will force drop on all spatial streams for that current frame. + int is_frame_dropped_overshoot_maxqp; // The frame rate for the lowest resolution. double low_res_framerate; /* The frame number of each reference frames */ diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c index 2b7f6795b..042d33ed0 100644 --- a/vp8/encoder/onyx_if.c +++ b/vp8/encoder/onyx_if.c @@ -220,7 +220,8 @@ static void save_layer_context(VP8_COMP *cpi) { lc->inter_frame_target = cpi->inter_frame_target; lc->total_byte_count = cpi->total_byte_count; lc->filter_level = cpi->common.filter_level; - + lc->frames_since_last_drop_overshoot = cpi->frames_since_last_drop_overshoot; + lc->force_maxqp = cpi->force_maxqp; lc->last_frame_percent_intra = cpi->last_frame_percent_intra; memcpy(lc->count_mb_ref_frame_usage, cpi->mb.count_mb_ref_frame_usage, @@ -256,7 +257,8 @@ static void restore_layer_context(VP8_COMP *cpi, const int layer) { cpi->inter_frame_target = lc->inter_frame_target; cpi->total_byte_count = lc->total_byte_count; cpi->common.filter_level = lc->filter_level; - + cpi->frames_since_last_drop_overshoot = lc->frames_since_last_drop_overshoot; + cpi->force_maxqp = lc->force_maxqp; cpi->last_frame_percent_intra = lc->last_frame_percent_intra; memcpy(cpi->mb.count_mb_ref_frame_usage, lc->count_mb_ref_frame_usage, @@ -1937,6 +1939,7 @@ struct VP8_COMP *vp8_create_compressor(VP8_CONFIG *oxcf) { cpi->common.refresh_alt_ref_frame = 0; cpi->force_maxqp = 0; + cpi->frames_since_last_drop_overshoot = 0; cpi->b_calculate_psnr = CONFIG_INTERNAL_STATS; #if CONFIG_INTERNAL_STATS @@ -4027,7 +4030,8 @@ static void encode_frame_to_data_rate(VP8_COMP *cpi, size_t *size, #else /* transform / motion compensation build reconstruction frame */ vp8_encode_frame(cpi); - if (cpi->oxcf.screen_content_mode == 2) { + + if (cpi->pass == 0 && cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER) { if (vp8_drop_encodedframe_overshoot(cpi, Q)) return; } diff --git a/vp8/encoder/onyx_int.h b/vp8/encoder/onyx_int.h index 53e8be84f..08f07851e 100644 --- a/vp8/encoder/onyx_int.h +++ b/vp8/encoder/onyx_int.h @@ -249,6 +249,10 @@ typedef struct { int filter_level; + int frames_since_last_drop_overshoot; + + int force_maxqp; + int last_frame_percent_intra; int count_mb_ref_frame_usage[MAX_REF_FRAMES]; @@ -505,6 +509,7 @@ typedef struct VP8_COMP { int mse_source_denoised; int force_maxqp; + int frames_since_last_drop_overshoot; // GF update for 1 pass cbr. int gf_update_onepass_cbr; diff --git a/vp8/encoder/ratectrl.c b/vp8/encoder/ratectrl.c index 422eee031..70aef6bc8 100644 --- a/vp8/encoder/ratectrl.c +++ b/vp8/encoder/ratectrl.c @@ -1442,12 +1442,33 @@ int vp8_pick_frame_size(VP8_COMP *cpi) { // If this just encoded frame (mcomp/transform/quant, but before loopfilter and // pack_bitstream) has large overshoot, and was not being encoded close to the // max QP, then drop this frame and force next frame to be encoded at max QP. -// Condition this on 1 pass CBR with screen content mode and frame dropper off. +// Allow this for screen_content_mode = 2, or if drop frames is allowed. // TODO(marpan): Should do this exit condition during the encode_frame // (i.e., halfway during the encoding of the frame) to save cycles. int vp8_drop_encodedframe_overshoot(VP8_COMP *cpi, int Q) { - if (cpi->pass == 0 && cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER && - cpi->drop_frames_allowed == 0 && cpi->common.frame_type != KEY_FRAME) { + int force_drop_overshoot = 0; +#if CONFIG_MULTI_RES_ENCODING + // Only check for dropping due to overshoot on the lowest stream. + // If the lowest stream of the multi-res encoding was dropped due to + // overshoot, then force dropping on all upper layer streams + // (mr_encoder_id > 0). + LOWER_RES_FRAME_INFO *low_res_frame_info = + (LOWER_RES_FRAME_INFO *)cpi->oxcf.mr_low_res_mode_info; + if (cpi->oxcf.mr_total_resolutions > 1 && cpi->oxcf.mr_encoder_id > 0) { + force_drop_overshoot = low_res_frame_info->is_frame_dropped_overshoot_maxqp; + if (!force_drop_overshoot) { + cpi->force_maxqp = 0; + cpi->frames_since_last_drop_overshoot++; + return 0; + } + } +#endif + if (cpi->common.frame_type != KEY_FRAME && + (cpi->oxcf.screen_content_mode == 2 || + (cpi->drop_frames_allowed && + (force_drop_overshoot || + (cpi->rate_correction_factor < (4.0f * MIN_BPB_FACTOR) && + cpi->frames_since_last_drop_overshoot > (int)cpi->framerate))))) { // Note: the "projected_frame_size" from encode_frame() only gives estimate // of mode/motion vector rate (in non-rd mode): so below we only require // that projected_frame_size is somewhat greater than per-frame-bandwidth, @@ -1458,17 +1479,20 @@ int vp8_drop_encodedframe_overshoot(VP8_COMP *cpi, int Q) { // Rate threshold, in bytes. int thresh_rate = 2 * (cpi->av_per_frame_bandwidth >> 3); // Threshold for the average (over all macroblocks) of the pixel-sum - // residual error over 16x16 block. Should add QP dependence on threshold? + // residual error over 16x16 block. int thresh_pred_err_mb = (200 << 4); int pred_err_mb = (int)(cpi->mb.prediction_error / cpi->common.MBs); - if (Q < thresh_qp && cpi->projected_frame_size > thresh_rate && - pred_err_mb > thresh_pred_err_mb) { + // Reduce/ignore thresh_rate if pred_err_mb much larger than its threshold, + // give more weight to pred_err metric for overshoot detection. + if (cpi->drop_frames_allowed && pred_err_mb > (thresh_pred_err_mb << 4)) + thresh_rate = thresh_rate >> 3; + if ((Q < thresh_qp && cpi->projected_frame_size > thresh_rate && + pred_err_mb > thresh_pred_err_mb) || + force_drop_overshoot) { + unsigned int i; double new_correction_factor; - const int target_size = cpi->av_per_frame_bandwidth; int target_bits_per_mb; - // Drop this frame: advance frame counters, and set force_maxqp flag. - cpi->common.current_video_frame++; - cpi->frames_since_key++; + const int target_size = cpi->av_per_frame_bandwidth; // Flag to indicate we will force next frame to be encoded at max QP. cpi->force_maxqp = 1; // Reset the buffer levels. @@ -1499,11 +1523,40 @@ int vp8_drop_encodedframe_overshoot(VP8_COMP *cpi, int Q) { if (cpi->rate_correction_factor > MAX_BPB_FACTOR) { cpi->rate_correction_factor = MAX_BPB_FACTOR; } + // Drop this frame: update frame counters. + cpi->common.current_video_frame++; + cpi->frames_since_key++; + cpi->temporal_pattern_counter++; + cpi->frames_since_last_drop_overshoot = 0; + if (cpi->oxcf.number_of_layers > 1) { + // Set max_qp and rate correction for all temporal layers if overshoot + // is detected. + for (i = 0; i < cpi->oxcf.number_of_layers; ++i) { + LAYER_CONTEXT *lc = &cpi->layer_context[i]; + lc->force_maxqp = 1; + lc->frames_since_last_drop_overshoot = 0; + lc->rate_correction_factor = cpi->rate_correction_factor; + } + } +#if CONFIG_MULTI_RES_ENCODING + if (cpi->oxcf.mr_total_resolutions > 1) + low_res_frame_info->is_frame_dropped_overshoot_maxqp = 1; +#endif return 1; } cpi->force_maxqp = 0; + cpi->frames_since_last_drop_overshoot++; +#if CONFIG_MULTI_RES_ENCODING + if (cpi->oxcf.mr_total_resolutions > 1) + low_res_frame_info->is_frame_dropped_overshoot_maxqp = 0; +#endif return 0; } cpi->force_maxqp = 0; + cpi->frames_since_last_drop_overshoot++; +#if CONFIG_MULTI_RES_ENCODING + if (cpi->oxcf.mr_total_resolutions > 1) + low_res_frame_info->is_frame_dropped_overshoot_maxqp = 0; +#endif return 0; } -- 2.40.0