From a03b04a55fded9dea0ebbb423381a93ea39b866e Mon Sep 17 00:00:00 2001 From: Jerome Jiang Date: Tue, 28 Aug 2018 14:24:53 -0700 Subject: [PATCH] vp9: postencode drop frame for screen content in CBR. Encode the next frame at max q. For layers: post_encode_drop is only check on base spatial layer, and if base is post-encoded-dropped, then whole superframe is dropped. Added API to guard postencode dropping. Turned off by default. Added unittest. BUG=b/112990050 Change-Id: I42fee279014aca616f7a4d9b582cb2bf5da2f2e7 --- test/svc_datarate_test.cc | 83 ++++++++++++++++++++++++++++++ test/vp9_datarate_test.cc | 50 +++++++++++++++++- vp9/encoder/vp9_encoder.c | 61 ++++++++++++++++------ vp9/encoder/vp9_ratectrl.c | 40 +++++++++++++- vp9/encoder/vp9_ratectrl.h | 11 ++++ vp9/encoder/vp9_svc_layercontext.c | 3 +- vp9/vp9_cx_iface.c | 9 ++++ vpx/vp8cx.h | 10 ++++ 8 files changed, 249 insertions(+), 18 deletions(-) diff --git a/test/svc_datarate_test.cc b/test/svc_datarate_test.cc index 117014fa6..c530beac6 100644 --- a/test/svc_datarate_test.cc +++ b/test/svc_datarate_test.cc @@ -60,6 +60,7 @@ class DatarateOnePassCbrSvc : public ::svc_test::OnePassCbrSvc { layer_sync_on_base_ = 0; force_intra_only_frame_ = 0; superframe_has_intra_only_ = 0; + use_post_encode_drop_ = 0; } virtual void BeginPassHook(unsigned int /*pass*/) {} @@ -174,6 +175,10 @@ class DatarateOnePassCbrSvc : public ::svc_test::OnePassCbrSvc { svc_drop_frame.max_consec_drop = 30; encoder->Control(VP9E_SET_SVC_FRAME_DROP_LAYER, &svc_drop_frame); } + + if (use_post_encode_drop_) { + encoder->Control(VP9E_SET_POSTENCODE_DROP, use_post_encode_drop_); + } } if (update_pattern_ && video->frame() >= 100) { @@ -482,6 +487,7 @@ class DatarateOnePassCbrSvc : public ::svc_test::OnePassCbrSvc { int layer_sync_on_base_; int force_intra_only_frame_; int superframe_has_intra_only_; + int use_post_encode_drop_; }; // Params: speed setting. @@ -1361,9 +1367,86 @@ TEST_P(DatarateOnePassCbrSvcSingleBR, OnePassCbrSvc2QL1TLScreen) { #endif } +// Params: speed setting. +class DatarateOnePassCbrSvcPostencodeDrop + : public DatarateOnePassCbrSvc, + public ::libvpx_test::CodecTestWithParam { + public: + DatarateOnePassCbrSvcPostencodeDrop() : DatarateOnePassCbrSvc(GET_PARAM(0)) { + memset(&svc_params_, 0, sizeof(svc_params_)); + } + virtual ~DatarateOnePassCbrSvcPostencodeDrop() {} + + protected: + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + speed_setting_ = GET_PARAM(1); + ResetModel(); + } +}; + +// Run SVC encoder for 2 quality layers (same resolution different, +// bitrates), 1 temporal layer, with screen content mode. +TEST_P(DatarateOnePassCbrSvcPostencodeDrop, OnePassCbrSvc2QL1TLScreen) { + cfg_.rc_buf_initial_sz = 200; + cfg_.rc_buf_optimal_sz = 200; + cfg_.rc_buf_sz = 400; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 52; + cfg_.rc_end_usage = VPX_CBR; + cfg_.g_lag_in_frames = 0; + cfg_.ss_number_layers = 2; + cfg_.ts_number_layers = 1; + cfg_.ts_rate_decimator[0] = 1; + cfg_.temporal_layering_mode = 0; + cfg_.g_error_resilient = 1; + cfg_.g_threads = 2; + svc_params_.scaling_factor_num[0] = 1; + svc_params_.scaling_factor_den[0] = 1; + svc_params_.scaling_factor_num[1] = 1; + svc_params_.scaling_factor_den[1] = 1; + cfg_.rc_dropframe_thresh = 30; + cfg_.kf_max_dist = 9999; + number_spatial_layers_ = cfg_.ss_number_layers; + number_temporal_layers_ = cfg_.ts_number_layers; + ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 300); + top_sl_width_ = 352; + top_sl_height_ = 288; + ResetModel(); + base_speed_setting_ = speed_setting_; + tune_content_ = 1; + use_post_encode_drop_ = 1; + // Set the layer bitrates, for 2 spatial layers, 1 temporal. + cfg_.rc_target_bitrate = 400; + cfg_.ss_target_bitrate[0] = 100; + cfg_.ss_target_bitrate[1] = 300; + cfg_.layer_target_bitrate[0] = 100; + cfg_.layer_target_bitrate[1] = 300; + for (int sl = 0; sl < 2; ++sl) { + float layer_framerate = 30.0; + layer_target_avg_bandwidth_[sl] = static_cast( + cfg_.layer_target_bitrate[sl] * 1000.0 / layer_framerate); + bits_in_buffer_model_[sl] = + cfg_.layer_target_bitrate[sl] * cfg_.rc_buf_initial_sz; + } + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + CheckLayerRateTargeting(number_spatial_layers_, number_temporal_layers_, 0.73, + 1.25); +#if CONFIG_VP9_DECODER + // The non-reference frames are expected to be mismatched frames as the + // encoder will avoid loopfilter on these frames. + EXPECT_EQ(num_nonref_frames_, GetMismatchFrames()); +#endif +} + VP9_INSTANTIATE_TEST_CASE(DatarateOnePassCbrSvcSingleBR, ::testing::Range(5, 10)); +VP9_INSTANTIATE_TEST_CASE(DatarateOnePassCbrSvcPostencodeDrop, + ::testing::Range(4, 5)); + VP9_INSTANTIATE_TEST_CASE(DatarateOnePassCbrSvcInterLayerPredSingleBR, ::testing::Range(5, 10), ::testing::Range(0, 3)); diff --git a/test/vp9_datarate_test.cc b/test/vp9_datarate_test.cc index da2609181..62f2852f6 100644 --- a/test/vp9_datarate_test.cc +++ b/test/vp9_datarate_test.cc @@ -22,7 +22,9 @@ namespace { class DatarateTestVP9 : public ::libvpx_test::EncoderTest { public: explicit DatarateTestVP9(const ::libvpx_test::CodecFactory *codec) - : EncoderTest(codec) {} + : EncoderTest(codec) { + tune_content_ = 0; + } protected: virtual ~DatarateTestVP9() {} @@ -114,6 +116,7 @@ class DatarateTestVP9 : public ::libvpx_test::EncoderTest { if (video->frame() == 0) { encoder->Control(VP8E_SET_CPUUSED, set_cpu_used_); encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); + encoder->Control(VP9E_SET_TUNE_CONTENT, tune_content_); } if (denoiser_offon_test_) { @@ -204,6 +207,7 @@ class DatarateTestVP9 : public ::libvpx_test::EncoderTest { vpx_codec_pts_t last_pts_; double timebase_; + int tune_content_; int frame_number_; // Counter for number of non-dropped/encoded frames. int tot_frame_number_; // Counter for total number of input frames. int64_t bits_total_[3]; @@ -713,6 +717,47 @@ TEST_P(DatarateTestVP9RealTime, RegionOfInterest) { free(roi_.roi_map); } +// Params: test mode, speed setting and index for bitrate array. +class DatarateTestVP9PostEncodeDrop + : public DatarateTestVP9, + public ::libvpx_test::CodecTestWithParam { + public: + DatarateTestVP9PostEncodeDrop() : DatarateTestVP9(GET_PARAM(0)) {} + + protected: + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + set_cpu_used_ = GET_PARAM(1); + ResetModel(); + } +}; + +// Check basic rate targeting for CBR mode, with 2 threads and dropped frames. +TEST_P(DatarateTestVP9PostEncodeDrop, PostEncodeDropScreenContent) { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_dropframe_thresh = 30; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 56; + cfg_.rc_end_usage = VPX_CBR; + cfg_.g_lag_in_frames = 0; + // Encode using multiple threads. + cfg_.g_threads = 2; + cfg_.g_error_resilient = 0; + tune_content_ = 1; + ::libvpx_test::I420VideoSource video("hantro_collage_w352h288.yuv", 352, 288, + 30, 1, 0, 300); + cfg_.rc_target_bitrate = 300; + ResetModel(); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + ASSERT_GE(effective_datarate_[0], cfg_.rc_target_bitrate * 0.85) + << " The datarate for the file is lower than target by too much!"; + ASSERT_LE(effective_datarate_[0], cfg_.rc_target_bitrate * 1.15) + << " The datarate for the file is greater than target by too much!"; +} + #if CONFIG_VP9_TEMPORAL_DENOISING // Params: speed setting. class DatarateTestVP9RealTimeDenoiser : public DatarateTestVP9RealTime { @@ -849,6 +894,9 @@ VP9_INSTANTIATE_TEST_CASE(DatarateTestVP9LargeVBR, VP9_INSTANTIATE_TEST_CASE(DatarateTestVP9RealTime, ::testing::Range(5, 10)); +VP9_INSTANTIATE_TEST_CASE(DatarateTestVP9PostEncodeDrop, + ::testing::Range(4, 5)); + #if CONFIG_VP9_TEMPORAL_DENOISING VP9_INSTANTIATE_TEST_CASE(DatarateTestVP9RealTimeDenoiser, ::testing::Range(5, 10)); diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c index fad492de7..40c7a626e 100644 --- a/vp9/encoder/vp9_encoder.c +++ b/vp9/encoder/vp9_encoder.c @@ -3488,6 +3488,11 @@ static void set_size_dependent_vars(VP9_COMP *cpi, int *q, int *bottom_index, // Decide q and q bounds. *q = vp9_rc_pick_q_and_bounds(cpi, bottom_index, top_index); + if (cpi->oxcf.rc_mode == VPX_CBR && cpi->rc.force_max_q) { + *q = cpi->rc.worst_quality; + cpi->rc.force_max_q = 0; + } + if (!frame_is_intra_only(cm)) { vp9_set_high_precision_mv(cpi, (*q) < HIGH_PRECISION_MV_QTHRESH); } @@ -3693,6 +3698,7 @@ static int encode_without_recode_loop(VP9_COMP *cpi, size_t *size, uint8_t *dest) { VP9_COMMON *const cm = &cpi->common; int q = 0, bottom_index = 0, top_index = 0; + int no_drop_scene_change = 0; const INTERP_FILTER filter_scaler = (is_one_pass_cbr_svc(cpi)) ? cpi->svc.downsample_filter_type[cpi->svc.spatial_layer_id] @@ -3823,9 +3829,11 @@ static int encode_without_recode_loop(VP9_COMP *cpi, size_t *size, // For 1 pass CBR, check if we are dropping this frame. // Never drop on key frame, if base layer is key for svc, // on scene change, or if superframe has layer sync. + if ((cpi->rc.high_source_sad || cpi->svc.high_source_sad_superframe) && + !(cpi->rc.use_post_encode_drop && cpi->svc.last_layer_dropped[0])) + no_drop_scene_change = 1; if (cpi->oxcf.pass == 0 && cpi->oxcf.rc_mode == VPX_CBR && - !frame_is_intra_only(cm) && !cpi->rc.high_source_sad && - !cpi->svc.high_source_sad_superframe && + !frame_is_intra_only(cm) && !no_drop_scene_change && !cpi->svc.superframe_has_layer_sync && (!cpi->use_svc || !cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame)) { @@ -3863,6 +3871,12 @@ static int encode_without_recode_loop(VP9_COMP *cpi, size_t *size, cpi->use_skin_detection = 1; } + // Enable post encode frame dropping for CBR on non key frame, when + // ext_use_post_encode_drop is specified by user. + cpi->rc.use_post_encode_drop = cpi->rc.ext_use_post_encode_drop && + cpi->oxcf.rc_mode == VPX_CBR && + cm->frame_type != KEY_FRAME; + vp9_set_quantizer(cm, q); vp9_set_variance_partition_thresholds(cpi, q, 0); @@ -3877,6 +3891,14 @@ static int encode_without_recode_loop(VP9_COMP *cpi, size_t *size, vp9_svc_assert_constraints_pattern(cpi); } + if (cpi->rc.last_post_encode_dropped_scene_change) { + cpi->rc.high_source_sad = 1; + cpi->svc.high_source_sad_superframe = 1; + // For now disable use_source_sad since Last_Source will not be the previous + // encoded but the dropped one. + cpi->sf.use_source_sad = 0; + cpi->rc.last_post_encode_dropped_scene_change = 0; + } // Check if this high_source_sad (scene/slide change) frame should be // encoded at high/max QP, and if so, set the q and adjust some rate // control parameters. @@ -4751,19 +4773,6 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi, size_t *size, cm->ref_frame_map[cpi->alt_fb_idx]); } - cpi->last_frame_dropped = 0; - cpi->svc.last_layer_dropped[cpi->svc.spatial_layer_id] = 0; - // Keep track of the frame buffer index updated/refreshed for the - // current encoded TL0 superframe. - if (cpi->svc.temporal_layer_id == 0) { - if (cpi->refresh_last_frame) - cpi->svc.fb_idx_upd_tl0[cpi->svc.spatial_layer_id] = cpi->lst_fb_idx; - else if (cpi->refresh_golden_frame) - cpi->svc.fb_idx_upd_tl0[cpi->svc.spatial_layer_id] = cpi->gld_fb_idx; - else if (cpi->refresh_alt_ref_frame) - cpi->svc.fb_idx_upd_tl0[cpi->svc.spatial_layer_id] = cpi->alt_fb_idx; - } - // Disable segmentation if it decrease rate/distortion ratio if (cpi->oxcf.aq_mode == LOOKAHEAD_AQ) vp9_try_disable_lookahead_aq(cpi, size, dest); @@ -4810,9 +4819,31 @@ static void encode_frame_to_data_rate(VP9_COMP *cpi, size_t *size, // Pick the loop filter level for the frame. loopfilter_frame(cpi, cm); + if (cpi->rc.use_post_encode_drop) save_coding_context(cpi); + // build the bitstream vp9_pack_bitstream(cpi, dest, size); + if (cpi->rc.use_post_encode_drop && cm->base_qindex < cpi->rc.worst_quality && + cpi->svc.spatial_layer_id == 0 && + post_encode_drop_screen_content(cpi, size)) { + restore_coding_context(cpi); + return; + } + + cpi->last_frame_dropped = 0; + cpi->svc.last_layer_dropped[cpi->svc.spatial_layer_id] = 0; + // Keep track of the frame buffer index updated/refreshed for the + // current encoded TL0 superframe. + if (cpi->svc.temporal_layer_id == 0) { + if (cpi->refresh_last_frame) + cpi->svc.fb_idx_upd_tl0[cpi->svc.spatial_layer_id] = cpi->lst_fb_idx; + else if (cpi->refresh_golden_frame) + cpi->svc.fb_idx_upd_tl0[cpi->svc.spatial_layer_id] = cpi->gld_fb_idx; + else if (cpi->refresh_alt_ref_frame) + cpi->svc.fb_idx_upd_tl0[cpi->svc.spatial_layer_id] = cpi->alt_fb_idx; + } + if (cm->seg.update_map) update_reference_segmentation_map(cpi); if (frame_is_intra_only(cm) == 0) { diff --git a/vp9/encoder/vp9_ratectrl.c b/vp9/encoder/vp9_ratectrl.c index c8931ae82..94066d0f0 100644 --- a/vp9/encoder/vp9_ratectrl.c +++ b/vp9/encoder/vp9_ratectrl.c @@ -398,6 +398,11 @@ void vp9_rc_init(const VP9EncoderConfig *oxcf, int pass, RATE_CONTROL *rc) { rc->max_gf_interval = vp9_rc_get_default_max_gf_interval( oxcf->init_framerate, rc->min_gf_interval); rc->baseline_gf_interval = (rc->min_gf_interval + rc->max_gf_interval) / 2; + + rc->force_max_q = 0; + rc->last_post_encode_dropped_scene_change = 0; + rc->use_post_encode_drop = 0; + rc->ext_use_post_encode_drop = 0; } static int check_buffer_above_thresh(VP9_COMP *cpi, int drop_mark) { @@ -515,6 +520,39 @@ static int drop_frame(VP9_COMP *cpi) { } } +int post_encode_drop_screen_content(VP9_COMP *cpi, size_t *size) { + size_t frame_size = *size << 3; + int64_t new_buffer_level = + cpi->rc.buffer_level + cpi->rc.avg_frame_bandwidth - (int64_t)frame_size; + + // For now we drop if new buffer level (given the encoded frame size) goes + // below 0. + if (new_buffer_level < 0) { + *size = 0; + vp9_rc_postencode_update_drop_frame(cpi); + // Update flag to use for next frame. + if (cpi->rc.high_source_sad || + (cpi->use_svc && cpi->svc.high_source_sad_superframe)) + cpi->rc.last_post_encode_dropped_scene_change = 1; + // Force max_q on next fame. + cpi->rc.force_max_q = 1; + cpi->rc.avg_frame_qindex[INTER_FRAME] = cpi->rc.worst_quality; + cpi->last_frame_dropped = 1; + cpi->ext_refresh_frame_flags_pending = 0; + if (cpi->use_svc) { + cpi->svc.last_layer_dropped[cpi->svc.spatial_layer_id] = 1; + cpi->svc.drop_spatial_layer[cpi->svc.spatial_layer_id] = 1; + cpi->svc.drop_count[cpi->svc.spatial_layer_id]++; + cpi->svc.skip_enhancement_layer = 1; + } + return 1; + } + + cpi->rc.force_max_q = 0; + cpi->rc.last_post_encode_dropped_scene_change = 0; + return 0; +} + int vp9_rc_drop_frame(VP9_COMP *cpi) { SVC *svc = &cpi->svc; int svc_prev_layer_dropped = 0; @@ -834,7 +872,7 @@ static int calc_active_worst_quality_one_pass_cbr(const VP9_COMP *cpi) { int active_worst_quality; int ambient_qp; unsigned int num_frames_weight_key = 5 * cpi->svc.number_temporal_layers; - if (frame_is_intra_only(cm) || rc->reset_high_source_sad) + if (frame_is_intra_only(cm) || rc->reset_high_source_sad || rc->force_max_q) return rc->worst_quality; // For ambient_qp we use minimum of avg_frame_qindex[KEY_FRAME/INTER_FRAME] // for the first few frames following key frame. These are both initialized diff --git a/vp9/encoder/vp9_ratectrl.h b/vp9/encoder/vp9_ratectrl.h index a9f75555e..d9233b77e 100644 --- a/vp9/encoder/vp9_ratectrl.h +++ b/vp9/encoder/vp9_ratectrl.h @@ -186,6 +186,14 @@ typedef struct { int force_qpmin; int reset_high_source_sad; double perc_arf_usage; + int force_max_q; + // Last frame was dropped post encode on scene change. + int last_post_encode_dropped_scene_change; + // Enable post encode frame dropping for screen content. Only enabled when + // ext_use_post_encode_drop is enabled by user. + int use_post_encode_drop; + // External flag to enable post encode frame dropping, controlled by user. + int ext_use_post_encode_drop; } RATE_CONTROL; struct VP9_COMP; @@ -247,6 +255,9 @@ void vp9_rc_postencode_update_drop_frame(struct VP9_COMP *cpi); // Changes only the rate correction factors in the rate control structure. void vp9_rc_update_rate_correction_factors(struct VP9_COMP *cpi); +// Post encode drop for CBR screen-content mode. +int post_encode_drop_screen_content(struct VP9_COMP *cpi, size_t *size); + // Decide if we should drop this frame: For 1-pass CBR. // Changes only the decimation count in the rate control structure int vp9_rc_drop_frame(struct VP9_COMP *cpi); diff --git a/vp9/encoder/vp9_svc_layercontext.c b/vp9/encoder/vp9_svc_layercontext.c index 1321c4575..7cab7ffb2 100644 --- a/vp9/encoder/vp9_svc_layercontext.c +++ b/vp9/encoder/vp9_svc_layercontext.c @@ -329,6 +329,7 @@ void vp9_restore_layer_context(VP9_COMP *const cpi) { LAYER_CONTEXT *const lc = get_layer_context(cpi); const int old_frame_since_key = cpi->rc.frames_since_key; const int old_frame_to_key = cpi->rc.frames_to_key; + const int old_ext_use_post_encode_drop = cpi->rc.ext_use_post_encode_drop; cpi->rc = lc->rc; cpi->twopass = lc->twopass; @@ -346,7 +347,7 @@ void vp9_restore_layer_context(VP9_COMP *const cpi) { cpi->rc.frames_since_key = old_frame_since_key; cpi->rc.frames_to_key = old_frame_to_key; } - + cpi->rc.ext_use_post_encode_drop = old_ext_use_post_encode_drop; // For spatial-svc, allow cyclic-refresh to be applied on the spatial layers, // for the base temporal layer. if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && diff --git a/vp9/vp9_cx_iface.c b/vp9/vp9_cx_iface.c index 3b2d9a866..1a886c643 100644 --- a/vp9/vp9_cx_iface.c +++ b/vp9/vp9_cx_iface.c @@ -1625,6 +1625,14 @@ static vpx_codec_err_t ctrl_set_render_size(vpx_codec_alg_priv_t *ctx, return update_extra_cfg(ctx, &extra_cfg); } +static vpx_codec_err_t ctrl_set_postencode_drop(vpx_codec_alg_priv_t *ctx, + va_list args) { + VP9_COMP *const cpi = ctx->cpi; + const unsigned int data = va_arg(args, unsigned int); + cpi->rc.ext_use_post_encode_drop = data; + return VPX_CODEC_OK; +} + static vpx_codec_ctrl_fn_map_t encoder_ctrl_maps[] = { { VP8_COPY_REFERENCE, ctrl_copy_reference }, @@ -1668,6 +1676,7 @@ static vpx_codec_ctrl_fn_map_t encoder_ctrl_maps[] = { { VP9E_SET_RENDER_SIZE, ctrl_set_render_size }, { VP9E_SET_TARGET_LEVEL, ctrl_set_target_level }, { VP9E_SET_ROW_MT, ctrl_set_row_mt }, + { VP9E_SET_POSTENCODE_DROP, ctrl_set_postencode_drop }, { VP9E_ENABLE_MOTION_VECTOR_UNIT_TEST, ctrl_enable_motion_vector_unit_test }, { VP9E_SET_SVC_INTER_LAYER_PRED, ctrl_set_svc_inter_layer_pred }, { VP9E_SET_SVC_FRAME_DROP_LAYER, ctrl_set_svc_frame_drop_layer }, diff --git a/vpx/vp8cx.h b/vpx/vp8cx.h index 7a232b0ef..3975d4ba4 100644 --- a/vpx/vp8cx.h +++ b/vpx/vp8cx.h @@ -660,6 +660,16 @@ enum vp8e_enc_control_id { * 1. The default value is set to be 1. */ VP9E_SET_TPL, + + /*!\brief Codec control function to enable postencode frame drop. + * + * This will allow encoder to drop frame after it's encoded. + * + * 0: Off (default), 1: Enabled + * + * Supported in codecs: VP9 + */ + VP9E_SET_POSTENCODE_DROP, }; /*!\brief vpx 1-D scaling mode -- 2.40.0