From 87ff8620b27a70c8411bf0941eb846e358e03ca1 Mon Sep 17 00:00:00 2001 From: Paul Wilkins Date: Wed, 2 Nov 2011 13:30:10 +0000 Subject: [PATCH] Segmentation: Reference frames Modify reference frame segmentation so that ONE or MORE reference frames may be marked as a available for a given segment. Fixed bugs relating to segment coding of INTRA and some INTER modes at the segment level. Modified Q boost for static areas based on ambient average Q. Strong results now on clips with significant static areas. (some data points in derf set as high as 9% and some static & slide show type content in YT set > 20%) Change-Id: Ia79f912efa84b977f35a23683ae3643251e24f0c --- vp8/common/entropy.c | 4 +- vp8/common/entropy.h | 2 +- vp8/common/seg_common.c | 49 +++++++++++++ vp8/common/seg_common.h | 26 +++++++ vp8/decoder/decodemv.c | 113 +++++++++++++++++++++++++----- vp8/decoder/decodframe.c | 2 +- vp8/encoder/bitstream.c | 141 +++++++++++++++++++++++++++++--------- vp8/encoder/encodeframe.c | 21 ++++-- vp8/encoder/onyx_if.c | 14 ++-- vp8/encoder/pickinter.c | 8 ++- vp8/encoder/rdopt.c | 8 ++- 11 files changed, 317 insertions(+), 71 deletions(-) diff --git a/vp8/common/entropy.c b/vp8/common/entropy.c index 62e38d772..ba1e5aed5 100644 --- a/vp8/common/entropy.c +++ b/vp8/common/entropy.c @@ -93,9 +93,9 @@ DECLARE_ALIGNED(64, short, vp8_default_zig_zag_mask_8x8[64]);//int64_t #endif #if CONFIG_SEGFEATURES -const int vp8_mb_feature_data_bits[SEG_LVL_MAX] = {7, 6, 2, 3, 4, 2}; +const int vp8_seg_feature_data_bits[SEG_LVL_MAX] = {7, 6, 4, 4, 4, 2}; #else -const int vp8_mb_feature_data_bits[SEG_LVL_MAX] = {7, 6}; +const int vp8_seg_feature_data_bits[SEG_LVL_MAX] = {7, 6}; #endif /* Array indices are identical to previously-existing CONTEXT_NODE indices */ diff --git a/vp8/common/entropy.h b/vp8/common/entropy.h index 4d15449c9..e75c3966b 100644 --- a/vp8/common/entropy.h +++ b/vp8/common/entropy.h @@ -106,7 +106,7 @@ extern short vp8_default_zig_zag_mask[16]; extern DECLARE_ALIGNED(64, const int, vp8_default_zig_zag1d_8x8[64]); extern short vp8_default_zig_zag_mask_8x8[64];//int64_t #endif -extern const int vp8_mb_feature_data_bits[SEG_LVL_MAX]; +extern const int vp8_seg_feature_data_bits[SEG_LVL_MAX]; void vp8_coef_tree_initialize(void); #endif diff --git a/vp8/common/seg_common.c b/vp8/common/seg_common.c index f7c2afdec..ddf7c6fcf 100644 --- a/vp8/common/seg_common.c +++ b/vp8/common/seg_common.c @@ -53,6 +53,55 @@ int is_segfeature_signed( SEG_LVL_FEATURES feature_id ) return ( segfeaturedata_signed[feature_id] ); } +void clear_segdata( MACROBLOCKD *xd, + int segment_id, + SEG_LVL_FEATURES feature_id) +{ + xd->segment_feature_data[segment_id][feature_id] = 0; +} + +void set_segdata( MACROBLOCKD *xd, + int segment_id, + SEG_LVL_FEATURES feature_id, + int seg_data ) +{ + xd->segment_feature_data[segment_id][feature_id] = seg_data; +} + +int get_segdata( MACROBLOCKD *xd, + int segment_id, + SEG_LVL_FEATURES feature_id ) +{ + return xd->segment_feature_data[segment_id][feature_id]; +} + +void clear_segref( MACROBLOCKD *xd, int segment_id ) +{ + xd->segment_feature_data[segment_id][SEG_LVL_REF_FRAME] = 0; +} + +void set_segref( MACROBLOCKD *xd, + int segment_id, + MV_REFERENCE_FRAME ref_frame ) +{ + xd->segment_feature_data[segment_id][SEG_LVL_REF_FRAME] |= + (1 << ref_frame); +} + +int check_segref( MACROBLOCKD *xd, + int segment_id, + MV_REFERENCE_FRAME ref_frame ) +{ + return ( xd->segment_feature_data[segment_id][SEG_LVL_REF_FRAME] & + (1 << ref_frame) ) ? 1 : 0; +} + +int check_segref_inter(MACROBLOCKD *xd, int segment_id) +{ + return ( xd->segment_feature_data[segment_id][SEG_LVL_REF_FRAME] & + ~(1 << INTRA_FRAME) ) ? 1 : 0; +} + // TBD? Functions to read and write segment data with range / validity checking #endif diff --git a/vp8/common/seg_common.h b/vp8/common/seg_common.h index cea16c885..b0498edf5 100644 --- a/vp8/common/seg_common.h +++ b/vp8/common/seg_common.h @@ -30,5 +30,31 @@ void disable_segfeature( MACROBLOCKD *xd, int is_segfeature_signed( SEG_LVL_FEATURES feature_id ); +void clear_segdata( MACROBLOCKD *xd, + int segment_id, + SEG_LVL_FEATURES feature_id); + +void set_segdata( MACROBLOCKD *xd, + int segment_id, + SEG_LVL_FEATURES feature_id, + int seg_data ); + +int get_segdata( MACROBLOCKD *xd, + int segment_id, + SEG_LVL_FEATURES feature_id ); + +void clear_segref( MACROBLOCKD *xd, int segment_id ); + +void set_segref( MACROBLOCKD *xd, + int segment_id, + MV_REFERENCE_FRAME ref_frame ); + +int check_segref( MACROBLOCKD *xd, + int segment_id, + MV_REFERENCE_FRAME ref_frame ); + +int check_segref_inter(MACROBLOCKD *xd, int segment_id); + + #endif /* __INC_SEG_COMMON_H__ */ diff --git a/vp8/decoder/decodemv.c b/vp8/decoder/decodemv.c index 597b6ea2a..34a3c84e0 100644 --- a/vp8/decoder/decodemv.c +++ b/vp8/decoder/decodemv.c @@ -232,26 +232,25 @@ static void read_mvcontexts(vp8_reader *bc, MV_CONTEXT *mvc) // Read the referncence frame static MV_REFERENCE_FRAME read_ref_frame( VP8D_COMP *pbi, + vp8_reader *const bc, unsigned char segment_id ) { MV_REFERENCE_FRAME ref_frame; + int seg_ref_active; #if CONFIG_SEGFEATURES MACROBLOCKD *const xd = &pbi->mb; - // Is the segment level refernce frame feature enabled for this segment - if ( segfeature_active( xd, segment_id, SEG_LVL_REF_FRAME ) ) - { - ref_frame = - xd->segment_feature_data[segment_id][SEG_LVL_REF_FRAME]; - } - else + seg_ref_active = segfeature_active( xd, + segment_id, + SEG_LVL_REF_FRAME ); +#else + seg_ref_active = 0; #endif - // Per MB read of the reference frame + // Segment reference frame features not available + if ( !seg_ref_active ) { - vp8_reader *const bc = &pbi->bc; - ref_frame = (MV_REFERENCE_FRAME) vp8_read(bc, pbi->prob_intra); @@ -265,6 +264,79 @@ static MV_REFERENCE_FRAME read_ref_frame( VP8D_COMP *pbi, } } +#if CONFIG_SEGFEATURES + // Segment reference frame features are enabled + else + { + // If there are no inter reference frames enabled we can set INTRA + if ( !check_segref_inter(xd, segment_id) ) + { + ref_frame = INTRA_FRAME; + } + else + { + // Else if there are both intra and inter options we need to read + // the inter / intra flag, else mark as inter. + if ( check_segref( xd, segment_id, INTRA_FRAME ) ) + ref_frame = (MV_REFERENCE_FRAME) vp8_read(bc, pbi->prob_intra); + else + ref_frame = 1; // note this unchanged = LAST + + if ( ref_frame ) + { + // Now consider last vs (golden or alt) flag.... + // If Last is not enabled + if ( !check_segref( xd, segment_id, LAST_FRAME ) ) + { + // If not golden then it must be altref + if (!check_segref( xd, segment_id, GOLDEN_FRAME )) + { + ref_frame = ALTREF_FRAME; + } + // Not Altref therefore must be Golden + else if (!check_segref( xd, segment_id, + ALTREF_FRAME )) + { + ref_frame = GOLDEN_FRAME; + } + // Else we must read bit to decide. + else + { + ref_frame = (MV_REFERENCE_FRAME)((int)ref_frame + + (int)(1 + vp8_read(bc, pbi->prob_gf))); + } + } + // Both last and at least one of alt or golden are enabled + else if ( check_segref( xd, segment_id, GOLDEN_FRAME ) || + check_segref( xd, segment_id, ALTREF_FRAME ) ) + { + // Read flag to indicate (golden or altref) vs last + if (vp8_read(bc, pbi->prob_last)) + { + // If not golden then it must be altref + if (!check_segref( xd, segment_id, GOLDEN_FRAME )) + { + ref_frame = ALTREF_FRAME; + } + // Not Altref therefore must be Golden + else if (!check_segref( xd, segment_id, + ALTREF_FRAME )) + { + ref_frame = GOLDEN_FRAME; + } + else + { + ref_frame = (MV_REFERENCE_FRAME)((int)ref_frame + + (int)(1 + vp8_read(bc, pbi->prob_gf))); + } + } + // ELSE LAST + } + } + } + } +#endif + return (MV_REFERENCE_FRAME)ref_frame; } @@ -458,7 +530,7 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, } // Read the reference frame - mbmi->ref_frame = read_ref_frame( pbi, mbmi->segment_id ); + mbmi->ref_frame = read_ref_frame( pbi, bc, mbmi->segment_id ); // If reference frame is an Inter frame if (mbmi->ref_frame) @@ -634,15 +706,22 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, } else { -#if CONFIG_SEGFEATURES - // TBD HANDLE INTRA MODE CASE -#endif - /* required for left and above block mv */ mbmi->mv.as_int = 0; - /* MB is intra coded */ - if ((mbmi->mode = (MB_PREDICTION_MODE) vp8_read_ymode(bc, pbi->common.fc.ymode_prob)) == B_PRED) +#if CONFIG_SEGFEATURES + if ( segfeature_active( xd, mbmi->segment_id, SEG_LVL_MODE ) ) + mbmi->mode = (MB_PREDICTION_MODE) + get_segdata( xd, mbmi->segment_id, SEG_LVL_MODE ); + else +#endif + { + mbmi->mode = (MB_PREDICTION_MODE) + vp8_read_ymode(bc, pbi->common.fc.ymode_prob); + } + + // If MB mode is BPRED read the block modes + if (mbmi->mode == B_PRED) { int j = 0; do diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c index bfa13caec..842bcc209 100644 --- a/vp8/decoder/decodframe.c +++ b/vp8/decoder/decodframe.c @@ -883,7 +883,7 @@ int vp8_decode_frame(VP8D_COMP *pbi) int mb_row; int i, j, k, l; - const int *const mb_feature_data_bits = vp8_mb_feature_data_bits; + const int *const mb_feature_data_bits = vp8_seg_feature_data_bits; int corrupt_tokens = 0; int prev_independent_partitions = pbi->independent_partitions; diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c index 7df367f11..9eb9de1ca 100644 --- a/vp8/encoder/bitstream.c +++ b/vp8/encoder/bitstream.c @@ -873,6 +873,90 @@ static void write_mb_features(vp8_writer *w, const MB_MODE_INFO *mi, const MACRO } } +// This function encodes the reference frame +static void encode_ref_frame( vp8_writer *const w, + MACROBLOCKD *xd, + int segment_id, + MV_REFERENCE_FRAME rf, + int prob_intra_coded, + int prob_last_coded, + int prob_gf_coded ) +{ + int seg_ref_active; +#if CONFIG_SEGFEATURES + seg_ref_active = segfeature_active( xd, + segment_id, + SEG_LVL_REF_FRAME ); +#else + seg_ref_active = 0; +#endif + + // No segment features or segment reference frame featuure is disabled + if ( !seg_ref_active ) + { + if (rf == INTRA_FRAME) + { + vp8_write(w, 0, prob_intra_coded); + } + else /* inter coded */ + { + vp8_write(w, 1, prob_intra_coded); + + if (rf == LAST_FRAME) + { + vp8_write(w, 0, prob_last_coded); + } + else + { + vp8_write(w, 1, prob_last_coded); + vp8_write(w, (rf == GOLDEN_FRAME) ? 0 : 1, prob_gf_coded); + } + } + } +#if CONFIG_SEGFEATURES + else + { + if (rf == INTRA_FRAME) + { + // This MB intra coded. If inter also allowed we must code + // an explicit inter/intra flag. + if ( check_segref_inter( xd, segment_id ) ) + vp8_write(w, 0, prob_intra_coded); + } + else /* inter coded */ + { + // If intra also allowed we must code an explicit intra/inter flag. + if ( check_segref( xd, segment_id, INTRA_FRAME ) ) + vp8_write(w, 1, prob_intra_coded); + + if (rf == LAST_FRAME) + { + // If GOLDEN or ALTREF allowed we must code explicit flag. + if ( check_segref( xd, segment_id, GOLDEN_FRAME ) || + check_segref( xd, segment_id, ALTREF_FRAME ) ) + { + vp8_write(w, 0, prob_last_coded); + } + } + else + { + // if LAST is allowed we must code explicit flag + if ( check_segref( xd, segment_id, LAST_FRAME ) ) + { + vp8_write(w, 1, prob_last_coded); + } + + // if GOLDEN and ALTREF allowed we must code an explicit flag + if ( check_segref( xd, segment_id, GOLDEN_FRAME ) && + check_segref( xd, segment_id, ALTREF_FRAME ) ) + { + vp8_write(w, (rf == GOLDEN_FRAME) ? 0 : 1, prob_gf_coded); + } + } + } + } +#endif +} static void pack_inter_mode_mvs(VP8_COMP *const cpi) { @@ -969,14 +1053,16 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) const MB_PREDICTION_MODE mode = mi->mode; const int segment_id = mi->segment_id; - //MACROBLOCKD *xd = &cpi->mb.e_mbd; - // Distance of Mb to the various image edges. // These specified to 8th pel as they are always compared to MV values that are in 1/8th pel units xd->mb_to_left_edge = -((mb_col * 16) << 3); xd->mb_to_right_edge = ((pc->mb_cols - 1 - mb_col) * 16) << 3; xd->mb_to_top_edge = -((mb_row * 16)) << 3; xd->mb_to_bottom_edge = ((pc->mb_rows - 1 - mb_row) * 16) << 3; + + // Make sure the MacroBlockD mode info pointer is set correctly + xd->mode_info_context = m; + #if CONFIG_SEGMENTATION xd->up_available = (mb_row != 0); xd->left_available = (mb_col != 0); @@ -1036,20 +1122,23 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) vp8_encode_bool(w, mi->mb_skip_coeff, prob_skip_false); } + // Encode the reference frame. + encode_ref_frame( w, xd, segment_id, rf, + cpi->prob_intra_coded, + prob_last_coded, prob_gf_coded ); + if (rf == INTRA_FRAME) { -#if CONFIG_SEGFEATURES - // Is the segment coding of reference frame enabled - if ( !segfeature_active( xd, segment_id, SEG_LVL_REF_FRAME ) ) -#endif - { - vp8_write(w, 0, cpi->prob_intra_coded); - } - -#ifdef ENTROPY_STATS + #ifdef ENTROPY_STATS active_section = 6; -#endif + #endif + +#if CONFIG_SEGFEATURES + if ( !segfeature_active( xd, segment_id, SEG_LVL_MODE ) ) + write_ymode(w, mode, pc->fc.ymode_prob); +#else write_ymode(w, mode, pc->fc.ymode_prob); +#endif if (mode == B_PRED) { @@ -1062,28 +1151,11 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) write_uv_mode(w, mi->uv_mode, pc->fc.uv_mode_prob); } - else /* inter coded */ + else { int_mv best_mv; vp8_prob mv_ref_p [VP8_MVREFS-1]; -#if CONFIG_SEGFEATURES - // Test to see if segment level coding of ref frame is enabled - if ( !segfeature_active( xd, segment_id, SEG_LVL_REF_FRAME ) ) -#endif - { - vp8_write(w, 1, cpi->prob_intra_coded); - - if (rf == LAST_FRAME) - vp8_write(w, 0, prob_last_coded); - else - { - vp8_write(w, 1, prob_last_coded); - vp8_write(w, (rf == GOLDEN_FRAME) - ? 0 : 1, prob_gf_coded); - } - } - { int_mv n1, n2; int ct[4]; @@ -1103,10 +1175,13 @@ static void pack_inter_mode_mvs(VP8_COMP *const cpi) #if CONFIG_SEGFEATURES // Is the segment coding of reference frame enabled if ( !segfeature_active( xd, segment_id, SEG_LVL_MODE ) ) -#endif { write_mv_ref(w, mode, mv_ref_p); - + } +#else + write_mv_ref(w, mode, mv_ref_p); +#endif + { switch (mode) /* new, split require MVs */ { case NEWMV: @@ -1913,7 +1988,7 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size) oh.version = pc->version; oh.first_partition_length_in_bytes = 0; - mb_feature_data_bits = vp8_mb_feature_data_bits; + mb_feature_data_bits = vp8_seg_feature_data_bits; cx_data += 3; #if defined(SECTIONBITS_OUTPUT) diff --git a/vp8/encoder/encodeframe.c b/vp8/encoder/encodeframe.c index ec679846e..7192ae539 100644 --- a/vp8/encoder/encodeframe.c +++ b/vp8/encoder/encodeframe.c @@ -1584,11 +1584,12 @@ int vp8cx_encode_inter_macroblock int intra_error = 0; int rate; int distortion; + int segment_id = xd->mode_info_context->mbmi.segment_id; x->skip = 0; if (xd->segmentation_enabled) - x->encode_breakout = cpi->segment_encode_breakout[xd->mode_info_context->mbmi.segment_id]; + x->encode_breakout = cpi->segment_encode_breakout[segment_id]; else x->encode_breakout = cpi->oxcf.encode_breakout; @@ -1650,7 +1651,7 @@ int vp8cx_encode_inter_macroblock if (cpi->cyclic_refresh_mode_enabled) { // Clear segment_id back to 0 if not coded (last frame 0,0) - if ( (xd->mode_info_context->mbmi.segment_id == 1) && + if ( (segment_id == 1) && ( (xd->mode_info_context->mbmi.ref_frame != LAST_FRAME) || (xd->mode_info_context->mbmi.mode != ZEROMV) ) ) { @@ -1711,10 +1712,20 @@ int vp8cx_encode_inter_macroblock } #if CONFIG_SEGFEATURES - // Dont increment usage count if ref frame coded at segment level - if ( !segfeature_active( xd, xd->mode_info_context->mbmi.segment_id, - SEG_LVL_REF_FRAME ) ) + + // If we have just a single reference frame coded for a segment then + // exclude from the reference frame counts used to work out + // probabilities. NOTE: At the moment we dont support custom trees + // for the reference frame coding for each segment but this is a + // possible future action. + if ( !segfeature_active( xd, segment_id, SEG_LVL_REF_FRAME ) || + ( ( check_segref( xd, segment_id, INTRA_FRAME ) + + check_segref( xd, segment_id, LAST_FRAME ) + + check_segref( xd, segment_id, GOLDEN_FRAME ) + + check_segref( xd, segment_id, ALTREF_FRAME ) ) > 1 ) ) + { cpi->count_mb_ref_frame_usage[xd->mode_info_context->mbmi.ref_frame]++; + } #else cpi->count_mb_ref_frame_usage[xd->mode_info_context->mbmi.ref_frame] ++; #endif diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c index 35b45673c..ee3a59eeb 100644 --- a/vp8/encoder/onyx_if.c +++ b/vp8/encoder/onyx_if.c @@ -531,7 +531,8 @@ static void init_seg_features(VP8_COMP *cpi) xd->update_mb_segmentation_map = 1; xd->update_mb_segmentation_data = 1; - xd->segment_feature_data[1][SEG_LVL_ALT_Q] = -3; + xd->segment_feature_data[1][SEG_LVL_ALT_Q] = + -(2+(cpi->ni_av_qi >> 3)); xd->segment_feature_data[1][SEG_LVL_ALT_LF] = -2; enable_segfeature(xd, 1, SEG_LVL_ALT_Q); @@ -563,8 +564,7 @@ static void init_seg_features(VP8_COMP *cpi) if ( high_q ) { - xd->segment_feature_data[1] - [SEG_LVL_REF_FRAME] = ALTREF_FRAME; + set_segref(xd, 1, ALTREF_FRAME); xd->segment_feature_data[1][SEG_LVL_MODE] = ZEROMV; xd->segment_feature_data[1][SEG_LVL_EOB] = 0; @@ -598,10 +598,12 @@ static void init_seg_features(VP8_COMP *cpi) enable_segfeature(xd, 0, SEG_LVL_REF_FRAME); enable_segfeature(xd, 0, SEG_LVL_MODE); - // All mbs should use ALTREF_FRAME, ZEROMV - xd->segment_feature_data[0][SEG_LVL_REF_FRAME] = ALTREF_FRAME; + // All mbs should use ALTREF_FRAME, ZEROMV exclusively + clear_segref(xd, 0); + set_segref(xd, 0, ALTREF_FRAME); + clear_segref(xd, 1); + set_segref(xd, 1, ALTREF_FRAME); xd->segment_feature_data[0][SEG_LVL_MODE] = ZEROMV; - xd->segment_feature_data[1][SEG_LVL_REF_FRAME] = ALTREF_FRAME; xd->segment_feature_data[1][SEG_LVL_MODE] = ZEROMV; // Skip all MBs if high Q diff --git a/vp8/encoder/pickinter.c b/vp8/encoder/pickinter.c index cb7a4859b..3eb446e1c 100644 --- a/vp8/encoder/pickinter.c +++ b/vp8/encoder/pickinter.c @@ -527,9 +527,11 @@ void vp8_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, { unsigned char segment_id = xd->mode_info_context->mbmi.segment_id; - if ( segfeature_active( xd, segment_id, SEG_LVL_REF_FRAME ) && - ( xd->mode_info_context->mbmi.ref_frame != - xd->segment_feature_data[segment_id][SEG_LVL_REF_FRAME])) + // If the segment reference frame feature is enabled.... + // then do nothing if the current ref frame is not allowed.. + if ( segfeature_active( xd, segment_id, SEG_LVL_REF_FRAME ) && + !check_segref( xd, segment_id, + xd->mode_info_context->mbmi.ref_frame ) ) { continue; } diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c index 2e52a00dc..644739371 100644 --- a/vp8/encoder/rdopt.c +++ b/vp8/encoder/rdopt.c @@ -2130,9 +2130,11 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int { unsigned char segment_id = xd->mode_info_context->mbmi.segment_id; - if ( segfeature_active( xd, segment_id, SEG_LVL_REF_FRAME ) && - ( xd->mode_info_context->mbmi.ref_frame != - xd->segment_feature_data[segment_id][SEG_LVL_REF_FRAME])) + // If the segment reference frame feature is enabled.... + // then do nothing if the current ref frame is not allowed.. + if ( segfeature_active( xd, segment_id, SEG_LVL_REF_FRAME ) && + !check_segref( xd, segment_id, + xd->mode_info_context->mbmi.ref_frame ) ) { continue; } -- 2.40.0