From d6261b812226bc61c0a55531501c51ea172cda9c Mon Sep 17 00:00:00 2001 From: Fiona Glaser Date: Fri, 22 May 2009 16:57:33 -0700 Subject: [PATCH] Various CABAC and CAVLC optimizations Backport CAVLC partial-inlining early termination to CABAC (~2-4% faster CABAC residual coding) --- encoder/cabac.c | 99 +++++++++++++++++++------------------------------ encoder/cavlc.c | 42 ++++++++++----------- 2 files changed, 58 insertions(+), 83 deletions(-) diff --git a/encoder/cabac.c b/encoder/cabac.c index ff6b7d28..82025826 100644 --- a/encoder/cabac.c +++ b/encoder/cabac.c @@ -54,7 +54,7 @@ static inline void x264_cabac_mb_type_intra( x264_t *h, x264_cabac_t *cb, int i_ else { x264_cabac_encode_decision( cb, ctx2, 1 ); - x264_cabac_encode_decision_noup( cb, ctx3, h->mb.i_cbp_chroma != 1 ); + x264_cabac_encode_decision_noup( cb, ctx3, h->mb.i_cbp_chroma>>1 ); } x264_cabac_encode_decision( cb, ctx4, i_pred>>1 ); x264_cabac_encode_decision_noup( cb, ctx5, i_pred&1 ); @@ -206,7 +206,7 @@ static void x264_cabac_mb_intra4x4_pred_mode( x264_cabac_t *cb, int i_pred, int i_mode--; x264_cabac_encode_decision( cb, 69, (i_mode )&0x01 ); x264_cabac_encode_decision( cb, 69, (i_mode >> 1)&0x01 ); - x264_cabac_encode_decision( cb, 69, (i_mode >> 2)&0x01 ); + x264_cabac_encode_decision( cb, 69, (i_mode >> 2) ); } } @@ -586,7 +586,7 @@ static const uint8_t coeff_abs_level_transition[2][8] = { static const int count_cat_m1[5] = {15, 14, 15, 3, 14}; #if !RDO_SKIP_BS -static void block_residual_write_cabac( x264_t *h, x264_cabac_t *cb, int i_ctxBlockCat, int i_idx, int16_t *l, int ctxidxinc ) +static void block_residual_write_cabac( x264_t *h, x264_cabac_t *cb, int i_ctxBlockCat, int16_t *l ) { const int i_ctx_sig = significant_coeff_flag_offset[h->mb.b_interlaced][i_ctxBlockCat]; const int i_ctx_last = last_coeff_flag_offset[h->mb.b_interlaced][i_ctxBlockCat]; @@ -599,18 +599,6 @@ static void block_residual_write_cabac( x264_t *h, x264_cabac_t *cb, int i_ctxBl int node_ctx = 0; int i = 0; - if( i_ctxBlockCat != DCT_LUMA_8x8 ) - { - /* coded block flag */ - if( h->mb.cache.non_zero_count[x264_scan8[i_idx]] ) - x264_cabac_encode_decision( cb, ctxidxinc, 1 ); - else - { - x264_cabac_encode_decision( cb, ctxidxinc, 0 ); - return; - } - } - i_last = h->quantf.coeff_last[i_ctxBlockCat](l); #define WRITE_SIGMAP( l8x8 )\ @@ -684,7 +672,7 @@ static void block_residual_write_cabac( x264_t *h, x264_cabac_t *cb, int i_ctxBl x264_cabac_encode_bypass( cb, i_coeff_sign[i_coeff] ); } while( i_coeff > 0 ); } -#define block_residual_write_cabac_8x8( h, cb, idx, l ) block_residual_write_cabac( h, cb, DCT_LUMA_8x8, idx, l, 0 ) +#define block_residual_write_cabac_8x8( h, cb, l ) block_residual_write_cabac( h, cb, DCT_LUMA_8x8, l ) #else @@ -692,35 +680,20 @@ static void block_residual_write_cabac( x264_t *h, x264_cabac_t *cb, int i_ctxBl * this is slightly incorrect because the sigmap is not reversible * (contexts are repeated). However, there is nearly no quality penalty * for this (~0.001db) and the speed boost (~30%) is worth it. */ -static void ALWAYS_INLINE block_residual_write_cabac_internal( x264_t *h, x264_cabac_t *cb, int i_ctxBlockCat, int i_idx, int16_t *l, int ctxidxinc, int b_8x8 ) +static void ALWAYS_INLINE block_residual_write_cabac_internal( x264_t *h, x264_cabac_t *cb, int i_ctxBlockCat, int16_t *l, int b_8x8 ) { const int i_ctx_sig = significant_coeff_flag_offset[h->mb.b_interlaced][i_ctxBlockCat]; const int i_ctx_last = last_coeff_flag_offset[h->mb.b_interlaced][i_ctxBlockCat]; const int i_ctx_level = coeff_abs_level_m1_offset[i_ctxBlockCat]; const uint8_t *significant_coeff_flag_offset = significant_coeff_flag_offset_8x8[h->mb.b_interlaced]; - int i_last, i_coeff_abs, ctx, i, node_ctx, i_count_m1; - - if( !b_8x8 ) - { - /* coded block flag */ - if( h->mb.cache.non_zero_count[x264_scan8[i_idx]] ) - x264_cabac_encode_decision( cb, ctxidxinc, 1 ); - else - { - x264_cabac_encode_decision( cb, ctxidxinc, 0 ); - return; - } - i_count_m1 = count_cat_m1[i_ctxBlockCat]; - } - else - i_count_m1 = 63; + int i_last, i_coeff_abs, ctx, i, node_ctx; i_last = h->quantf.coeff_last[i_ctxBlockCat](l); i_coeff_abs = abs(l[i_last]); ctx = coeff_abs_level1_ctx[0] + i_ctx_level; - if( i_last != i_count_m1 ) + if( i_last != (b_8x8 ? 63 : count_cat_m1[i_ctxBlockCat]) ) { x264_cabac_encode_decision( cb, i_ctx_sig + (b_8x8?significant_coeff_flag_offset[i_last]:i_last), 1 ); x264_cabac_encode_decision( cb, i_ctx_last + (b_8x8?last_coeff_flag_offset_8x8[i_last]:i_last), 1 ); @@ -788,20 +761,26 @@ static void ALWAYS_INLINE block_residual_write_cabac_internal( x264_t *h, x264_c } } -static void block_residual_write_cabac_8x8( x264_t *h, x264_cabac_t *cb, int i_idx, int16_t *l ) +static void block_residual_write_cabac_8x8( x264_t *h, x264_cabac_t *cb, int16_t *l ) { - block_residual_write_cabac_internal( h, cb, DCT_LUMA_8x8, i_idx, l, 0, 1 ); + block_residual_write_cabac_internal( h, cb, DCT_LUMA_8x8, l, 1 ); } -static void block_residual_write_cabac( x264_t *h, x264_cabac_t *cb, int i_ctxBlockCat, int i_idx, int16_t *l, int ctxidxinc ) +static void block_residual_write_cabac( x264_t *h, x264_cabac_t *cb, int i_ctxBlockCat, int16_t *l ) { - block_residual_write_cabac_internal( h, cb, i_ctxBlockCat, i_idx, l, ctxidxinc, 0 ); + block_residual_write_cabac_internal( h, cb, i_ctxBlockCat, l, 0 ); } #endif -#define block_residual_write_cabac( h, cb, i_ctxBlockCat, i_idx, l, b_intra ) \ +#define block_residual_write_cabac_cbf( h, cb, i_ctxBlockCat, i_idx, l, b_intra ) \ { \ int ctxidxinc = x264_cabac_mb_cbf_ctxidxinc( h, i_ctxBlockCat, i_idx, b_intra ); \ - block_residual_write_cabac( h, cb, i_ctxBlockCat, i_idx, l, ctxidxinc ); \ + if( h->mb.cache.non_zero_count[x264_scan8[i_idx]] )\ + {\ + x264_cabac_encode_decision( cb, ctxidxinc, 1 );\ + block_residual_write_cabac( h, cb, i_ctxBlockCat, l ); \ + }\ + else\ + x264_cabac_encode_decision( cb, ctxidxinc, 0 );\ } void x264_macroblock_write_cabac( x264_t *h, x264_cabac_t *cb ) @@ -855,7 +834,7 @@ void x264_macroblock_write_cabac( x264_t *h, x264_cabac_t *cb ) if( i_mb_type != I_16x16 ) { - int di = (i_mb_type == I_8x8) ? 4 : 1; + int di = h->mb.b_transform_8x8 ? 4 : 1; for( i = 0; i < 16; i += di ) { const int i_pred = x264_mb_predict_intra4x4_mode( h, i ); @@ -1011,35 +990,35 @@ void x264_macroblock_write_cabac( x264_t *h, x264_cabac_t *cb ) if( i_mb_type == I_16x16 ) { /* DC Luma */ - block_residual_write_cabac( h, cb, DCT_LUMA_DC, 24, h->dct.luma16x16_dc, 1 ); + block_residual_write_cabac_cbf( h, cb, DCT_LUMA_DC, 24, h->dct.luma16x16_dc, 1 ); /* AC Luma */ if( h->mb.i_cbp_luma != 0 ) for( i = 0; i < 16; i++ ) - block_residual_write_cabac( h, cb, DCT_LUMA_AC, i, h->dct.luma4x4[i]+1, 1 ); + block_residual_write_cabac_cbf( h, cb, DCT_LUMA_AC, i, h->dct.luma4x4[i]+1, 1 ); } else if( h->mb.b_transform_8x8 ) { for( i = 0; i < 4; i++ ) if( h->mb.i_cbp_luma & ( 1 << i ) ) - block_residual_write_cabac_8x8( h, cb, i, h->dct.luma8x8[i] ); + block_residual_write_cabac_8x8( h, cb, h->dct.luma8x8[i] ); } else { for( i = 0; i < 16; i++ ) if( h->mb.i_cbp_luma & ( 1 << ( i / 4 ) ) ) - block_residual_write_cabac( h, cb, DCT_LUMA_4x4, i, h->dct.luma4x4[i], b_intra ); + block_residual_write_cabac_cbf( h, cb, DCT_LUMA_4x4, i, h->dct.luma4x4[i], b_intra ); } - if( h->mb.i_cbp_chroma &0x03 ) /* Chroma DC residual present */ + if( h->mb.i_cbp_chroma&0x03 ) /* Chroma DC residual present */ { - block_residual_write_cabac( h, cb, DCT_CHROMA_DC, 25, h->dct.chroma_dc[0], b_intra ); - block_residual_write_cabac( h, cb, DCT_CHROMA_DC, 26, h->dct.chroma_dc[1], b_intra ); + block_residual_write_cabac_cbf( h, cb, DCT_CHROMA_DC, 25, h->dct.chroma_dc[0], b_intra ); + block_residual_write_cabac_cbf( h, cb, DCT_CHROMA_DC, 26, h->dct.chroma_dc[1], b_intra ); } if( h->mb.i_cbp_chroma&0x02 ) /* Chroma AC residual present */ { for( i = 16; i < 24; i++ ) - block_residual_write_cabac( h, cb, DCT_CHROMA_AC, i, h->dct.luma4x4[i]+1, b_intra ); + block_residual_write_cabac_cbf( h, cb, DCT_CHROMA_AC, i, h->dct.luma4x4[i]+1, b_intra ); } } @@ -1089,17 +1068,17 @@ static void x264_partition_size_cabac( x264_t *h, x264_cabac_t *cb, int i8, int if( h->mb.i_cbp_luma & (1 << i8) ) { if( h->mb.b_transform_8x8 ) - block_residual_write_cabac_8x8( h, cb, i8, h->dct.luma8x8[i8] ); + block_residual_write_cabac_8x8( h, cb, h->dct.luma8x8[i8] ); else { int i4; for( i4 = 0; i4 < 4; i4++ ) - block_residual_write_cabac( h, cb, DCT_LUMA_4x4, i4+i8*4, h->dct.luma4x4[i4+i8*4], 0 ); + block_residual_write_cabac_cbf( h, cb, DCT_LUMA_4x4, i4+i8*4, h->dct.luma4x4[i4+i8*4], 0 ); } } - block_residual_write_cabac( h, cb, DCT_CHROMA_AC, 16+i8, h->dct.luma4x4[16+i8]+1, 0 ); - block_residual_write_cabac( h, cb, DCT_CHROMA_AC, 20+i8, h->dct.luma4x4[20+i8]+1, 0 ); + block_residual_write_cabac_cbf( h, cb, DCT_CHROMA_AC, 16+i8, h->dct.luma4x4[16+i8]+1, 0 ); + block_residual_write_cabac_cbf( h, cb, DCT_CHROMA_AC, 20+i8, h->dct.luma4x4[20+i8]+1, 0 ); i8 += x264_pixel_size[i_pixel].h >> 3; } @@ -1108,7 +1087,7 @@ static void x264_partition_size_cabac( x264_t *h, x264_cabac_t *cb, int i8, int static void x264_subpartition_size_cabac( x264_t *h, x264_cabac_t *cb, int i4, int i_pixel ) { int b_8x4 = i_pixel == PIXEL_8x4; - block_residual_write_cabac( h, cb, DCT_LUMA_4x4, i4, h->dct.luma4x4[i4], 0 ); + block_residual_write_cabac_cbf( h, cb, DCT_LUMA_4x4, i4, h->dct.luma4x4[i4], 0 ); if( i_pixel == PIXEL_4x4 ) { x264_cabac_mb_mvd( h, cb, 0, i4, 1, 1 ); @@ -1116,7 +1095,7 @@ static void x264_subpartition_size_cabac( x264_t *h, x264_cabac_t *cb, int i4, i else { x264_cabac_mb_mvd( h, cb, 0, i4, 1+b_8x4, 2-b_8x4 ); - block_residual_write_cabac( h, cb, DCT_LUMA_4x4, i4+2-b_8x4, h->dct.luma4x4[i4+2-b_8x4], 0 ); + block_residual_write_cabac_cbf( h, cb, DCT_LUMA_4x4, i4+2-b_8x4, h->dct.luma4x4[i4+2-b_8x4], 0 ); } } @@ -1127,7 +1106,7 @@ static void x264_partition_i8x8_size_cabac( x264_t *h, x264_cabac_t *cb, int i8, x264_cabac_mb_intra4x4_pred_mode( cb, i_pred, i_mode ); x264_cabac_mb_cbp_luma( h, cb ); if( h->mb.i_cbp_luma & (1 << i8) ) - block_residual_write_cabac_8x8( h, cb, 4*i8, h->dct.luma8x8[i8] ); + block_residual_write_cabac_8x8( h, cb, h->dct.luma8x8[i8] ); } static void x264_partition_i4x4_size_cabac( x264_t *h, x264_cabac_t *cb, int i4, int i_mode ) @@ -1135,7 +1114,7 @@ static void x264_partition_i4x4_size_cabac( x264_t *h, x264_cabac_t *cb, int i4, const int i_pred = x264_mb_predict_intra4x4_mode( h, i4 ); i_mode = x264_mb_pred_mode4x4_fix( i_mode ); x264_cabac_mb_intra4x4_pred_mode( cb, i_pred, i_mode ); - block_residual_write_cabac( h, cb, DCT_LUMA_4x4, i4, h->dct.luma4x4[i4], 1 ); + block_residual_write_cabac_cbf( h, cb, DCT_LUMA_4x4, i4, h->dct.luma4x4[i4], 1 ); } static void x264_i8x8_chroma_size_cabac( x264_t *h, x264_cabac_t *cb ) @@ -1144,14 +1123,14 @@ static void x264_i8x8_chroma_size_cabac( x264_t *h, x264_cabac_t *cb ) x264_cabac_mb_cbp_chroma( h, cb ); if( h->mb.i_cbp_chroma > 0 ) { - block_residual_write_cabac( h, cb, DCT_CHROMA_DC, 25, h->dct.chroma_dc[0], 1 ); - block_residual_write_cabac( h, cb, DCT_CHROMA_DC, 26, h->dct.chroma_dc[1], 1 ); + block_residual_write_cabac_cbf( h, cb, DCT_CHROMA_DC, 25, h->dct.chroma_dc[0], 1 ); + block_residual_write_cabac_cbf( h, cb, DCT_CHROMA_DC, 26, h->dct.chroma_dc[1], 1 ); if( h->mb.i_cbp_chroma == 2 ) { int i; for( i = 16; i < 24; i++ ) - block_residual_write_cabac( h, cb, DCT_CHROMA_AC, i, h->dct.luma4x4[i]+1, 1 ); + block_residual_write_cabac_cbf( h, cb, DCT_CHROMA_AC, i, h->dct.luma4x4[i]+1, 1 ); } } } diff --git a/encoder/cavlc.c b/encoder/cavlc.c index d0176c19..9789af24 100644 --- a/encoder/cavlc.c +++ b/encoder/cavlc.c @@ -141,8 +141,7 @@ static int block_residual_write_cavlc( x264_t *h, bs_t *s, int i_ctxBlockCat, in bs_write_vlc( s, x264_coeff_token[nC][i_total*4+i_trailing-4] ); i_suffix_length = i_total > 10 && i_trailing < 3; - if( i_trailing > 0 || RDO_SKIP_BS ) - bs_write( s, i_trailing, i_sign ); + bs_write( s, i_trailing, i_sign ); if( i_trailing < i_total ) { @@ -182,8 +181,8 @@ static int block_residual_write_cavlc( x264_t *h, bs_t *s, int i_ctxBlockCat, in for( i = 0; i < i_total-1 && i_total_zero > 0; i++ ) { - int i_zl = X264_MIN( i_total_zero - 1, 6 ); - bs_write_vlc( s, x264_run_before[i_zl][runlevel.run[i]] ); + int i_zl = X264_MIN( i_total_zero, 7 ); + bs_write_vlc( s, x264_run_before[i_zl-1][runlevel.run[i]] ); i_total_zero -= runlevel.run[i]; } @@ -626,26 +625,24 @@ void x264_macroblock_write_cavlc( x264_t *h, bs_t *s ) *****************************************************************************/ static int x264_partition_size_cavlc( x264_t *h, int i8, int i_pixel ) { - bs_t s; const int i_mb_type = h->mb.i_type; int b_8x16 = h->mb.i_partition == D_8x16; int j; - - s.i_bits_encoded = 0; + h->out.bs.i_bits_encoded = 0; if( i_mb_type == P_8x8 ) - cavlc_mb8x8_mvd( h, &s, 0, i8 ); + cavlc_mb8x8_mvd( h, &h->out.bs, 0, i8 ); else if( i_mb_type == P_L0 ) - cavlc_mb_mvd( h, &s, 0, 4*i8, 4>>b_8x16 ); + cavlc_mb_mvd( h, &h->out.bs, 0, 4*i8, 4>>b_8x16 ); else if( i_mb_type > B_DIRECT && i_mb_type < B_8x8 ) { - if( x264_mb_type_list_table[ i_mb_type ][0][!!i8] ) cavlc_mb_mvd( h, &s, 0, 4*i8, 4>>b_8x16 ); - if( x264_mb_type_list_table[ i_mb_type ][1][!!i8] ) cavlc_mb_mvd( h, &s, 1, 4*i8, 4>>b_8x16 ); + if( x264_mb_type_list_table[ i_mb_type ][0][!!i8] ) cavlc_mb_mvd( h, &h->out.bs, 0, 4*i8, 4>>b_8x16 ); + if( x264_mb_type_list_table[ i_mb_type ][1][!!i8] ) cavlc_mb_mvd( h, &h->out.bs, 1, 4*i8, 4>>b_8x16 ); } else if( i_mb_type == B_8x8 ) { - cavlc_mb8x8_mvd( h, &s, 0, i8 ); - cavlc_mb8x8_mvd( h, &s, 1, i8 ); + cavlc_mb8x8_mvd( h, &h->out.bs, 0, i8 ); + cavlc_mb8x8_mvd( h, &h->out.bs, 1, i8 ); } else { @@ -655,29 +652,28 @@ static int x264_partition_size_cavlc( x264_t *h, int i8, int i_pixel ) for( j = (i_pixel < PIXEL_8x8); j >= 0; j-- ) { - x264_macroblock_luma_write_cavlc( h, &s, i8, i8 ); - block_residual_write_cavlc( h, &s, DCT_CHROMA_AC, 16+i8, h->dct.luma4x4[16+i8]+1 ); - block_residual_write_cavlc( h, &s, DCT_CHROMA_AC, 20+i8, h->dct.luma4x4[20+i8]+1 ); + x264_macroblock_luma_write_cavlc( h, &h->out.bs, i8, i8 ); + block_residual_write_cavlc( h, &h->out.bs, DCT_CHROMA_AC, 16+i8, h->dct.luma4x4[16+i8]+1 ); + block_residual_write_cavlc( h, &h->out.bs, DCT_CHROMA_AC, 20+i8, h->dct.luma4x4[20+i8]+1 ); i8 += x264_pixel_size[i_pixel].h >> 3; } - return s.i_bits_encoded; + return h->out.bs.i_bits_encoded; } static int x264_subpartition_size_cavlc( x264_t *h, int i4, int i_pixel ) { - bs_t s; int b_8x4 = i_pixel == PIXEL_8x4; - s.i_bits_encoded = 0; - cavlc_mb_mvd( h, &s, 0, i4, 1+b_8x4 ); - block_residual_write_cavlc( h, &s, DCT_LUMA_4x4, i4, h->dct.luma4x4[i4] ); + h->out.bs.i_bits_encoded = 0; + cavlc_mb_mvd( h, &h->out.bs, 0, i4, 1+b_8x4 ); + block_residual_write_cavlc( h, &h->out.bs, DCT_LUMA_4x4, i4, h->dct.luma4x4[i4] ); if( i_pixel != PIXEL_4x4 ) { i4 += 2-b_8x4; - block_residual_write_cavlc( h, &s, DCT_LUMA_4x4, i4, h->dct.luma4x4[i4] ); + block_residual_write_cavlc( h, &h->out.bs, DCT_LUMA_4x4, i4, h->dct.luma4x4[i4] ); } - return s.i_bits_encoded; + return h->out.bs.i_bits_encoded; } static int cavlc_intra4x4_pred_size( x264_t *h, int i4, int i_mode ) -- 2.40.0