From dfac34deef1ab22c011a1314a07dd23bbbeac289 Mon Sep 17 00:00:00 2001 From: Anton Mitrofanov Date: Sun, 1 Mar 2020 14:00:34 +0300 Subject: [PATCH] Fix undefined behavior: index out of bounds --- encoder/encoder.c | 6 ++++-- encoder/slicetype.c | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/encoder/encoder.c b/encoder/encoder.c index 6606641a..2cce74c3 100644 --- a/encoder/encoder.c +++ b/encoder/encoder.c @@ -3958,8 +3958,6 @@ static int encoder_frame_end( x264_t *h, x264_t *thread_current, for( int i = 0; i < X264_MBTYPE_MAX; i++ ) h->stat.i_mb_count[h->sh.i_type][i] += h->stat.frame.i_mb_count[i]; - for( int i = 0; i < X264_PARTTYPE_MAX; i++ ) - h->stat.i_mb_partition[h->sh.i_type][i] += h->stat.frame.i_mb_partition[i]; for( int i = 0; i < 2; i++ ) h->stat.i_mb_count_8x8dct[i] += h->stat.frame.i_mb_count_8x8dct[i]; for( int i = 0; i < 6; i++ ) @@ -3968,9 +3966,13 @@ static int encoder_frame_end( x264_t *h, x264_t *thread_current, for( int j = 0; j < 13; j++ ) h->stat.i_mb_pred_mode[i][j] += h->stat.frame.i_mb_pred_mode[i][j]; if( h->sh.i_type != SLICE_TYPE_I ) + { + for( int i = 0; i < X264_PARTTYPE_MAX; i++ ) + h->stat.i_mb_partition[h->sh.i_type][i] += h->stat.frame.i_mb_partition[i]; for( int i_list = 0; i_list < 2; i_list++ ) for( int i = 0; i < X264_REF_MAX*2; i++ ) h->stat.i_mb_count_ref[h->sh.i_type][i_list][i] += h->stat.frame.i_mb_count_ref[i_list][i]; + } for( int i = 0; i < 3; i++ ) h->stat.i_mb_field[i] += h->stat.frame.i_mb_field[i]; if( h->sh.i_type == SLICE_TYPE_P && h->param.analyse.i_weighted_pred >= X264_WEIGHTP_SIMPLE ) diff --git a/encoder/slicetype.c b/encoder/slicetype.c index 95b79221..de76795f 100644 --- a/encoder/slicetype.c +++ b/encoder/slicetype.c @@ -528,8 +528,8 @@ static void slicetype_mb_cost( x264_t *h, x264_mb_analysis_t *a, const int i_stride = fenc->i_stride_lowres; const int i_pel_offset = 8 * (i_mb_x + i_mb_y * i_stride); const int i_bipred_weight = h->param.analyse.b_weighted_bipred ? 64 - (dist_scale_factor>>2) : 32; - int16_t (*fenc_mvs[2])[2] = { &fenc->lowres_mvs[0][b-p0-1][i_mb_xy], &fenc->lowres_mvs[1][p1-b-1][i_mb_xy] }; - int (*fenc_costs[2]) = { &fenc->lowres_mv_costs[0][b-p0-1][i_mb_xy], &fenc->lowres_mv_costs[1][p1-b-1][i_mb_xy] }; + int16_t (*fenc_mvs[2])[2] = { b != p0 ? &fenc->lowres_mvs[0][b-p0-1][i_mb_xy] : NULL, b != p1 ? &fenc->lowres_mvs[1][p1-b-1][i_mb_xy] : NULL }; + int (*fenc_costs[2]) = { b != p0 ? &fenc->lowres_mv_costs[0][b-p0-1][i_mb_xy] : NULL, b != p1 ? &fenc->lowres_mv_costs[1][p1-b-1][i_mb_xy] : NULL }; int b_frame_score_mb = (i_mb_x > 0 && i_mb_x < h->mb.i_mb_width - 1 && i_mb_y > 0 && i_mb_y < h->mb.i_mb_height - 1) || h->mb.i_mb_width <= 2 || h->mb.i_mb_height <= 2; @@ -1054,7 +1054,7 @@ static void macroblock_tree_propagate( x264_t *h, x264_frame_t **frames, float a uint16_t *ref_costs[2] = {frames[p0]->i_propagate_cost,frames[p1]->i_propagate_cost}; int dist_scale_factor = ( ((b-p0) << 8) + ((p1-p0) >> 1) ) / (p1-p0); int i_bipred_weight = h->param.analyse.b_weighted_bipred ? 64 - (dist_scale_factor>>2) : 32; - int16_t (*mvs[2])[2] = { frames[b]->lowres_mvs[0][b-p0-1], frames[b]->lowres_mvs[1][p1-b-1] }; + int16_t (*mvs[2])[2] = { b != p0 ? frames[b]->lowres_mvs[0][b-p0-1] : NULL, b != p1 ? frames[b]->lowres_mvs[1][p1-b-1] : NULL }; int bipred_weights[2] = {i_bipred_weight, 64 - i_bipred_weight}; int16_t *buf = h->scratch_buffer; uint16_t *propagate_cost = frames[b]->i_propagate_cost; -- 2.40.0