From 6eab44d4295801c28184ec13f03f9727c60129cc Mon Sep 17 00:00:00 2001 From: Steven Walters Date: Fri, 7 Aug 2009 16:18:01 -0700 Subject: [PATCH] Gracefully terminate in the case of a malloc failure Fuzz tests show that all mallocs appear to be checked correctly now. --- common/common.c | 64 +++++++++--------------- common/common.h | 13 ++--- common/frame.c | 20 ++++---- common/osdep.h | 18 ++++--- common/set.c | 11 ++-- common/visualize.c | 7 ++- common/visualize.h | 2 +- encoder/analyse.c | 22 +++++--- encoder/analyse.h | 3 +- encoder/encoder.c | 114 +++++++++++++++++++++++++----------------- encoder/ratecontrol.c | 69 +++++++++++++++---------- encoder/set.c | 12 ++++- encoder/set.h | 2 +- encoder/slicetype.c | 12 ++++- matroska.c | 5 +- muxers.c | 28 +++++++++-- tools/checkasm.c | 5 ++ x264.c | 28 ++++++++--- x264.h | 7 +-- 19 files changed, 271 insertions(+), 171 deletions(-) diff --git a/common/common.c b/common/common.c index 371ed1ed..ce7a6131 100644 --- a/common/common.c +++ b/common/common.c @@ -606,11 +606,14 @@ int x264_param_parse( x264_param_t *p, const char *name, const char *value ) ****************************************************************************/ void x264_log( x264_t *h, int i_level, const char *psz_fmt, ... ) { - if( i_level <= h->param.i_log_level ) + if( !h || i_level <= h->param.i_log_level ) { va_list arg; va_start( arg, psz_fmt ); - h->param.pf_log( h->param.p_log_private, i_level, psz_fmt, arg ); + if( !h ) + x264_log_default( NULL, i_level, psz_fmt, arg ); + else + h->param.pf_log( h->param.p_log_private, i_level, psz_fmt, arg ); va_end( arg ); } } @@ -643,18 +646,21 @@ static void x264_log_default( void *p_unused, int i_level, const char *psz_fmt, /**************************************************************************** * x264_picture_alloc: ****************************************************************************/ -void x264_picture_alloc( x264_picture_t *pic, int i_csp, int i_width, int i_height ) +int x264_picture_alloc( x264_picture_t *pic, int i_csp, int i_width, int i_height ) { pic->i_type = X264_TYPE_AUTO; pic->i_qpplus1 = 0; pic->img.i_csp = i_csp; pic->img.i_plane = 3; pic->img.plane[0] = x264_malloc( 3 * i_width * i_height / 2 ); + if( !pic->img.plane[0] ) + return -1; pic->img.plane[1] = pic->img.plane[0] + i_width * i_height; pic->img.plane[2] = pic->img.plane[1] + i_width * i_height / 4; pic->img.i_stride[0] = i_width; pic->img.i_stride[1] = i_width / 2; pic->img.i_stride[2] = i_width / 2; + return 0; } /**************************************************************************** @@ -717,22 +723,25 @@ int x264_nal_encode( void *p_data, int *pi_data, int b_annexeb, x264_nal_t *nal ****************************************************************************/ void *x264_malloc( int i_size ) { + uint8_t *align_buf = NULL; #ifdef SYS_MACOSX /* Mac OS X always returns 16 bytes aligned memory */ - return malloc( i_size ); + align_buf = malloc( i_size ); #elif defined( HAVE_MALLOC_H ) - return memalign( 16, i_size ); + align_buf = memalign( 16, i_size ); #else - uint8_t * buf; - uint8_t * align_buf; - buf = (uint8_t *) malloc( i_size + 15 + sizeof( void ** ) + - sizeof( int ) ); - align_buf = buf + 15 + sizeof( void ** ) + sizeof( int ); - align_buf -= (intptr_t) align_buf & 15; - *( (void **) ( align_buf - sizeof( void ** ) ) ) = buf; - *( (int *) ( align_buf - sizeof( void ** ) - sizeof( int ) ) ) = i_size; - return align_buf; + uint8_t *buf = malloc( i_size + 15 + sizeof(void **) + sizeof(int) ); + if( buf ) + { + align_buf = buf + 15 + sizeof(void **) + sizeof(int); + align_buf -= (intptr_t) align_buf & 15; + *( (void **) ( align_buf - sizeof(void **) ) ) = buf; + *( (int *) ( align_buf - sizeof(void **) - sizeof(int) ) ) = i_size; + } #endif + if( !align_buf ) + x264_log( NULL, X264_LOG_ERROR, "malloc of size %d failed\n", i_size ); + return align_buf; } /**************************************************************************** @@ -750,31 +759,6 @@ void x264_free( void *p ) } } -/**************************************************************************** - * x264_realloc: - ****************************************************************************/ -void *x264_realloc( void *p, int i_size ) -{ -#ifdef HAVE_MALLOC_H - return realloc( p, i_size ); -#else - int i_old_size = 0; - uint8_t * p_new; - if( p ) - { - i_old_size = *( (int*) ( (uint8_t*) p - sizeof( void ** ) - - sizeof( int ) ) ); - } - p_new = x264_malloc( i_size ); - if( i_old_size > 0 && i_size > 0 ) - { - memcpy( p_new, p, ( i_old_size < i_size ) ? i_old_size : i_size ); - } - x264_free( p ); - return p_new; -#endif -} - /**************************************************************************** * x264_reduce_fraction: ****************************************************************************/ @@ -838,6 +822,8 @@ char *x264_param2string( x264_param_t *p, int b_res ) if( p->rc.psz_zones ) len += strlen(p->rc.psz_zones); buf = s = x264_malloc( len ); + if( !buf ) + return NULL; if( b_res ) { diff --git a/common/common.h b/common/common.h index 30163abc..166ced26 100644 --- a/common/common.h +++ b/common/common.h @@ -37,14 +37,16 @@ #define FIX8(f) ((int)(f*(1<<8)+.5)) #define CHECKED_MALLOC( var, size )\ -{\ +do {\ var = x264_malloc( size );\ if( !var )\ - {\ - x264_log( h, X264_LOG_ERROR, "malloc failed\n" );\ goto fail;\ - }\ -} +} while( 0 ) +#define CHECKED_MALLOCZERO( var, size )\ +do {\ + CHECKED_MALLOC( var, size );\ + memset( var, 0, size );\ +} while( 0 ) #define X264_BFRAME_MAX 16 #define X264_THREAD_MAX 128 @@ -83,7 +85,6 @@ /* x264_malloc : will do or emulate a memalign * you have to use x264_free for buffers allocated with x264_malloc */ void *x264_malloc( int ); -void *x264_realloc( void *p, int i_size ); void x264_free( void * ); /* x264_slurp_file: malloc space for the whole file and read it */ diff --git a/common/frame.c b/common/frame.c index c6d7693a..e81f6239 100644 --- a/common/frame.c +++ b/common/frame.c @@ -28,7 +28,7 @@ x264_frame_t *x264_frame_new( x264_t *h ) { - x264_frame_t *frame = x264_malloc( sizeof(x264_frame_t) ); + x264_frame_t *frame; int i, j; int i_mb_count = h->mb.i_mb_count; @@ -38,9 +38,7 @@ x264_frame_t *x264_frame_new( x264_t *h ) int chroma_plane_size; int align = h->param.cpu&X264_CPU_CACHELINE_64 ? 64 : h->param.cpu&X264_CPU_CACHELINE_32 ? 32 : 16; - if( !frame ) return NULL; - - memset( frame, 0, sizeof(x264_frame_t) ); + CHECKED_MALLOCZERO( frame, sizeof(x264_frame_t) ); /* allocate frame data (+64 for extra data for me) */ i_width = ALIGN( h->param.i_width, 16 ); @@ -92,8 +90,7 @@ x264_frame_t *x264_frame_new( x264_t *h ) for( j = 0; j <= !!h->param.i_bframe; j++ ) for( i = 0; i <= h->param.i_bframe; i++ ) { - CHECKED_MALLOC( frame->lowres_mvs[j][i], 2*h->mb.i_mb_count*sizeof(int16_t) ); - memset( frame->lowres_mvs[j][i], 0, 2*h->mb.i_mb_count*sizeof(int16_t) ); + CHECKED_MALLOCZERO( frame->lowres_mvs[j][i], 2*h->mb.i_mb_count*sizeof(int16_t) ); CHECKED_MALLOC( frame->lowres_mv_costs[j][i], h->mb.i_mb_count*sizeof(int) ); } CHECKED_MALLOC( frame->i_intra_cost, i_mb_count * sizeof(uint16_t) ); @@ -149,13 +146,15 @@ x264_frame_t *x264_frame_new( x264_t *h ) CHECKED_MALLOC( frame->i_inv_qscale_factor, h->mb.i_mb_count * sizeof(uint16_t) ); } - x264_pthread_mutex_init( &frame->mutex, NULL ); - x264_pthread_cond_init( &frame->cv, NULL ); + if( x264_pthread_mutex_init( &frame->mutex, NULL ) ) + goto fail; + if( x264_pthread_cond_init( &frame->cv, NULL ) ) + goto fail; return frame; fail: - x264_frame_delete( frame ); + x264_free( frame ); return NULL; } @@ -956,7 +955,8 @@ x264_frame_t *x264_frame_pop_unused( x264_t *h ) frame = x264_frame_pop( h->frames.unused ); else frame = x264_frame_new( h ); - assert( frame->i_reference_count == 0 ); + if( !frame ) + return NULL; frame->i_reference_count = 1; frame->b_intra_calculated = 0; return frame; diff --git a/common/osdep.h b/common/osdep.h index 20951987..104ac247 100644 --- a/common/osdep.h +++ b/common/osdep.h @@ -90,10 +90,16 @@ #if defined(SYS_BEOS) #include #define x264_pthread_t thread_id -#define x264_pthread_create(t,u,f,d) { *(t)=spawn_thread(f,"",10,d); \ - resume_thread(*(t)); } +static inline int x264_pthread_create( x264_pthread_t *t, void *a, void *(*f)(void *), void *d ) +{ + *t = spawn_thread( f, "", 10, d ); + if( *t < B_NO_ERROR ) + return -1; + resume_thread( *t ); + return 0; +} #define x264_pthread_join(t,s) { long tmp; \ - wait_for_thread(t,(s)?(long*)(s):&tmp); } + wait_for_thread(t,(s)?(long*)(*(s)):&tmp); } #ifndef usleep #define usleep(t) snooze(t) #endif @@ -105,7 +111,7 @@ #else #define x264_pthread_t int -#define x264_pthread_create(t,u,f,d) +#define x264_pthread_create(t,u,f,d) 0 #define x264_pthread_join(t,s) #endif //SYS_* @@ -125,12 +131,12 @@ #define x264_pthread_cond_wait pthread_cond_wait #else #define x264_pthread_mutex_t int -#define x264_pthread_mutex_init(m,f) +#define x264_pthread_mutex_init(m,f) 0 #define x264_pthread_mutex_destroy(m) #define x264_pthread_mutex_lock(m) #define x264_pthread_mutex_unlock(m) #define x264_pthread_cond_t int -#define x264_pthread_cond_init(c,f) +#define x264_pthread_cond_init(c,f) 0 #define x264_pthread_cond_destroy(c) #define x264_pthread_cond_broadcast(c) #define x264_pthread_cond_wait(c,m) diff --git a/common/set.c b/common/set.c index 6c7ddc46..7d4d23fc 100644 --- a/common/set.c +++ b/common/set.c @@ -93,9 +93,9 @@ int x264_cqm_init( x264_t *h ) } else { - h-> quant4_mf[i] = x264_malloc(52*size*sizeof(uint16_t) ); - h->dequant4_mf[i] = x264_malloc( 6*size*sizeof(int) ); - h->unquant4_mf[i] = x264_malloc(52*size*sizeof(int) ); + CHECKED_MALLOC( h-> quant4_mf[i], 52*size*sizeof(uint16_t) ); + CHECKED_MALLOC( h->dequant4_mf[i], 6*size*sizeof(int) ); + CHECKED_MALLOC( h->unquant4_mf[i], 52*size*sizeof(int) ); } for( j = (i<4 ? 0 : 4); j < i; j++ ) @@ -105,7 +105,7 @@ int x264_cqm_init( x264_t *h ) if( j < i ) h->quant4_bias[i] = h->quant4_bias[j]; else - h->quant4_bias[i] = x264_malloc(52*size*sizeof(uint16_t) ); + CHECKED_MALLOC( h->quant4_bias[i], 52*size*sizeof(uint16_t) ); } for( q = 0; q < 6; q++ ) @@ -171,6 +171,9 @@ int x264_cqm_init( x264_t *h ) return -1; } return 0; +fail: + x264_cqm_delete( h ); + return -1; } void x264_cqm_delete( x264_t *h ) diff --git a/common/visualize.c b/common/visualize.c index f7100f00..1d3dd845 100644 --- a/common/visualize.c +++ b/common/visualize.c @@ -94,10 +94,13 @@ static void mv(int x0, int y0, int16_t dmv[2], int ref, int zoom, char *col) /* }}} */ /* {{{ [fold] void x264_visualize_init( x264_t *h ) */ -void x264_visualize_init( x264_t *h ) +int x264_visualize_init( x264_t *h ) { int mb = h->sps->i_mb_width * h->sps->i_mb_height; - h->visualize = x264_malloc(mb * sizeof(visualize_t)); + CHECKED_MALLOC( h->visualize, mb * sizeof(visualize_t) ); + return 0; +fail: + return -1; } /* }}} */ /* {{{ [fold] void x264_visualize_mb( x264_t *h ) */ diff --git a/common/visualize.h b/common/visualize.h index b611f6cb..f9753d71 100644 --- a/common/visualize.h +++ b/common/visualize.h @@ -23,7 +23,7 @@ #include "common/common.h" -void x264_visualize_init( x264_t *h ); +int x264_visualize_init( x264_t *h ); void x264_visualize_mb( x264_t *h ); void x264_visualize_show( x264_t *h ); void x264_visualize_close( x264_t *h ); diff --git a/encoder/analyse.c b/encoder/analyse.c index 7cc00c59..95569d13 100644 --- a/encoder/analyse.c +++ b/encoder/analyse.c @@ -213,7 +213,7 @@ uint16_t *x264_cost_mv_fpel[92][4]; uint16_t x264_cost_ref[92][3][33]; /* initialize an array of lambda*nbits for all possible mvs */ -static void x264_mb_analyse_load_costs( x264_t *h, x264_mb_analysis_t *a ) +static int x264_mb_analyse_load_costs( x264_t *h, x264_mb_analysis_t *a ) { static int16_t *p_cost_mv[92]; int i, j; @@ -223,7 +223,7 @@ static void x264_mb_analyse_load_costs( x264_t *h, x264_mb_analysis_t *a ) x264_emms(); /* could be faster, but isn't called many times */ /* factor of 4 from qpel, 2 from sign, and 2 because mv can be opposite from mvp */ - p_cost_mv[a->i_lambda] = x264_malloc( (4*4*2048 + 1) * sizeof(int16_t) ); + CHECKED_MALLOC( p_cost_mv[a->i_lambda], (4*4*2048 + 1) * sizeof(int16_t) ); p_cost_mv[a->i_lambda] += 2*4*2048; for( i = 0; i <= 2*4*2048; i++ ) { @@ -243,12 +243,15 @@ static void x264_mb_analyse_load_costs( x264_t *h, x264_mb_analysis_t *a ) { for( j=0; j<4; j++ ) { - x264_cost_mv_fpel[a->i_lambda][j] = x264_malloc( (4*2048 + 1) * sizeof(int16_t) ); + CHECKED_MALLOC( x264_cost_mv_fpel[a->i_lambda][j], (4*2048 + 1) * sizeof(int16_t) ); x264_cost_mv_fpel[a->i_lambda][j] += 2*2048; for( i = -2*2048; i < 2*2048; i++ ) x264_cost_mv_fpel[a->i_lambda][j][i] = p_cost_mv[a->i_lambda][i*4+j]; } } + return 0; +fail: + return -1; } static void x264_mb_analyse_init( x264_t *h, x264_mb_analysis_t *a, int i_qp ) @@ -2253,7 +2256,7 @@ static inline void x264_mb_analyse_qp_rd( x264_t *h, x264_mb_analysis_t *a ) /***************************************************************************** * x264_macroblock_analyse: *****************************************************************************/ -void x264_macroblock_analyse( x264_t *h ) +int x264_macroblock_analyse( x264_t *h ) { x264_mb_analysis_t analysis; int i_cost = COST_MAX; @@ -2328,12 +2331,13 @@ void x264_macroblock_analyse( x264_t *h ) int i_thresh16x8; int i_satd_inter, i_satd_intra; - x264_mb_analyse_load_costs( h, &analysis ); + if( x264_mb_analyse_load_costs( h, &analysis ) ) + return -1; x264_mb_analyse_inter_p16x16( h, &analysis ); if( h->mb.i_type == P_SKIP ) - return; + return 0; if( flags & X264_ANALYSE_PSUB16x16 ) { @@ -2620,7 +2624,8 @@ void x264_macroblock_analyse( x264_t *h ) int i_satd_inter = 0; // shut up uninitialized warning h->mb.b_skip_mc = 0; - x264_mb_analyse_load_costs( h, &analysis ); + if( x264_mb_analyse_load_costs( h, &analysis ) ) + return -1; /* select best inter mode */ /* direct must be first */ @@ -2646,7 +2651,7 @@ void x264_macroblock_analyse( x264_t *h ) { h->mb.i_type = B_SKIP; x264_analyse_update_cache( h, &analysis ); - return; + return 0; } } @@ -2871,6 +2876,7 @@ void x264_macroblock_analyse( x264_t *h ) x264_psy_trellis_init( h, 0 ); if( h->mb.b_trellis == 1 || h->mb.b_noise_reduction ) h->mb.i_skip_intra = 0; + return 0; } /*-------------------- Update MB from the analysis ----------------------*/ diff --git a/encoder/analyse.h b/encoder/analyse.h index b8c828f6..a2a04a55 100644 --- a/encoder/analyse.h +++ b/encoder/analyse.h @@ -24,7 +24,8 @@ #ifndef X264_ANALYSE_H #define X264_ANALYSE_H -void x264_macroblock_analyse( x264_t *h ); +int x264_macroblock_analyse( x264_t *h ); void x264_slicetype_decide( x264_t *h ); +int x264_lowres_context_alloc( x264_t *h ); #endif diff --git a/encoder/encoder.c b/encoder/encoder.c index 26d3e3ca..76e8b472 100644 --- a/encoder/encoder.c +++ b/encoder/encoder.c @@ -43,8 +43,8 @@ #define bs_write_ue bs_write_ue_big static int x264_encoder_frame_end( x264_t *h, x264_t *thread_current, - x264_nal_t **pp_nal, int *pi_nal, - x264_picture_t *pic_out ); + x264_nal_t **pp_nal, int *pi_nal, + x264_picture_t *pic_out ); /**************************************************************************** * @@ -295,17 +295,18 @@ static void x264_slice_header_write( bs_t *s, x264_slice_header_t *sh, int i_nal /* If we are within a reasonable distance of the end of the memory allocated for the bitstream, */ /* reallocate, adding an arbitrary amount of space (100 kilobytes). */ -static void x264_bitstream_check_buffer( x264_t *h ) +static int x264_bitstream_check_buffer( x264_t *h ) { + uint8_t *bs_bak = h->out.p_bitstream; if( ( h->param.b_cabac && (h->cabac.p_end - h->cabac.p < 2500) ) || ( h->out.bs.p_end - h->out.bs.p < 2500 ) ) { - uint8_t *bs_bak = h->out.p_bitstream; intptr_t delta; int i; h->out.i_bitstream += 100000; - h->out.p_bitstream = x264_realloc( h->out.p_bitstream, h->out.i_bitstream ); + CHECKED_MALLOC( h->out.p_bitstream, h->out.i_bitstream ); + h->mc.memcpy_aligned( h->out.p_bitstream, bs_bak, (h->out.i_bitstream - 100000) & ~15 ); delta = h->out.p_bitstream - bs_bak; h->out.bs.p_start += delta; @@ -318,7 +319,12 @@ static void x264_bitstream_check_buffer( x264_t *h ) for( i = 0; i <= h->out.i_nal; i++ ) h->out.nal[i].p_payload += delta; + x264_free( bs_bak ); } + return 0; +fail: + x264_free( bs_bak ); + return -1; } /**************************************************************************** @@ -666,27 +672,21 @@ static void mbcmp_init( x264_t *h ) ****************************************************************************/ x264_t *x264_encoder_open ( x264_param_t *param ) { - x264_t *h = x264_malloc( sizeof( x264_t ) ); + x264_t *h; char buf[1000], *p; int i; - memset( h, 0, sizeof( x264_t ) ); + CHECKED_MALLOCZERO( h, sizeof(x264_t) ); /* Create a copy of param */ - memcpy( &h->param, param, sizeof( x264_param_t ) ); + memcpy( &h->param, param, sizeof(x264_param_t) ); if( x264_validate_parameters( h ) < 0 ) - { - x264_free( h ); - return NULL; - } + goto fail; if( h->param.psz_cqm_file ) if( x264_cqm_parse_file( h, h->param.psz_cqm_file ) < 0 ) - { - x264_free( h ); - return NULL; - } + goto fail; if( h->param.rc.psz_stat_out ) h->param.rc.psz_stat_out = strdup( h->param.rc.psz_stat_out ); @@ -737,10 +737,7 @@ x264_t *x264_encoder_open ( x264_param_t *param ) x264_validate_levels( h, 1 ); if( x264_cqm_init( h ) < 0 ) - { - x264_free( h ); - return NULL; - } + goto fail; h->mb.i_mb_count = h->sps->i_mb_width * h->sps->i_mb_height; @@ -822,20 +819,25 @@ x264_t *x264_encoder_open ( x264_param_t *param ) h->thread[0] = h; h->i_thread_num = 0; for( i = 1; i < h->param.i_threads; i++ ) - h->thread[i] = x264_malloc( sizeof(x264_t) ); + CHECKED_MALLOC( h->thread[i], sizeof(x264_t) ); for( i = 0; i < h->param.i_threads; i++ ) { if( i > 0 ) *h->thread[i] = *h; h->thread[i]->fdec = x264_frame_pop_unused( h ); - h->thread[i]->out.p_bitstream = x264_malloc( h->out.i_bitstream ); + if( !h->thread[i]->fdec ) + goto fail; + CHECKED_MALLOC( h->thread[i]->out.p_bitstream, h->out.i_bitstream ); if( x264_macroblock_cache_init( h->thread[i] ) < 0 ) - return NULL; + goto fail; } if( x264_ratecontrol_new( h ) < 0 ) - return NULL; + goto fail; + + if( x264_lowres_context_alloc( h ) ) + goto fail; if( h->param.psz_dump_yuv ) { @@ -846,8 +848,7 @@ x264_t *x264_encoder_open ( x264_param_t *param ) else { x264_log( h, X264_LOG_ERROR, "can't write to fdec.yuv\n" ); - x264_free( h ); - return NULL; + goto fail; } } @@ -858,6 +859,9 @@ x264_t *x264_encoder_open ( x264_param_t *param ) "High 4:4:4 Predictive", h->sps->i_level_idc/10, h->sps->i_level_idc%10 ); return h; +fail: + x264_free( h ); + return NULL; } /**************************************************************************** @@ -938,7 +942,8 @@ int x264_encoder_headers( x264_t *h, x264_nal_t **pp_nal, int *pi_nal ) { /* identify ourself */ x264_nal_start( h, NAL_SEI, NAL_PRIORITY_DISPOSABLE ); - x264_sei_version_write( h, &h->out.bs ); + if( x264_sei_version_write( h, &h->out.bs ) ) + return -1; x264_nal_end( h ); /* generate sequence parameters */ @@ -1106,7 +1111,7 @@ static void x264_fdec_filter_row( x264_t *h, int mb_y ) } } -static inline void x264_reference_update( x264_t *h ) +static inline int x264_reference_update( x264_t *h ) { int i; @@ -1119,8 +1124,10 @@ static inline void x264_reference_update( x264_t *h ) { x264_frame_push_unused( h, h->fdec ); h->fdec = x264_frame_pop_unused( h ); + if( !h->fdec ) + return -1; } - return; + return 0; } /* move lowres copy of the image to the ref frame */ @@ -1139,6 +1146,9 @@ static inline void x264_reference_update( x264_t *h ) if( h->frames.reference[h->frames.i_max_dpb] ) x264_frame_push_unused( h, x264_frame_shift( h->frames.reference ) ); h->fdec = x264_frame_pop_unused( h ); + if( !h->fdec ) + return -1; + return 0; } static inline void x264_reference_reset( x264_t *h ) @@ -1188,7 +1198,7 @@ static inline void x264_slice_init( x264_t *h, int i_nal_type, int i_global_qp ) x264_macroblock_slice_init( h ); } -static void x264_slice_write( x264_t *h ) +static int x264_slice_write( x264_t *h ) { int i_skip; int mb_xy, i_mb_x, i_mb_y; @@ -1232,12 +1242,14 @@ static void x264_slice_write( x264_t *h ) * Slice I: choose I_4x4 or I_16x16 mode * Slice P: choose between using P mode or intra (4x4 or 16x16) * */ - x264_macroblock_analyse( h ); + if( x264_macroblock_analyse( h ) ) + return -1; /* encode this macroblock -> be careful it can change the mb type to P_SKIP if needed */ x264_macroblock_encode( h ); - x264_bitstream_check_buffer( h ); + if( x264_bitstream_check_buffer( h ) ) + return -1; if( h->param.b_cabac ) { @@ -1347,6 +1359,7 @@ static void x264_slice_write( x264_t *h ) + NALU_OVERHEAD * 8 - h->stat.frame.i_tex_bits - h->stat.frame.i_mv_bits; + return 0; } static void x264_thread_sync_context( x264_t *dst, x264_t *src ) @@ -1375,7 +1388,7 @@ static void x264_thread_sync_stat( x264_t *dst, x264_t *src ) memcpy( &dst->stat.i_slice_count, &src->stat.i_slice_count, sizeof(dst->stat) - sizeof(dst->stat.frame) ); } -static int x264_slices_write( x264_t *h ) +static void *x264_slices_write( x264_t *h ) { int i_frame_size; @@ -1387,10 +1400,12 @@ static int x264_slices_write( x264_t *h ) #if VISUALIZE if( h->param.b_visualize ) - x264_visualize_init( h ); + if( x264_visualize_init( h ) ) + return (void *)-1; #endif - x264_stack_align( x264_slice_write, h ); + if( x264_stack_align( x264_slice_write, h ) ) + return (void *)-1; i_frame_size = h->out.nal[h->out.i_nal-1].i_payload; #if VISUALIZE @@ -1402,7 +1417,7 @@ static int x264_slices_write( x264_t *h ) #endif h->out.i_frame_size = i_frame_size; - return 0; + return (void *)0; } /**************************************************************************** @@ -1448,7 +1463,8 @@ int x264_encoder_encode( x264_t *h, } // ok to call this before encoding any frames, since the initial values of fdec have b_kept_as_ref=0 - x264_reference_update( h ); + if( x264_reference_update( h ) ) + return -1; h->fdec->i_lines_completed = -1; /* no data out */ @@ -1460,6 +1476,8 @@ int x264_encoder_encode( x264_t *h, { /* 1: Copy the picture to a frame and move it to a buffer */ x264_frame_t *fenc = x264_frame_pop_unused( h ); + if( !fenc ) + return -1; if( x264_frame_copy_picture( h, fenc, pic_in ) < 0 ) return -1; @@ -1635,7 +1653,8 @@ int x264_encoder_encode( x264_t *h, { /* identify ourself */ x264_nal_start( h, NAL_SEI, NAL_PRIORITY_DISPOSABLE ); - x264_sei_version_write( h, &h->out.bs ); + if( x264_sei_version_write( h, &h->out.bs ) ) + return -1; x264_nal_end( h ); } @@ -1653,27 +1672,30 @@ int x264_encoder_encode( x264_t *h, /* Write frame */ if( h->param.i_threads > 1 ) { - x264_pthread_create( &h->thread_handle, NULL, (void*)x264_slices_write, h ); + if( x264_pthread_create( &h->thread_handle, NULL, (void*)x264_slices_write, h ) ) + return -1; h->b_thread_active = 1; } else - x264_slices_write( h ); + if( (intptr_t)x264_slices_write( h ) ) + return -1; - if( x264_encoder_frame_end( thread_oldest, thread_current, pp_nal, pi_nal, pic_out ) < 0 ) - return -1; - return 0; + return x264_encoder_frame_end( thread_oldest, thread_current, pp_nal, pi_nal, pic_out ); } static int x264_encoder_frame_end( x264_t *h, x264_t *thread_current, - x264_nal_t **pp_nal, int *pi_nal, - x264_picture_t *pic_out ) + x264_nal_t **pp_nal, int *pi_nal, + x264_picture_t *pic_out ) { int i, i_list; char psz_message[80]; if( h->b_thread_active ) { - x264_pthread_join( h->thread_handle, NULL ); + void *ret = NULL; + x264_pthread_join( h->thread_handle, &ret ); + if( (intptr_t)ret ) + return (intptr_t)ret; h->b_thread_active = 0; } if( !h->out.i_nal ) diff --git a/encoder/ratecontrol.c b/encoder/ratecontrol.c index 9d913587..1bc76a89 100644 --- a/encoder/ratecontrol.c +++ b/encoder/ratecontrol.c @@ -302,6 +302,8 @@ fail: static char *x264_strcat_filename( char *input, char *suffix ) { char *output = x264_malloc( strlen( input ) + strlen( suffix ) + 1 ); + if( !output ) + return NULL; strcpy( output, input ); strcat( output, suffix ); return output; @@ -314,8 +316,8 @@ int x264_ratecontrol_new( x264_t *h ) x264_emms(); - rc = h->rc = x264_malloc( h->param.i_threads * sizeof(x264_ratecontrol_t) ); - memset( rc, 0, h->param.i_threads * sizeof(x264_ratecontrol_t) ); + CHECKED_MALLOCZERO( h->rc, h->param.i_threads * sizeof(x264_ratecontrol_t) ); + rc = h->rc; rc->b_abr = h->param.rc.i_rc_method != X264_RC_CQP && !h->param.rc.b_stat_read; rc->b_2pass = h->param.rc.i_rc_method == X264_RC_ABR && h->param.rc.b_stat_read; @@ -426,8 +428,8 @@ int x264_ratecontrol_new( x264_t *h ) rc->lstep = pow( 2, h->param.rc.i_qp_step / 6.0 ); rc->last_qscale = qp2qscale(26); - rc->pred = x264_malloc( 5*sizeof(predictor_t) ); - rc->pred_b_from_p = x264_malloc( sizeof(predictor_t) ); + CHECKED_MALLOC( rc->pred, 5*sizeof(predictor_t) ); + CHECKED_MALLOC( rc->pred_b_from_p, sizeof(predictor_t) ); for( i = 0; i < 5; i++ ) { rc->last_qscale_for[i] = qp2qscale( ABR_INIT_QP ); @@ -466,6 +468,8 @@ int x264_ratecontrol_new( x264_t *h ) if( h->param.rc.b_mb_tree ) { char *mbtree_stats_in = x264_strcat_filename( h->param.rc.psz_stat_in, ".mbtree" ); + if( !mbtree_stats_in ) + return -1; rc->p_mbtree_stat_file_in = fopen( mbtree_stats_in, "rb" ); x264_free( mbtree_stats_in ); if( !rc->p_mbtree_stat_file_in ) @@ -548,8 +552,7 @@ int x264_ratecontrol_new( x264_t *h ) return -1; } - rc->entry = (ratecontrol_entry_t*) x264_malloc(rc->num_entries * sizeof(ratecontrol_entry_t)); - memset(rc->entry, 0, rc->num_entries * sizeof(ratecontrol_entry_t)); + CHECKED_MALLOCZERO( rc->entry, rc->num_entries * sizeof(ratecontrol_entry_t) ); /* init all to skipped p frames */ for(i=0; inum_entries; i++) @@ -626,6 +629,8 @@ int x264_ratecontrol_new( x264_t *h ) { char *p; rc->psz_stat_file_tmpname = x264_strcat_filename( h->param.rc.psz_stat_out, ".temp" ); + if( !rc->psz_stat_file_tmpname ) + return -1; rc->p_stat_file_out = fopen( rc->psz_stat_file_tmpname, "wb" ); if( rc->p_stat_file_out == NULL ) @@ -635,12 +640,15 @@ int x264_ratecontrol_new( x264_t *h ) } p = x264_param2string( &h->param, 1 ); - fprintf( rc->p_stat_file_out, "#options: %s\n", p ); + if( p ) + fprintf( rc->p_stat_file_out, "#options: %s\n", p ); x264_free( p ); if( h->param.rc.b_mb_tree && !h->param.rc.b_stat_read ) { rc->psz_mbtree_stat_file_tmpname = x264_strcat_filename( h->param.rc.psz_stat_out, ".mbtree.temp" ); rc->psz_mbtree_stat_file_name = x264_strcat_filename( h->param.rc.psz_stat_out, ".mbtree" ); + if( !rc->psz_mbtree_stat_file_tmpname || !rc->psz_mbtree_stat_file_name ) + return -1; rc->p_mbtree_stat_file_out = fopen( rc->psz_mbtree_stat_file_tmpname, "wb" ); if( rc->p_mbtree_stat_file_out == NULL ) @@ -652,11 +660,7 @@ int x264_ratecontrol_new( x264_t *h ) } if( h->param.rc.b_mb_tree && (h->param.rc.b_stat_read || h->param.rc.b_stat_write) ) - { - rc->qp_buffer = x264_malloc( h->mb.i_mb_count * sizeof(uint16_t)); - if( !rc->qp_buffer ) - return -1; - } + CHECKED_MALLOC( rc->qp_buffer, h->mb.i_mb_count * sizeof(uint16_t) ); for( i=0; iparam.i_threads; i++ ) { @@ -664,12 +668,14 @@ int x264_ratecontrol_new( x264_t *h ) if( i ) { rc[i] = rc[0]; - memcpy( &h->thread[i]->param, &h->param, sizeof( x264_param_t ) ); + memcpy( &h->thread[i]->param, &h->param, sizeof(x264_param_t) ); h->thread[i]->mb.b_variable_qp = h->mb.b_variable_qp; } } return 0; +fail: + return -1; } static int parse_zone( x264_t *h, x264_zone_t *z, char *p ) @@ -692,7 +698,7 @@ static int parse_zone( x264_t *h, x264_zone_t *z, char *p ) p += len; if( !*p ) return 0; - z->param = x264_malloc( sizeof(x264_param_t) ); + CHECKED_MALLOC( z->param, sizeof(x264_param_t) ); memcpy( z->param, &h->param, sizeof(x264_param_t) ); while( (tok = strtok_r( p, ",", &saveptr )) ) { @@ -710,6 +716,8 @@ static int parse_zone( x264_t *h, x264_zone_t *z, char *p ) p = NULL; } return 0; +fail: + return -1; } static int parse_zones( x264_t *h ) @@ -718,13 +726,13 @@ static int parse_zones( x264_t *h ) int i; if( h->param.rc.psz_zones && !h->param.rc.i_zones ) { - char *p, *tok, UNUSED *saveptr; - char *psz_zones = x264_malloc( strlen(h->param.rc.psz_zones)+1 ); + char *psz_zones, *p, *tok, UNUSED *saveptr; + CHECKED_MALLOC( psz_zones, strlen( h->param.rc.psz_zones )+1 ); strcpy( psz_zones, h->param.rc.psz_zones ); h->param.rc.i_zones = 1; for( p = psz_zones; *p; p++ ) h->param.rc.i_zones += (*p == '/'); - h->param.rc.zones = x264_malloc( h->param.rc.i_zones * sizeof(x264_zone_t) ); + CHECKED_MALLOC( h->param.rc.zones, h->param.rc.i_zones * sizeof(x264_zone_t) ); p = psz_zones; for( i = 0; i < h->param.rc.i_zones; i++ ) { @@ -756,7 +764,7 @@ static int parse_zones( x264_t *h ) } rc->i_zones = h->param.rc.i_zones + 1; - rc->zones = x264_malloc( rc->i_zones * sizeof(x264_zone_t) ); + CHECKED_MALLOC( rc->zones, rc->i_zones * sizeof(x264_zone_t) ); memcpy( rc->zones+1, h->param.rc.zones, (rc->i_zones-1) * sizeof(x264_zone_t) ); // default zone to fall back to if none of the others match @@ -764,7 +772,7 @@ static int parse_zones( x264_t *h ) rc->zones[0].i_end = INT_MAX; rc->zones[0].b_force_qp = 0; rc->zones[0].f_bitrate_factor = 1; - rc->zones[0].param = x264_malloc( sizeof(x264_param_t) ); + CHECKED_MALLOC( rc->zones[0].param, sizeof(x264_param_t) ); memcpy( rc->zones[0].param, &h->param, sizeof(x264_param_t) ); for( i = 1; i < rc->i_zones; i++ ) { @@ -774,6 +782,8 @@ static int parse_zones( x264_t *h ) } return 0; +fail: + return -1; } static x264_zone_t *get_zone( x264_t *h, int frame_num ) @@ -1821,7 +1831,7 @@ static double count_expected_bits( x264_t *h ) return expected_bits; } -static void vbv_pass2( x264_t *h ) +static int vbv_pass2( x264_t *h ) { /* for each interval of buffer_full .. underflow, uniformly increase the qp of all * frames in the interval until either buffer is full at some intermediate frame or the @@ -1829,7 +1839,7 @@ static void vbv_pass2( x264_t *h ) * Then do the converse to put bits back into overflow areas until target size is met */ x264_ratecontrol_t *rcc = h->rc; - double *fills = x264_malloc((rcc->num_entries+1)*sizeof(double)); + double *fills; double all_available_bits = h->param.rc.i_bitrate * 1000. * rcc->num_entries / rcc->fps; double expected_bits = 0; double adjustment; @@ -1839,6 +1849,7 @@ static void vbv_pass2( x264_t *h ) double qscale_max = qp2qscale(h->param.rc.i_qp_max); int iterations = 0; int adj_min, adj_max; + CHECKED_MALLOC( fills, (rcc->num_entries+1)*sizeof(double) ); fills++; @@ -1880,6 +1891,9 @@ static void vbv_pass2( x264_t *h ) rcc->entry[i].expected_vbv = rcc->buffer_size - fills[i]; x264_free(fills-1); + return 0; +fail: + return -1; } static int init_pass2( x264_t *h ) @@ -1947,9 +1961,9 @@ static int init_pass2( x264_t *h ) rce->blurred_complexity = cplx_sum / weight_sum; } - qscale = x264_malloc(sizeof(double)*rcc->num_entries); - if(filter_size > 1) - blurred_qscale = x264_malloc(sizeof(double)*rcc->num_entries); + CHECKED_MALLOC( qscale, sizeof(double)*rcc->num_entries ); + if( filter_size > 1 ) + CHECKED_MALLOC( blurred_qscale, sizeof(double)*rcc->num_entries ); else blurred_qscale = qscale; @@ -2031,7 +2045,8 @@ static int init_pass2( x264_t *h ) x264_free(blurred_qscale); if(rcc->b_vbv) - vbv_pass2(h); + if( vbv_pass2( h ) ) + return -1; expected_bits = count_expected_bits(h); if(fabs(expected_bits/all_available_bits - 1.0) > 0.01) @@ -2066,6 +2081,6 @@ static int init_pass2( x264_t *h ) } return 0; +fail: + return -1; } - - diff --git a/encoder/set.c b/encoder/set.c index 3103fcd8..5c00096d 100644 --- a/encoder/set.c +++ b/encoder/set.c @@ -470,7 +470,7 @@ void x264_pps_write( bs_t *s, x264_pps_t *pps ) bs_rbsp_trailing( s ); } -void x264_sei_version_write( x264_t *h, bs_t *s ) +int x264_sei_version_write( x264_t *h, bs_t *s ) { int i; // random ID number generated according to ISO-11578 @@ -479,9 +479,13 @@ void x264_sei_version_write( x264_t *h, bs_t *s ) 0x96, 0x2c, 0xd8, 0x20, 0xd9, 0x23, 0xee, 0xef }; char *opts = x264_param2string( &h->param, 0 ); - char *version = x264_malloc( 200 + strlen(opts) ); + char *version; int length; + if( !opts ) + return -1; + CHECKED_MALLOC( version, 200 + strlen( opts ) ); + sprintf( version, "x264 - core %d%s - H.264/MPEG-4 AVC codec - " "Copyleft 2003-2009 - http://www.videolan.org/x264.html - options: %s", X264_BUILD, X264_VERSION, opts ); @@ -502,6 +506,10 @@ void x264_sei_version_write( x264_t *h, bs_t *s ) x264_free( opts ); x264_free( version ); + return 0; +fail: + x264_free( opts ); + return -1; } const x264_level_t x264_levels[] = diff --git a/encoder/set.h b/encoder/set.h index 3611c9ad..e76e6511 100644 --- a/encoder/set.h +++ b/encoder/set.h @@ -28,7 +28,7 @@ void x264_sps_init( x264_sps_t *sps, int i_id, x264_param_t *param ); void x264_sps_write( bs_t *s, x264_sps_t *sps ); void x264_pps_init( x264_pps_t *pps, int i_id, x264_param_t *param, x264_sps_t *sps ); void x264_pps_write( bs_t *s, x264_pps_t *pps ); -void x264_sei_version_write( x264_t *h, bs_t *s ); +int x264_sei_version_write( x264_t *h, bs_t *s ); int x264_validate_levels( x264_t *h, int verbose ); #endif diff --git a/encoder/slicetype.c b/encoder/slicetype.c index b5b55c68..a393dcab 100644 --- a/encoder/slicetype.c +++ b/encoder/slicetype.c @@ -30,14 +30,22 @@ #include "me.h" -static void x264_lowres_context_init( x264_t *h, x264_mb_analysis_t *a ) +static int x264_lowres_context_init( x264_t *h, x264_mb_analysis_t *a ) { a->i_qp = 12; // arbitrary, but low because SATD scores are 1/4 normal a->i_lambda = x264_lambda_tab[ a->i_qp ]; - x264_mb_analyse_load_costs( h, a ); + if( x264_mb_analyse_load_costs( h, a ) ) + return -1; h->mb.i_me_method = X264_MIN( X264_ME_HEX, h->param.analyse.i_me_method ); // maybe dia? h->mb.i_subpel_refine = 4; // 3 should be enough, but not tweaking for speed now h->mb.b_chroma_me = 0; + return 0; +} + +int x264_lowres_context_alloc( x264_t *h ) +{ + x264_mb_analysis_t a; + return x264_lowres_context_init( h, &a ); } static int x264_slicetype_mb_cost( x264_t *h, x264_mb_analysis_t *a, diff --git a/matroska.c b/matroska.c index 35ae4cdb..68bef0bc 100644 --- a/matroska.c +++ b/matroska.c @@ -62,12 +62,11 @@ static mk_Context *mk_createContext(mk_Writer *w, mk_Context *parent, unsigned i w->freelist = w->freelist->next; } else { c = malloc(sizeof(*c)); + if (c == NULL) + return NULL; memset(c, 0, sizeof(*c)); } - if (c == NULL) - return NULL; - c->parent = parent; c->owner = w; c->id = id; diff --git a/muxers.c b/muxers.c index d62be5cd..ef2f5ff1 100644 --- a/muxers.c +++ b/muxers.c @@ -63,6 +63,8 @@ typedef struct { int open_file_yuv( char *psz_filename, hnd_t *p_handle, x264_param_t *p_param ) { yuv_input_t *h = malloc(sizeof(yuv_input_t)); + if( !h ) + return -1; h->width = p_param->i_width; h->height = p_param->i_height; h->next_frame = 0; @@ -141,6 +143,8 @@ int open_file_y4m( char *psz_filename, hnd_t *p_handle, x264_param_t *p_param ) char header[MAX_YUV4_HEADER+10]; char *tokstart, *tokend, *header_end; y4m_input_t *h = malloc(sizeof(y4m_input_t)); + if( !h ) + return -1; h->next_frame = 0; @@ -335,6 +339,8 @@ typedef struct { int open_file_avis( char *psz_filename, hnd_t *p_handle, x264_param_t *p_param ) { avis_input_t *h = malloc(sizeof(avis_input_t)); + if( !h ) + return -1; AVISTREAMINFO info; int i; @@ -440,13 +446,19 @@ typedef struct thread_input_arg_t { int open_file_thread( char *psz_filename, hnd_t *p_handle, x264_param_t *p_param ) { thread_input_t *h = malloc(sizeof(thread_input_t)); - x264_picture_alloc( &h->pic, X264_CSP_I420, p_param->i_width, p_param->i_height ); + if( !h || x264_picture_alloc( &h->pic, X264_CSP_I420, p_param->i_width, p_param->i_height ) < 0 ) + { + fprintf( stderr, "x264 [error]: malloc failed\n" ); + return -1; + } h->p_read_frame = p_read_frame; h->p_close_infile = p_close_infile; h->p_handle = *p_handle; h->in_progress = 0; h->next_frame = -1; h->next_args = malloc(sizeof(thread_input_arg_t)); + if( !h->next_args ) + return -1; h->next_args->h = h; h->next_args->status = 0; h->frame_total = p_get_frame_total( h->p_handle ); @@ -469,12 +481,11 @@ static void read_frame_thread_int( thread_input_arg_t *i ) int read_frame_thread( x264_picture_t *p_pic, hnd_t handle, int i_frame ) { thread_input_t *h = handle; - UNUSED void *stuff; int ret = 0; if( h->next_frame >= 0 ) { - x264_pthread_join( h->tid, &stuff ); + x264_pthread_join( h->tid, NULL ); ret |= h->next_args->status; h->in_progress = 0; } @@ -493,7 +504,8 @@ int read_frame_thread( x264_picture_t *p_pic, hnd_t handle, int i_frame ) h->next_frame = h->next_args->i_frame = i_frame+1; h->next_args->pic = &h->pic; - x264_pthread_create( &h->tid, NULL, (void*)read_frame_thread_int, h->next_args ); + if( x264_pthread_create( &h->tid, NULL, (void*)read_frame_thread_int, h->next_args ) ) + return -1; h->in_progress = 1; } else @@ -731,8 +743,12 @@ int write_nalu_mp4( hnd_t handle, uint8_t *p_nalu, int i_size ) p_mp4->p_config->profile_compatibility = p_nalu[6]; p_mp4->p_config->AVCLevelIndication = p_nalu[7]; p_slot = (GF_AVCConfigSlot *)malloc(sizeof(GF_AVCConfigSlot)); + if( !p_slot ) + return -1; p_slot->size = i_size - 4; p_slot->data = (char *)malloc(p_slot->size); + if( p_slot->data ) + return -1; memcpy(p_slot->data, p_nalu + 4, i_size - 4); gf_list_add(p_mp4->p_config->sequenceParameterSets, p_slot); p_slot = NULL; @@ -745,8 +761,12 @@ int write_nalu_mp4( hnd_t handle, uint8_t *p_nalu, int i_size ) if (!p_mp4->b_pps) { p_slot = (GF_AVCConfigSlot *)malloc(sizeof(GF_AVCConfigSlot)); + if( !p_slot ) + return -1; p_slot->size = i_size - 4; p_slot->data = (char *)malloc(p_slot->size); + if( !p_slot->data ) + return -1; memcpy(p_slot->data, p_nalu + 4, i_size - 4); gf_list_add(p_mp4->p_config->pictureParameterSets, p_slot); p_slot = NULL; diff --git a/tools/checkasm.c b/tools/checkasm.c index ff2e445a..f599f00c 100644 --- a/tools/checkasm.c +++ b/tools/checkasm.c @@ -1584,6 +1584,11 @@ int main(int argc, char *argv[]) srand( i ); buf1 = x264_malloc( 0x3e00 + 16*BENCH_ALIGNS ); + if( !buf1 ) + { + fprintf( stderr, "malloc failed, unable to initiate tests!\n" ); + return -1; + } buf2 = buf1 + 0xf00; buf3 = buf2 + 0xf00; buf4 = buf3 + 0x1000; diff --git a/x264.c b/x264.c index 74e32b7c..f5073853 100644 --- a/x264.c +++ b/x264.c @@ -985,7 +985,8 @@ generic_option: { if( open_file_thread( NULL, &opt->hin, param ) ) { - fprintf( stderr, "x264 [warning]: threaded input failed\n" ); + fprintf( stderr, "x264 [error]: threaded input failed\n" ); + return -1; } else { @@ -1065,12 +1066,13 @@ static int Encode_frame( x264_t *h, hnd_t hout, x264_picture_t *pic ) { x264_picture_t pic_out; x264_nal_t *nal; - int i_nal, i; + int i_nal, i, i_nalu_size; int i_file = 0; if( x264_encoder_encode( h, &nal, &i_nal, pic, &pic_out ) < 0 ) { fprintf( stderr, "x264 [error]: x264_encoder_encode failed\n" ); + return -1; } for( i = 0; i < i_nal; i++ ) @@ -1082,11 +1084,16 @@ static int Encode_frame( x264_t *h, hnd_t hout, x264_picture_t *pic ) mux_buffer_size = nal[i].i_payload * 2 + 4; x264_free( mux_buffer ); mux_buffer = x264_malloc( mux_buffer_size ); + if( !mux_buffer ) + return -1; } i_size = mux_buffer_size; x264_nal_encode( mux_buffer, &i_size, 1, &nal[i] ); - i_file += p_write_nalu( hout, mux_buffer, i_size ); + i_nalu_size = p_write_nalu( hout, mux_buffer, i_size ); + if( i_nalu_size < 0 ) + return -1; + i_file += i_nalu_size; } if (i_nal) p_set_eop( hout, &pic_out ); @@ -1131,7 +1138,11 @@ static int Encode( x264_param_t *param, cli_opt_t *opt ) } /* Create a new pic */ - x264_picture_alloc( &pic, X264_CSP_I420, param->i_width, param->i_height ); + if( x264_picture_alloc( &pic, X264_CSP_I420, param->i_width, param->i_height ) < 0 ) + { + fprintf( stderr, "x264 [error]: malloc failed\n" ); + return -1; + } i_start = x264_mdate(); @@ -1152,7 +1163,10 @@ static int Encode( x264_param_t *param, cli_opt_t *opt ) pic.i_qpplus1 = 0; } - i_file += Encode_frame( h, opt->hout, &pic ); + i_frame_size = Encode_frame( h, opt->hout, &pic ); + if( i_frame_size < 0 ) + return -1; + i_file += i_frame_size; i_frame++; @@ -1180,8 +1194,10 @@ static int Encode( x264_param_t *param, cli_opt_t *opt ) } /* Flush delayed B-frames */ do { - i_file += i_frame_size = Encode_frame( h, opt->hout, NULL ); + if( i_frame_size < 0 ) + return -1; + i_file += i_frame_size; } while( i_frame_size ); i_end = x264_mdate(); diff --git a/x264.h b/x264.h index 4982b2e0..14162483 100644 --- a/x264.h +++ b/x264.h @@ -35,7 +35,7 @@ #include -#define X264_BUILD 69 +#define X264_BUILD 70 /* x264_t: * opaque handler for encoder */ @@ -360,8 +360,9 @@ typedef struct } x264_picture_t; /* x264_picture_alloc: - * alloc data for a picture. You must call x264_picture_clean on it. */ -void x264_picture_alloc( x264_picture_t *pic, int i_csp, int i_width, int i_height ); + * alloc data for a picture. You must call x264_picture_clean on it. + * returns 0 on success, or -1 on malloc failure. */ +int x264_picture_alloc( x264_picture_t *pic, int i_csp, int i_width, int i_height ); /* x264_picture_clean: * free associated resource for a x264_picture_t allocated with -- 2.40.0