check some mallocs' return value

git-svn-id: svn://svn.videolan.org/x264/trunk@529 df754926-b1dd-0310-bc7b-ec298dee348c

diff --git a/common/common.h b/common/common.h
index b9fbae44482c72b6f99bb848dc768aace02403ac..799d19542f7411c904344a148301252cf83f2c47 100644 (file)
--- a/common/common.h
+++ b/common/common.h
@@ -77,6 +77,16 @@
 #define UNUSED
 #endif
 
+#define CHECKED_MALLOC( var, size )\
+{\
+    var = x264_malloc( size );\
+    if( !var )\
+    {\
+        x264_log( h, X264_LOG_ERROR, "malloc failed\n" );\
+        goto fail;\
+    }\
+}
+
 #define X264_BFRAME_MAX 16
 #define X264_SLICE_MAX 4
 #define X264_NAL_MAX (4 + X264_SLICE_MAX)

diff --git a/common/frame.c b/common/frame.c
index bd0cb7af3cd247bb8d30c8f392e50a5944eca99e..260359e151dfbb1c425ac418629a7a45147a4dac 100644 (file)
--- a/common/frame.c
+++ b/common/frame.c
@@ -28,13 +28,15 @@
 
 x264_frame_t *x264_frame_new( x264_t *h )
 {
-    x264_frame_t   *frame = x264_malloc( sizeof( x264_frame_t ) );
+    x264_frame_t *frame = x264_malloc( sizeof(x264_frame_t) );
     int i, j;
 
     int i_mb_count = h->mb.i_mb_count;
     int i_stride;
     int i_lines;
 
+    if( !frame ) return NULL;
+
     memset( frame, 0, sizeof(x264_frame_t) );
 
     /* allocate frame data (+64 for extra data for me) */
@@ -55,9 +57,8 @@ x264_frame_t *x264_frame_new( x264_t *h )
         }
         frame->i_stride[i] = i_stride / i_divw;
         frame->i_lines[i] = i_lines / i_divh;
-        frame->buffer[i] = x264_malloc( frame->i_stride[i] *
-                                        ( frame->i_lines[i] + 64 / i_divh ) );
-
+        CHECKED_MALLOC( frame->buffer[i],
+                        frame->i_stride[i] * ( frame->i_lines[i] + 64 / i_divh ) );
         frame->plane[i] = ((uint8_t*)frame->buffer[i]) +
                           frame->i_stride[i] * 32 / i_divh + 32 / i_divw;
     }
@@ -69,9 +70,8 @@ x264_frame_t *x264_frame_new( x264_t *h )
     frame->filtered[0] = frame->plane[0];
     for( i = 0; i < 3; i++ )
     {
-        frame->buffer[4+i] = x264_malloc( frame->i_stride[0] *
-                                        ( frame->i_lines[0] + 64 ) );
-
+        CHECKED_MALLOC( frame->buffer[4+i],
+                        frame->i_stride[0] * ( frame->i_lines[0] + 64 ) );
         frame->filtered[i+1] = ((uint8_t*)frame->buffer[4+i]) +
                                 frame->i_stride[0] * 32 + 32;
     }
@@ -82,8 +82,8 @@ x264_frame_t *x264_frame_new( x264_t *h )
         frame->i_lines_lowres = frame->i_lines[0]/2;
         for( i = 0; i < 4; i++ )
         {
-            frame->buffer[7+i] = x264_malloc( frame->i_stride_lowres *
-                                            ( frame->i_lines[0]/2 + 64 ) );
+            CHECKED_MALLOC( frame->buffer[7+i],
+                            frame->i_stride_lowres * ( frame->i_lines[0]/2 + 64 ) );
             frame->lowres[i] = ((uint8_t*)frame->buffer[7+i]) +
                                 frame->i_stride_lowres * 32 + 32;
         }
@@ -91,7 +91,8 @@ x264_frame_t *x264_frame_new( x264_t *h )
 
     if( h->param.analyse.i_me_method == X264_ME_ESA )
     {
-        frame->buffer[11] = x264_malloc( frame->i_stride[0] * (frame->i_lines[0] + 64) * sizeof(uint16_t) );
+        CHECKED_MALLOC( frame->buffer[11],
+                        frame->i_stride[0] * (frame->i_lines[0] + 64) * sizeof(uint16_t) );
         frame->integral = (uint16_t*)frame->buffer[11] + frame->i_stride[0] * 32 + 32;
     }
 
@@ -102,13 +103,13 @@ x264_frame_t *x264_frame_new( x264_t *h )
     frame->i_frame = -1;
     frame->i_frame_num = -1;
 
-    frame->mb_type= x264_malloc( i_mb_count * sizeof( int8_t) );
-    frame->mv[0]  = x264_malloc( 2*16 * i_mb_count * sizeof( int16_t ) );
-    frame->ref[0] = x264_malloc( 4 * i_mb_count * sizeof( int8_t ) );
+    CHECKED_MALLOC( frame->mb_type, i_mb_count * sizeof(int8_t));
+    CHECKED_MALLOC( frame->mv[0], 2*16 * i_mb_count * sizeof(int16_t) );
+    CHECKED_MALLOC( frame->ref[0], 4 * i_mb_count * sizeof(int8_t) );
     if( h->param.i_bframe )
     {
-        frame->mv[1]  = x264_malloc( 2*16 * i_mb_count * sizeof( int16_t ) );
-        frame->ref[1] = x264_malloc( 4 * i_mb_count * sizeof( int8_t ) );
+        CHECKED_MALLOC( frame->mv[1], 2*16 * i_mb_count * sizeof(int16_t) );
+        CHECKED_MALLOC( frame->ref[1], 4 * i_mb_count * sizeof(int8_t) );
     }
     else
     {
@@ -116,21 +117,23 @@ x264_frame_t *x264_frame_new( x264_t *h )
         frame->ref[1] = NULL;
     }
 
-    frame->i_row_bits = x264_malloc( i_lines/16 * sizeof( int ) );
-    frame->i_row_qp   = x264_malloc( i_lines/16 * sizeof( int ) );
+    CHECKED_MALLOC( frame->i_row_bits, i_lines/16 * sizeof(int) );
+    CHECKED_MALLOC( frame->i_row_qp, i_lines/16 * sizeof(int) );
     for( i = 0; i < h->param.i_bframe + 2; i++ )
         for( j = 0; j < h->param.i_bframe + 2; j++ )
-            frame->i_row_satds[i][j] = x264_malloc( i_lines/16 * sizeof( int ) );
+            CHECKED_MALLOC( frame->i_row_satds[i][j], i_lines/16 * sizeof(int) );
 
     return frame;
+
+fail:
+    x264_frame_delete( frame );
+    return NULL;
 }
 
 void x264_frame_delete( x264_frame_t *frame )
 {
     int i, j;
-    for( i = 0; i < frame->i_plane; i++ )
-        x264_free( frame->buffer[i] );
-    for( i = 4; i < 12; i++ ) /* filtered planes */
+    for( i = 0; i < 12; i++ )
         x264_free( frame->buffer[i] );
     for( i = 0; i < X264_BFRAME_MAX+2; i++ )
         for( j = 0; j < X264_BFRAME_MAX+2; j++ )

diff --git a/common/macroblock.c b/common/macroblock.c
index 1507cf57b88b428e8bad8c510dc789fbabc943ff..bb440733d1cb0b180ab4ced4e952f64f6e5005b0 100644 (file)
--- a/common/macroblock.c
+++ b/common/macroblock.c
@@ -824,7 +824,7 @@ void x264_mb_mc( x264_t *h )
     }
 }
 
-void x264_macroblock_cache_init( x264_t *h )
+int x264_macroblock_cache_init( x264_t *h )
 {
     int i, j;
     int i_mb_count = h->mb.i_mb_count;
@@ -833,34 +833,37 @@ void x264_macroblock_cache_init( x264_t *h )
     h->mb.i_b8_stride = h->sps->i_mb_width * 2;
     h->mb.i_b4_stride = h->sps->i_mb_width * 4;
 
-    h->mb.qp  = x264_malloc( i_mb_count * sizeof(int8_t) );
-    h->mb.cbp = x264_malloc( i_mb_count * sizeof(int16_t) );
-    h->mb.skipbp = x264_malloc( i_mb_count * sizeof(int8_t) );
-    h->mb.mb_transform_size = x264_malloc( i_mb_count * sizeof(int8_t) );
+    CHECKED_MALLOC( h->mb.qp, i_mb_count * sizeof(int8_t) );
+    CHECKED_MALLOC( h->mb.cbp, i_mb_count * sizeof(int16_t) );
+    CHECKED_MALLOC( h->mb.skipbp, i_mb_count * sizeof(int8_t) );
+    CHECKED_MALLOC( h->mb.mb_transform_size, i_mb_count * sizeof(int8_t) );
 
     /* 0 -> 3 top(4), 4 -> 6 : left(3) */
-    h->mb.intra4x4_pred_mode = x264_malloc( i_mb_count * 7 * sizeof( int8_t ) );
+    CHECKED_MALLOC( h->mb.intra4x4_pred_mode, i_mb_count * 7 * sizeof(int8_t) );
 
     /* all coeffs */
-    h->mb.non_zero_count = x264_malloc( i_mb_count * 24 * sizeof( uint8_t ) );
+    CHECKED_MALLOC( h->mb.non_zero_count, i_mb_count * 24 * sizeof(uint8_t) );
 
     if( h->param.b_cabac )
     {
-        h->mb.chroma_pred_mode = x264_malloc( i_mb_count * sizeof( int8_t) );
-        h->mb.mvd[0] = x264_malloc( 2*16 * i_mb_count * sizeof( int16_t ) );
-        h->mb.mvd[1] = x264_malloc( 2*16 * i_mb_count * sizeof( int16_t ) );
+        CHECKED_MALLOC( h->mb.chroma_pred_mode, i_mb_count * sizeof(int8_t) );
+        CHECKED_MALLOC( h->mb.mvd[0], 2*16 * i_mb_count * sizeof(int16_t) );
+        CHECKED_MALLOC( h->mb.mvd[1], 2*16 * i_mb_count * sizeof(int16_t) );
     }
 
     for( i=0; i<2; i++ )
     {
         int i_refs = (i ? 1 : h->param.i_frame_reference) + h->param.b_bframe_pyramid;
         for( j=0; j < i_refs && j < 16; j++ )
-            h->mb.mvr[i][j] = x264_malloc( 2 * i_mb_count * sizeof( int16_t ) );
+            CHECKED_MALLOC( h->mb.mvr[i][j], 2 * i_mb_count * sizeof(int16_t) );
     }
 
     /* init with not avaiable (for top right idx=7,15) */
     memset( h->mb.cache.ref[0], -2, X264_SCAN8_SIZE * sizeof( int8_t ) );
     memset( h->mb.cache.ref[1], -2, X264_SCAN8_SIZE * sizeof( int8_t ) );
+
+    return 0;
+fail: return -1;
 }
 void x264_macroblock_cache_end( x264_t *h )
 {

diff --git a/common/macroblock.h b/common/macroblock.h
index 69f4af73219446352dd9e8a9d227ce40d249459e..0bdd4baf6ab168b35cb86108c6f830329e180a21 100644 (file)
--- a/common/macroblock.h
+++ b/common/macroblock.h
@@ -229,7 +229,7 @@ enum cabac_ctx_block_cat_e
 };
 
 
-void x264_macroblock_cache_init( x264_t *h );
+int  x264_macroblock_cache_init( x264_t *h );
 void x264_macroblock_slice_init( x264_t *h );
 void x264_macroblock_cache_load( x264_t *h, int i_mb_x, int i_mb_y );
 void x264_macroblock_cache_save( x264_t *h );

diff --git a/encoder/encoder.c b/encoder/encoder.c
index 08480af6192774ef133e0f1d180c39916b07783c..a8955d06688c679ed5f5d1435da15af355895994 100644 (file)
--- a/encoder/encoder.c
+++ b/encoder/encoder.c
@@ -559,10 +559,14 @@ x264_t *x264_encoder_open   ( x264_param_t *param )
     for( i = 0; i < 1 + h->frames.i_delay; i++ )
     {
         h->frames.unused[i] =  x264_frame_new( h );
+        if( !h->frames.unused[i] )
+            return NULL;
     }
     for( i = 0; i < h->frames.i_max_dpb; i++ )
     {
         h->frames.reference[i] = x264_frame_new( h );
+        if( !h->frames.reference[i] )
+            return NULL;
     }
     h->frames.reference[h->frames.i_max_dpb] = NULL;
     h->frames.i_last_idr = - h->param.i_keyint_max;
@@ -574,7 +578,8 @@ x264_t *x264_encoder_open   ( x264_param_t *param )
 
     h->fdec = h->frames.reference[0];
 
-    x264_macroblock_cache_init( h );
+    if( x264_macroblock_cache_init( h ) < 0 );
+        return NULL;
     x264_rdo_init( );
 
     /* init CPU functions */