Fix undefined behavior: applying [non-]zero offset to null pointer

diff --git a/common/base.h b/common/base.h
index 2851bd59afc20e960c309267528b30f3e89e50cb..cc0bce4ea212b08e0887dd7382079ec7741fcfad 100644 (file)
--- a/common/base.h
+++ b/common/base.h
@@ -310,7 +310,7 @@ do {\
 do {\
     CHECKED_MALLOC( ptr, prealloc_size );\
     while( prealloc_idx-- )\
-        *preallocs[prealloc_idx] += (intptr_t)ptr;\
+        *preallocs[prealloc_idx] = (uint8_t*)((intptr_t)(*preallocs[prealloc_idx]) + (intptr_t)ptr);\
 } while( 0 )
 
 #endif

diff --git a/common/deblock.c b/common/deblock.c
index 402b48aef0e0d650a677c3555351d5e3809b3cf5..63351f90229d04c61ae53ae1f019ee286121553f 100644 (file)
--- a/common/deblock.c
+++ b/common/deblock.c
@@ -399,12 +399,13 @@ void x264_frame_deblock_row( x264_t *h, int mb_y )
         uint8_t (*bs)[8][4] = h->deblock_strength[mb_y&1][h->param.b_sliced_threads?mb_xy:mb_x];
 
         pixel *pixy = h->fdec->plane[0] + 16*mb_y*stridey  + 16*mb_x;
-        pixel *pixuv = h->fdec->plane[1] + chroma_height*mb_y*strideuv + 16*mb_x;
+        pixel *pixuv = CHROMA_FORMAT ? h->fdec->plane[1] + chroma_height*mb_y*strideuv + 16*mb_x : NULL;
 
         if( mb_y & MB_INTERLACED )
         {
             pixy -= 15*stridey;
-            pixuv -= (chroma_height-1)*strideuv;
+            if( CHROMA_FORMAT )
+                pixuv -= (chroma_height-1)*strideuv;
         }
 
         int stride2y  = stridey << MB_INTERLACED;

diff --git a/common/frame.c b/common/frame.c
index b96679da5b0e22131d3be9300408797986bb06a0..046016c65fcb3a0383760af6576a7110f3f291ae 100644 (file)
--- a/common/frame.c
+++ b/common/frame.c
@@ -250,7 +250,8 @@ static x264_frame_t *frame_new( x264_t *h, int b_fdec )
             for( int i = 0; i < 4; i++ )
             {
                 frame->filtered[p][i] = frame->buffer[p] + i*luma_plane_size + frame->i_stride[p] * i_padv + PADH + padh_align;
-                frame->filtered_fld[p][i] = frame->buffer_fld[p] + i*luma_plane_size + frame->i_stride[p] * i_padv + PADH + padh_align;
+                if( PARAM_INTERLACED )
+                    frame->filtered_fld[p][i] = frame->buffer_fld[p] + i*luma_plane_size + frame->i_stride[p] * i_padv + PADH + padh_align;
             }
             frame->plane[p] = frame->filtered[p][0];
             frame->plane_fld[p] = frame->filtered_fld[p][0];
@@ -258,7 +259,8 @@ static x264_frame_t *frame_new( x264_t *h, int b_fdec )
         else
         {
             frame->filtered[p][0] = frame->plane[p] = frame->buffer[p] + frame->i_stride[p] * i_padv + PADH + padh_align;
-            frame->filtered_fld[p][0] = frame->plane_fld[p] = frame->buffer_fld[p] + frame->i_stride[p] * i_padv + PADH + padh_align;
+            if( PARAM_INTERLACED )
+                frame->filtered_fld[p][0] = frame->plane_fld[p] = frame->buffer_fld[p] + frame->i_stride[p] * i_padv + PADH + padh_align;
         }
     }
 

diff --git a/common/macroblock.c b/common/macroblock.c
index d21a8fea05c108f4dd7ccc34730a28db7ce6a431..53d3758633b44ba37acdf33193794ca9fcdfa142 100644 (file)
--- a/common/macroblock.c
+++ b/common/macroblock.c
@@ -555,7 +555,7 @@ void x264_prefetch_fenc( x264_t *h, x264_frame_t *fenc, int i_mb_x, int i_mb_y )
     int off_y  = 16 * i_mb_x + 16 * i_mb_y * stride_y;
     int off_uv = 16 * i_mb_x + (16 * i_mb_y * stride_uv >> CHROMA_V_SHIFT);
     h->mc.prefetch_fenc( fenc->plane[0]+off_y, stride_y,
-                         fenc->plane[1]+off_uv, stride_uv, i_mb_x );
+                         fenc->plane[1] != NULL ? fenc->plane[1]+off_uv : NULL, stride_uv, i_mb_x );
 }
 
 NOINLINE void x264_copy_column8( pixel *dst, pixel *src )
@@ -626,8 +626,9 @@ static ALWAYS_INLINE void macroblock_load_pic_pointers( x264_t *h, int mb_x, int
 
         if( !b_chroma )
         {
-            for( int k = 1; k < 4; k++ )
-                h->mb.pic.p_fref[0][j][i*4+k] = filtered_src[k] + ref_pix_offset[j&1];
+            if( h->param.analyse.i_subpel_refine )
+                for( int k = 1; k < 4; k++ )
+                    h->mb.pic.p_fref[0][j][i*4+k] = filtered_src[k] + ref_pix_offset[j&1];
             if( !i )
             {
                 if( h->sh.weight[j][0].weightfn )
@@ -652,7 +653,7 @@ static ALWAYS_INLINE void macroblock_load_pic_pointers( x264_t *h, int mb_x, int
             }
             h->mb.pic.p_fref[1][j][i*4] = plane_src + ref_pix_offset[j&1];
 
-            if( !b_chroma )
+            if( !b_chroma && h->param.analyse.i_subpel_refine )
                 for( int k = 1; k < 4; k++ )
                     h->mb.pic.p_fref[1][j][i*4+k] = filtered_src[k] + ref_pix_offset[j&1];
         }
@@ -806,9 +807,9 @@ static ALWAYS_INLINE void macroblock_cache_load_neighbours( x264_t *h, int mb_x,
                 x264_prefetch( &h->mb.cbp[top] );
                 x264_prefetch( h->mb.intra4x4_pred_mode[top] );
                 x264_prefetch( &h->mb.non_zero_count[top][12] );
-                /* These aren't always allocated, but prefetching an invalid address can't hurt. */
                 x264_prefetch( &h->mb.mb_transform_size[top] );
-                x264_prefetch( &h->mb.skipbp[top] );
+                if( h->param.b_cabac )
+                    x264_prefetch( &h->mb.skipbp[top] );
             }
         }
 
@@ -894,7 +895,8 @@ static ALWAYS_INLINE void macroblock_cache_load( x264_t *h, int mb_x, int mb_y,
              * once every 4 MBs, so one extra prefetch is worthwhile */
             x264_prefetch( &h->mb.mv[l][top_4x4+4] );
             x264_prefetch( &h->mb.ref[l][top_8x8-1] );
-            x264_prefetch( &h->mb.mvd[l][top] );
+            if( h->param.b_cabac )
+                x264_prefetch( &h->mb.mvd[l][top] );
         }
     }
     else
@@ -1779,9 +1781,7 @@ void x264_macroblock_cache_save( x264_t *h )
     if( h->sh.i_type != SLICE_TYPE_I )
     {
         int16_t (*mv0)[2] = &h->mb.mv[0][i_mb_4x4];
-        int16_t (*mv1)[2] = &h->mb.mv[1][i_mb_4x4];
         int8_t *ref0 = &h->mb.ref[0][i_mb_8x8];
-        int8_t *ref1 = &h->mb.ref[1][i_mb_8x8];
         if( !IS_INTRA( i_mb_type ) )
         {
             ref0[0+0*s8x8] = h->mb.cache.ref[0][x264_scan8[0]];
@@ -1794,6 +1794,8 @@ void x264_macroblock_cache_save( x264_t *h )
             CP128( &mv0[3*s4x4], h->mb.cache.mv[0][x264_scan8[0]+8*3] );
             if( h->sh.i_type == SLICE_TYPE_B )
             {
+                int16_t (*mv1)[2] = &h->mb.mv[1][i_mb_4x4];
+                int8_t *ref1 = &h->mb.ref[1][i_mb_8x8];
                 ref1[0+0*s8x8] = h->mb.cache.ref[1][x264_scan8[0]];
                 ref1[1+0*s8x8] = h->mb.cache.ref[1][x264_scan8[4]];
                 ref1[0+1*s8x8] = h->mb.cache.ref[1][x264_scan8[8]];
@@ -1814,6 +1816,8 @@ void x264_macroblock_cache_save( x264_t *h )
             M128( &mv0[3*s4x4] ) = M128_ZERO;
             if( h->sh.i_type == SLICE_TYPE_B )
             {
+                int16_t (*mv1)[2] = &h->mb.mv[1][i_mb_4x4];
+                int8_t *ref1 = &h->mb.ref[1][i_mb_8x8];
                 M16( &ref1[0*s8x8] ) = (uint8_t)(-1) * 0x0101;
                 M16( &ref1[1*s8x8] ) = (uint8_t)(-1) * 0x0101;
                 M128( &mv1[0*s4x4] ) = M128_ZERO;
@@ -1827,7 +1831,6 @@ void x264_macroblock_cache_save( x264_t *h )
     if( h->param.b_cabac )
     {
         uint8_t (*mvd0)[2] = h->mb.mvd[0][i_mb_xy];
-        uint8_t (*mvd1)[2] = h->mb.mvd[1][i_mb_xy];
         if( IS_INTRA(i_mb_type) && i_mb_type != I_PCM )
             h->mb.chroma_pred_mode[i_mb_xy] = x264_mb_chroma_pred_mode_fix[h->mb.i_chroma_pred_mode];
         else
@@ -1841,6 +1844,7 @@ void x264_macroblock_cache_save( x264_t *h )
             CP16( mvd0[6], h->mb.cache.mvd[0][x264_scan8[13]] );
             if( h->sh.i_type == SLICE_TYPE_B )
             {
+                uint8_t (*mvd1)[2] = h->mb.mvd[1][i_mb_xy];
                 CP64( mvd1[0], h->mb.cache.mvd[1][x264_scan8[10]] );
                 CP16( mvd1[4], h->mb.cache.mvd[1][x264_scan8[5 ]] );
                 CP16( mvd1[5], h->mb.cache.mvd[1][x264_scan8[7 ]] );
@@ -1851,7 +1855,10 @@ void x264_macroblock_cache_save( x264_t *h )
         {
             M128( mvd0[0] ) = M128_ZERO;
             if( h->sh.i_type == SLICE_TYPE_B )
+            {
+                uint8_t (*mvd1)[2] = h->mb.mvd[1][i_mb_xy];
                 M128( mvd1[0] ) = M128_ZERO;
+            }
         }
 
         if( h->sh.i_type == SLICE_TYPE_B )

diff --git a/encoder/analyse.c b/encoder/analyse.c
index 1987ba3311b710d7a9588c953208274f79d82975..75518547af2fcaa771d622d597ceddc66bcc12e3 100644 (file)
--- a/encoder/analyse.c
+++ b/encoder/analyse.c
@@ -1090,8 +1090,8 @@ static void intra_rd_refine( x264_t *h, x264_mb_analysis_t *a )
         for( int idx = 0; idx < 16; idx++ )
         {
             pixel *dst[3] = {h->mb.pic.p_fdec[0] + block_idx_xy_fdec[idx],
-                             h->mb.pic.p_fdec[1] + block_idx_xy_fdec[idx],
-                             h->mb.pic.p_fdec[2] + block_idx_xy_fdec[idx]};
+                             CHROMA_FORMAT ? h->mb.pic.p_fdec[1] + block_idx_xy_fdec[idx] : NULL,
+                             CHROMA_FORMAT ? h->mb.pic.p_fdec[2] + block_idx_xy_fdec[idx] : NULL};
             i_best = COST_MAX64;
 
             const int8_t *predict_mode = predict_4x4_mode_available( a->b_avoid_topright, h->mb.i_neighbour4[idx], idx );
@@ -1145,8 +1145,8 @@ static void intra_rd_refine( x264_t *h, x264_mb_analysis_t *a )
             int y = idx>>1;
             int s8 = X264_SCAN8_0 + 2*x + 16*y;
             pixel *dst[3] = {h->mb.pic.p_fdec[0] + 8*x + 8*y*FDEC_STRIDE,
-                             h->mb.pic.p_fdec[1] + 8*x + 8*y*FDEC_STRIDE,
-                             h->mb.pic.p_fdec[2] + 8*x + 8*y*FDEC_STRIDE};
+                             CHROMA_FORMAT ? h->mb.pic.p_fdec[1] + 8*x + 8*y*FDEC_STRIDE : NULL,
+                             CHROMA_FORMAT ? h->mb.pic.p_fdec[2] + 8*x + 8*y*FDEC_STRIDE : NULL};
             int cbp_luma_new = 0;
             int i_thresh = a->b_early_terminate ? a->i_satd_i8x8_dir[idx][a->i_predict8x8[idx]] * 11/8 : COST_MAX;
 
@@ -1207,30 +1207,40 @@ static void intra_rd_refine( x264_t *h, x264_mb_analysis_t *a )
     (m)->i_stride[1] = h->mb.pic.i_stride[1]; \
     (m)->i_stride[2] = h->mb.pic.i_stride[2]; \
     (m)->p_fenc[0] = &(src)[0][(xoff)+(yoff)*FENC_STRIDE]; \
-    (m)->p_fenc[1] = &(src)[1][((xoff)>>CHROMA_H_SHIFT)+((yoff)>>CHROMA_V_SHIFT)*FENC_STRIDE]; \
-    (m)->p_fenc[2] = &(src)[2][((xoff)>>CHROMA_H_SHIFT)+((yoff)>>CHROMA_V_SHIFT)*FENC_STRIDE]; \
+    if( CHROMA_FORMAT ) \
+    { \
+        (m)->p_fenc[1] = &(src)[1][((xoff)>>CHROMA_H_SHIFT)+((yoff)>>CHROMA_V_SHIFT)*FENC_STRIDE]; \
+        (m)->p_fenc[2] = &(src)[2][((xoff)>>CHROMA_H_SHIFT)+((yoff)>>CHROMA_V_SHIFT)*FENC_STRIDE]; \
+    } \
 }
 
 #define LOAD_HPELS(m, src, list, ref, xoff, yoff) \
 { \
     (m)->p_fref_w = (m)->p_fref[0] = &(src)[0][(xoff)+(yoff)*(m)->i_stride[0]]; \
-    (m)->p_fref[1] = &(src)[1][(xoff)+(yoff)*(m)->i_stride[0]]; \
-    (m)->p_fref[2] = &(src)[2][(xoff)+(yoff)*(m)->i_stride[0]]; \
-    (m)->p_fref[3] = &(src)[3][(xoff)+(yoff)*(m)->i_stride[0]]; \
+    if( h->param.analyse.i_subpel_refine ) \
+    { \
+        (m)->p_fref[1] = &(src)[1][(xoff)+(yoff)*(m)->i_stride[0]]; \
+        (m)->p_fref[2] = &(src)[2][(xoff)+(yoff)*(m)->i_stride[0]]; \
+        (m)->p_fref[3] = &(src)[3][(xoff)+(yoff)*(m)->i_stride[0]]; \
+    } \
     if( CHROMA444 ) \
     { \
         (m)->p_fref[ 4] = &(src)[ 4][(xoff)+(yoff)*(m)->i_stride[1]]; \
-        (m)->p_fref[ 5] = &(src)[ 5][(xoff)+(yoff)*(m)->i_stride[1]]; \
-        (m)->p_fref[ 6] = &(src)[ 6][(xoff)+(yoff)*(m)->i_stride[1]]; \
-        (m)->p_fref[ 7] = &(src)[ 7][(xoff)+(yoff)*(m)->i_stride[1]]; \
         (m)->p_fref[ 8] = &(src)[ 8][(xoff)+(yoff)*(m)->i_stride[2]]; \
-        (m)->p_fref[ 9] = &(src)[ 9][(xoff)+(yoff)*(m)->i_stride[2]]; \
-        (m)->p_fref[10] = &(src)[10][(xoff)+(yoff)*(m)->i_stride[2]]; \
-        (m)->p_fref[11] = &(src)[11][(xoff)+(yoff)*(m)->i_stride[2]]; \
+        if( h->param.analyse.i_subpel_refine ) \
+        { \
+            (m)->p_fref[ 5] = &(src)[ 5][(xoff)+(yoff)*(m)->i_stride[1]]; \
+            (m)->p_fref[ 6] = &(src)[ 6][(xoff)+(yoff)*(m)->i_stride[1]]; \
+            (m)->p_fref[ 7] = &(src)[ 7][(xoff)+(yoff)*(m)->i_stride[1]]; \
+            (m)->p_fref[ 9] = &(src)[ 9][(xoff)+(yoff)*(m)->i_stride[2]]; \
+            (m)->p_fref[10] = &(src)[10][(xoff)+(yoff)*(m)->i_stride[2]]; \
+            (m)->p_fref[11] = &(src)[11][(xoff)+(yoff)*(m)->i_stride[2]]; \
+        } \
     } \
-    else \
+    else if( CHROMA_FORMAT ) \
         (m)->p_fref[4] = &(src)[4][(xoff)+((yoff)>>CHROMA_V_SHIFT)*(m)->i_stride[1]]; \
-    (m)->integral = &h->mb.pic.p_integral[list][ref][(xoff)+(yoff)*(m)->i_stride[0]]; \
+    if( h->param.analyse.i_me_method >= X264_ME_ESA ) \
+        (m)->integral = &h->mb.pic.p_integral[list][ref][(xoff)+(yoff)*(m)->i_stride[0]]; \
     (m)->weight = x264_weight_none; \
     (m)->i_ref = ref; \
 }

diff --git a/encoder/me.c b/encoder/me.c
index d69122498e3336f5a3d44f0c08562944a94a592e..95ca9e59e432001bc535a8d22d3db565190abeaf 100644 (file)
--- a/encoder/me.c
+++ b/encoder/me.c
@@ -1043,8 +1043,8 @@ static ALWAYS_INLINE void me_refine_bidir( x264_t *h, x264_me_t *m0, x264_me_t *
     int chroma_x = (8 >> CHROMA_H_SHIFT) * x;
     int chroma_y = (8 >> chroma_v_shift) * y;
     pixel *pix  = &h->mb.pic.p_fdec[0][8*x + 8*y*FDEC_STRIDE];
-    pixel *pixu = &h->mb.pic.p_fdec[1][chroma_x + chroma_y*FDEC_STRIDE];
-    pixel *pixv = &h->mb.pic.p_fdec[2][chroma_x + chroma_y*FDEC_STRIDE];
+    pixel *pixu = CHROMA_FORMAT ? &h->mb.pic.p_fdec[1][chroma_x + chroma_y*FDEC_STRIDE] : NULL;
+    pixel *pixv = CHROMA_FORMAT ? &h->mb.pic.p_fdec[2][chroma_x + chroma_y*FDEC_STRIDE] : NULL;
     int ref0 = h->mb.cache.ref[0][s8];
     int ref1 = h->mb.cache.ref[1][s8];
     const int mv0y_offset = chroma_v_shift & MB_INTERLACED & ref0 ? (h->mb.i_mb_y & 1)*4 - 2 : 0;
@@ -1256,11 +1256,16 @@ void x264_me_refine_qpel_rd( x264_t *h, x264_me_t *m, int i_lambda2, int i4, int
         pixu = &h->mb.pic.p_fdec[1][block_idx_xy_fdec[i4]];
         pixv = &h->mb.pic.p_fdec[2][block_idx_xy_fdec[i4]];
     }
-    else
+    else if( CHROMA_FORMAT )
     {
         pixu = &h->mb.pic.p_fdec[1][(i8>>1)*(8*FDEC_STRIDE>>chroma_v_shift)+(i8&1)*4];
         pixv = &h->mb.pic.p_fdec[2][(i8>>1)*(8*FDEC_STRIDE>>chroma_v_shift)+(i8&1)*4];
     }
+    else
+    {
+        pixu = NULL;
+        pixv = NULL;
+    }
 
     h->mb.b_skip_mc = 1;