Fix regression in r1066

With some combinations of video width and other settings, the scratch buffer was slightly too small.
This caused heap corruption on some systems.
Also prevent merange from being raised during encoding with esa/tesa through encoder_reconfig, as this no longer works.

diff --git a/common/macroblock.c b/common/macroblock.c
index 94a5d4fca3ed457ea9b59bb948db6d02165fe96c..fed6e2b4f4b67cebf67aadc6415d644fec63e620 100644 (file)
--- a/common/macroblock.c
+++ b/common/macroblock.c
@@ -839,11 +839,11 @@ int x264_macroblock_cache_init( x264_t *h )
     h->mb.i_neighbour4[15] =
     h->mb.i_neighbour8[3] = MB_LEFT|MB_TOP|MB_TOPLEFT;
 
-    int buf_hpel = (h->param.i_width+40) * sizeof(int16_t);
+    int buf_hpel = (h->param.i_width+48) * sizeof(int16_t);
     int buf_ssim = h->param.analyse.b_ssim * 8 * (h->param.i_width/4+3) * sizeof(int);
     int me_range = X264_MIN(h->param.analyse.i_me_range, h->param.analyse.i_mv_range);
     int buf_tesa = (h->param.analyse.i_me_method >= X264_ME_ESA) *
-        ((me_range*2+18) * sizeof(int16_t) + (me_range+1) * (me_range+1) * 4 * sizeof(mvsad_t));
+        ((me_range*2+18) * sizeof(int16_t) + (me_range+4) * (me_range+1) * 4 * sizeof(mvsad_t));
     CHECKED_MALLOC( h->scratch_buffer, X264_MAX3( buf_hpel, buf_ssim, buf_tesa ) );
 
     return 0;

diff --git a/encoder/encoder.c b/encoder/encoder.c
index d682db729a6cef5f56e50d79350935f6354a2132..132b26de368adb18c1df5af4ee4fe18a7138e1c5 100644 (file)
--- a/encoder/encoder.c
+++ b/encoder/encoder.c
@@ -826,7 +826,9 @@ int x264_encoder_reconfig( x264_t *h, x264_param_t *param )
     COPY( analyse.intra );
     COPY( analyse.inter );
     COPY( analyse.i_direct_mv_pred );
-    COPY( analyse.i_me_range );
+    /* Scratch buffer prevents me_range from being increased for esa/tesa */
+    if( h->param.analyse.i_me_method < X264_ME_ESA || param->analyse.i_me_range < h->param.analyse.i_me_range )
+        COPY( analyse.i_me_range );
     COPY( analyse.i_noise_reduction );
     /* We can't switch out of subme=0 during encoding. */
     if( h->param.analyse.i_subpel_refine )