From: Loren Merritt <pengvado@akuvian.org>
Date: Thu, 13 Aug 2009 05:02:59 +0000 (+0000)
Subject: fix lowres padding, which failed to extrapolate the right side for some resolutions.
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=f52973d2e93810bc86b6e8e3358d0365b97a409d;p=libx264

fix lowres padding, which failed to extrapolate the right side for some resolutions.
fix a buffer overread in x264_mbtree_propagate_cost_sse2. no effect on actual behavior, only theoretical correctness.
fix x264_slicetype_frame_cost_recalculate on I-frames, which previously used all 0 mb costs.
shut up a valgrind warning in predict_8x8_filter_mmx.
---

diff --git a/common/frame.c b/common/frame.c
index 90dd0705..b08c70fc 100644
--- a/common/frame.c
+++ b/common/frame.c
@@ -93,15 +93,15 @@ x264_frame_t *x264_frame_new( x264_t *h )
                 CHECKED_MALLOCZERO( frame->lowres_mvs[j][i], 2*h->mb.i_mb_count*sizeof(int16_t) );
                 CHECKED_MALLOC( frame->lowres_mv_costs[j][i], h->mb.i_mb_count*sizeof(int) );
             }
-        CHECKED_MALLOC( frame->i_intra_cost, i_mb_count * sizeof(uint16_t) );
-        memset( frame->i_intra_cost, -1, i_mb_count * sizeof(uint16_t) );
-        CHECKED_MALLOC( frame->i_propagate_cost, i_mb_count * sizeof(uint16_t) );
+        CHECKED_MALLOC( frame->i_propagate_cost, (i_mb_count+3) * sizeof(uint16_t) );
         for( j = 0; j <= h->param.i_bframe+1; j++ )
             for( i = 0; i <= h->param.i_bframe+1; i++ )
             {
-                CHECKED_MALLOC( frame->lowres_costs[j][i], i_mb_count * sizeof(uint16_t) );
-                CHECKED_MALLOC( frame->lowres_inter_types[j][i], i_mb_count * sizeof(uint8_t) );
+                CHECKED_MALLOC( frame->lowres_costs[j][i], (i_mb_count+3) * sizeof(uint16_t) );
+                CHECKED_MALLOC( frame->lowres_inter_types[j][i], (i_mb_count+3) * sizeof(uint8_t) );
             }
+        frame->i_intra_cost = frame->lowres_costs[0][0];
+        memset( frame->i_intra_cost, -1, (i_mb_count+3) * sizeof(uint16_t) );
     }
 
     if( h->param.analyse.i_me_method >= X264_ME_ESA )
@@ -143,7 +143,8 @@ x264_frame_t *x264_frame_new( x264_t *h )
     {
         CHECKED_MALLOC( frame->f_qp_offset, h->mb.i_mb_count * sizeof(float) );
         if( h->frames.b_have_lowres )
-            CHECKED_MALLOC( frame->i_inv_qscale_factor, h->mb.i_mb_count * sizeof(uint16_t) );
+            /* shouldn't really be initialized, just silences a valgrind false-positive in x264_mbtree_propagate_cost_sse2 */
+            CHECKED_MALLOCZERO( frame->i_inv_qscale_factor, (h->mb.i_mb_count+3) * sizeof(uint16_t) );
     }
 
     if( x264_pthread_mutex_init( &frame->mutex, NULL ) )
@@ -183,7 +184,6 @@ void x264_frame_delete( x264_frame_t *frame )
         }
     x264_free( frame->f_qp_offset );
     x264_free( frame->i_inv_qscale_factor );
-    x264_free( frame->i_intra_cost );
     x264_free( frame->i_row_bits );
     x264_free( frame->i_row_qp );
     x264_free( frame->mb_type );
@@ -312,7 +312,7 @@ void x264_frame_expand_border_lowres( x264_frame_t *frame )
 {
     int i;
     for( i = 0; i < 4; i++ )
-        plane_expand_border( frame->lowres[i], frame->i_stride_lowres, frame->i_stride_lowres - 2*PADH, frame->i_lines_lowres, PADH, PADV, 1, 1 );
+        plane_expand_border( frame->lowres[i], frame->i_stride_lowres, frame->i_width_lowres, frame->i_lines_lowres, PADH, PADV, 1, 1 );
 }
 
 void x264_frame_expand_border_mod16( x264_t *h, x264_frame_t *frame )
diff --git a/common/macroblock.c b/common/macroblock.c
index cd66c717..4b282285 100644
--- a/common/macroblock.c
+++ b/common/macroblock.c
@@ -702,7 +702,8 @@ int x264_macroblock_cache_init( x264_t *h )
     for( i=0; i<=h->param.b_interlaced; i++ )
         for( j=0; j<3; j++ )
         {
-            CHECKED_MALLOC( h->mb.intra_border_backup[i][j], h->fdec->i_stride[j] );
+            /* shouldn't really be initialized, just silences a valgrind false-positive in predict_8x8_filter_mmx */
+            CHECKED_MALLOCZERO( h->mb.intra_border_backup[i][j], h->fdec->i_stride[j] );
             h->mb.intra_border_backup[i][j] += 8;
         }
 
diff --git a/common/x86/mc-a2.asm b/common/x86/mc-a2.asm
index ced38173..2af47120 100644
--- a/common/x86/mc-a2.asm
+++ b/common/x86/mc-a2.asm
@@ -1090,10 +1090,10 @@ FRAME_INIT_LOWRES ssse3, 12
 cglobal x264_mbtree_propagate_cost_sse2, 6,6
     shl r5d, 1
     lea r0, [r0+r5*2]
-    lea r1, [r1+r5]
-    lea r2, [r2+r5]
-    lea r3, [r3+r5]
-    lea r4, [r4+r5]
+    add r1, r5
+    add r2, r5
+    add r3, r5
+    add r4, r5
     neg r5
     pxor      xmm5, xmm5
     movdqa    xmm4, [pd_128 GLOBAL]
diff --git a/encoder/slicetype.c b/encoder/slicetype.c
index da5ae454..6ef43203 100644
--- a/encoder/slicetype.c
+++ b/encoder/slicetype.c
@@ -261,7 +261,7 @@ lowres_intra_mb:
         }
     }
 
-    frames[b]->lowres_costs[b-p0][p1-b][i_mb_xy] = i_bcost;
+    fenc->lowres_costs[b-p0][p1-b][i_mb_xy] = i_bcost;
 
     return i_bcost;
 }