From: Anton Mitrofanov <BugMaster@narod.ru>
Date: Tue, 8 Oct 2013 19:32:37 +0000 (+0400)
Subject: Fix possible crashes in resize and crop filters with high bitdepth input
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=50a0c33b9b5fa57d0a129b7441a6af55f7a08005;p=libx264

Fix possible crashes in resize and crop filters with high bitdepth input
---

diff --git a/filters/video/crop.c b/filters/video/crop.c
index a58813b7..98faab74 100644
--- a/filters/video/crop.c
+++ b/filters/video/crop.c
@@ -105,8 +105,7 @@ static int get_frame( hnd_t handle, cli_pic_t *output, int frame )
     for( int i = 0; i < output->img.planes; i++ )
     {
         intptr_t offset = output->img.stride[i] * h->dims[1] * h->csp->height[i];
-        offset += h->dims[0] * h->csp->width[i];
-        offset *= x264_cli_csp_depth_factor( output->img.csp );
+        offset += h->dims[0] * h->csp->width[i] * x264_cli_csp_depth_factor( output->img.csp );
         output->img.plane[i] += offset;
     }
     return 0;
diff --git a/filters/video/resize.c b/filters/video/resize.c
index 197a0e38..19747105 100644
--- a/filters/video/resize.c
+++ b/filters/video/resize.c
@@ -392,7 +392,7 @@ static int check_resizer( resizer_hnd_t *h, cli_pic_t *in )
     h->scale = input_prop;
     if( !h->buffer_allocated )
     {
-        if( x264_cli_pic_alloc( &h->buffer, h->dst_csp, h->dst.width, h->dst.height ) )
+        if( x264_cli_pic_alloc_aligned( &h->buffer, h->dst_csp, h->dst.width, h->dst.height ) )
             return -1;
         h->buffer_allocated = 1;
     }
diff --git a/input/input.c b/input/input.c
index 9c6763d2..5cb277cf 100644
--- a/input/input.c
+++ b/input/input.c
@@ -74,7 +74,7 @@ uint64_t x264_cli_pic_size( int csp, int width, int height )
     return size;
 }
 
-int x264_cli_pic_alloc( cli_pic_t *pic, int csp, int width, int height )
+static int x264_cli_pic_alloc_internal( cli_pic_t *pic, int csp, int width, int height, int align )
 {
     memset( pic, 0, sizeof(cli_pic_t) );
     int csp_mask = csp & X264_CSP_MASK;
@@ -87,15 +87,29 @@ int x264_cli_pic_alloc( cli_pic_t *pic, int csp, int width, int height )
     pic->img.height = height;
     for( int i = 0; i < pic->img.planes; i++ )
     {
-         pic->img.plane[i] = x264_malloc( x264_cli_pic_plane_size( csp, width, height, i ) );
-         if( !pic->img.plane[i] )
-             return -1;
-         pic->img.stride[i] = width * x264_cli_csps[csp_mask].width[i] * x264_cli_csp_depth_factor( csp );
+        int stride = width * x264_cli_csps[csp_mask].width[i];
+        stride *= x264_cli_csp_depth_factor( csp );
+        stride = ALIGN( stride, align );
+        uint64_t size = (uint64_t)(height * x264_cli_csps[csp_mask].height[i]) * stride;
+        pic->img.plane[i] = x264_malloc( size );
+        if( !pic->img.plane[i] )
+            return -1;
+        pic->img.stride[i] = stride;
     }
 
     return 0;
 }
 
+int x264_cli_pic_alloc( cli_pic_t *pic, int csp, int width, int height )
+{
+    return x264_cli_pic_alloc_internal( pic, csp, width, height, 1 );
+}
+
+int x264_cli_pic_alloc_aligned( cli_pic_t *pic, int csp, int width, int height )
+{
+    return x264_cli_pic_alloc_internal( pic, csp, width, height, NATIVE_ALIGN );
+}
+
 void x264_cli_pic_clean( cli_pic_t *pic )
 {
     for( int i = 0; i < pic->img.planes; i++ )
diff --git a/input/input.h b/input/input.h
index 5137be31..a33d22a9 100644
--- a/input/input.h
+++ b/input/input.h
@@ -124,6 +124,7 @@ extern const x264_cli_csp_t x264_cli_csps[];
 int      x264_cli_csp_is_invalid( int csp );
 int      x264_cli_csp_depth_factor( int csp );
 int      x264_cli_pic_alloc( cli_pic_t *pic, int csp, int width, int height );
+int      x264_cli_pic_alloc_aligned( cli_pic_t *pic, int csp, int width, int height );
 void     x264_cli_pic_clean( cli_pic_t *pic );
 uint64_t x264_cli_pic_plane_size( int csp, int width, int height, int plane );
 uint64_t x264_cli_pic_size( int csp, int width, int height );