libhb: fix comb detection crash and decomb crash

hb_detect_comb() could crash because the dimensions of the video buffer
don't have to match the dimensions returned by work object info() method
if the video has segments of differeing resolutions.

decomb was allocating reference buffers that were too small.  This bug appears
to have always existed but doesn't usually get triggered because malloc
usually rounds allocation sizes up.

git-svn-id: svn://svn.handbrake.fr/HandBrake/trunk@4812 b64f7644-9d1e-0410-96f1-a4d463321fa5

diff --git a/libhb/decomb.c b/libhb/decomb.c
index 40bcb2a6c7920b1452bc81fe4c520970c039098d..e9519700f06836840a0c14f26f5e3cdf4d10a405 100644 (file)
--- a/libhb/decomb.c
+++ b/libhb/decomb.c
@@ -1997,8 +1997,8 @@ static int hb_decomb_init( hb_filter_object_t * filter,
     for( i = 0; i < 3; i++ )
     {
         int is_chroma = !!i;
-        int w = ((init->width   + 31) & (~31))>>is_chroma;
-        int h = ((init->height+6+ 31) & (~31))>>is_chroma;
+        int w = ((init->width >>is_chroma)  + 31) & (~31);
+        int h = ((init->height>>is_chroma)+6+ 31) & (~31);
 
         pv->ref_stride[i] = w;
 
@@ -2012,8 +2012,8 @@ static int hb_decomb_init( hb_filter_object_t * filter,
     for( i = 0; i < 3; i++ )
     {
         int is_chroma = !!i;
-        int w = ((init->width   + 31) & (~31))>>is_chroma;
-        int h = ((init->height+6+ 31) & (~31))>>is_chroma;
+        int w = ((init->width >>is_chroma)  + 31) & (~31);
+        int h = ((init->height>>is_chroma)+6+ 31) & (~31);
 
         pv->mask[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w;
         pv->mask_filtered[i] = calloc( 1, w*h*sizeof(uint8_t) ) + 3*w;
@@ -2027,8 +2027,8 @@ static int hb_decomb_init( hb_filter_object_t * filter,
         for( i = 0; i < 3; i++ )
         {
             int is_chroma = !!i;
-            int w = ((init->width   + 31) & (~31))>>is_chroma;
-            int h = ((height+6+ 31) & (~31))>>is_chroma;
+            int w = ((init->width>>is_chroma)   + 31) & (~31);
+            int h = ((height>>is_chroma)    + 6 + 31) & (~31);
 
             for( j = 0; j < 4; j++ )
             {
@@ -2037,12 +2037,11 @@ static int hb_decomb_init( hb_filter_object_t * filter,
         }
 
         /* Allocate full-height eedi2 buffers */
-        height = init->height;
         for( i = 0; i < 3; i++ )
         {
             int is_chroma = !!i;
-            int w = ((init->width   + 31) & (~31))>>is_chroma;
-            int h = ((height+6+ 31) & (~31))>>is_chroma;
+            int w = ((init->width >>is_chroma)  + 31) & (~31);
+            int h = ((init->height>>is_chroma)+6+ 31) & (~31);
 
             for( j = 0; j < 5; j++ )
             {

diff --git a/libhb/hb.c b/libhb/hb.c
index 0ad101aafae760f607ecf1e8444f3325afebe64b..9e331f0b3e09f9358900feb957132b289619e28a 100644 (file)
--- a/libhb/hb.c
+++ b/libhb/hb.c
@@ -865,15 +865,13 @@ void hb_get_preview( hb_handle_t * h, hb_title_t * title, int picture,
  * @param prog_diff   Sensitivity for detecting different colors on progressive frames
  * @param prog_threshold Sensitivity for flagging progressive frames as combed
  */
-int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, int color_diff, int threshold, int prog_equal, int prog_diff, int prog_threshold )
+int hb_detect_comb( hb_buffer_t * buf, int color_equal, int color_diff, int threshold, int prog_equal, int prog_diff, int prog_threshold )
 {
     int j, k, n, off, cc_1, cc_2, cc[3];
        // int flag[3] ; // debugging flag
     uint16_t s1, s2, s3, s4;
     cc_1 = 0; cc_2 = 0;
 
-    int offset = 0;
-
     if ( buf->s.flags & 16 )
     {
         /* Frame is progressive, be more discerning. */
@@ -885,21 +883,10 @@ int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, i
     /* One pas for Y, one pass for Cb, one pass for Cr */    
     for( k = 0; k < 3; k++ )
     {
-        if( k == 1 )
-        {
-            /* Y has already been checked, now offset by Y's dimensions
-               and divide all the other values by 2, since Cr and Cb
-               are half-size compared to Y.                               */
-            offset = width * height;
-            width >>= 1;
-            height >>= 1;
-        }
-        else if ( k == 2 )
-        {
-            /* Y and Cb are done, so the offset needs to be bumped
-               so it's width*height + (width / 2) * (height / 2)  */
-            offset *= 5/4;
-        }
+        uint8_t * data = buf->plane[k].data;
+        int width = buf->plane[k].width;
+        int stride = buf->plane[k].stride;
+        int height = buf->plane[k].height;
 
         for( j = 0; j < width; ++j )
         {
@@ -908,10 +895,10 @@ int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, i
             for( n = 0; n < ( height - 4 ); n = n + 2 )
             {
                 /* Look at groups of 4 sequential horizontal lines */
-                s1 = ( ( buf->data + offset )[ off + j             ] & 0xff );
-                s2 = ( ( buf->data + offset )[ off + j + width     ] & 0xff );
-                s3 = ( ( buf->data + offset )[ off + j + 2 * width ] & 0xff );
-                s4 = ( ( buf->data + offset )[ off + j + 3 * width ] & 0xff );
+                s1 = ( ( data )[ off + j              ] & 0xff );
+                s2 = ( ( data )[ off + j +     stride ] & 0xff );
+                s3 = ( ( data )[ off + j + 2 * stride ] & 0xff );
+                s4 = ( ( data )[ off + j + 3 * stride ] & 0xff );
 
                 /* Note if the 1st and 2nd lines are more different in
                    color than the 1st and 3rd lines are similar in color.*/
@@ -926,7 +913,7 @@ int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, i
                         ++cc_2;
 
                 /* Now move down 2 horizontal lines before starting over.*/
-                off += 2 * width;
+                off += 2 * stride;
             }
         }
 

diff --git a/libhb/hb.h b/libhb/hb.h
index a594d93fc1d3f82b2640b118559fcec315face36..da23118341949941a392f668e1239fdae7049328 100644 (file)
--- a/libhb/hb.h
+++ b/libhb/hb.h
@@ -57,7 +57,7 @@ hb_list_t   * hb_get_titles( hb_handle_t * );
 /* hb_detect_comb()
    Analyze a frame for interlacing artifacts, returns true if they're found.
    Taken from Thomas Oestreich's 32detect filter in the Transcode project.  */
-int hb_detect_comb( hb_buffer_t * buf, int width, int height, int color_equal, int color_diff, int threshold, int prog_equal, int prog_diff, int prog_threshold );
+int hb_detect_comb( hb_buffer_t * buf, int color_equal, int color_diff, int threshold, int prog_equal, int prog_diff, int prog_threshold );
 
 int           hb_save_preview( hb_handle_t * h, int title, int preview, 
                                hb_buffer_t *buf );

diff --git a/libhb/scan.c b/libhb/scan.c
index e06f14c5f536354dcecf3b6a5da8dee93b0f45c8..4776e08c4f7804483030fdc57665926f7155e0f3 100644 (file)
--- a/libhb/scan.c
+++ b/libhb/scan.c
@@ -739,7 +739,7 @@ static int DecodePreviews( hb_scan_t * data, hb_title_t * title )
         }
 
         /* Check preview for interlacing artifacts */
-        if( hb_detect_comb( vid_buf, vid_info.width, vid_info.height, 10, 30, 9, 10, 30, 9 ) )
+        if( hb_detect_comb( vid_buf, 10, 30, 9, 10, 30, 9 ) )
         {
             hb_deep_log( 2, "Interlacing detected in preview frame %i", i+1);
             interlaced_preview_count++;