From: Loren Merritt <pengvado@videolan.org>
Date: Mon, 23 Apr 2007 08:34:52 +0000 (+0000)
Subject: fix nnz computation in cavlc+8x8dct+deblock. (regression in r607)
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=b5ef788a7c6d2c7ba2cdb664ffd5544603979495;p=libx264

fix nnz computation in cavlc+8x8dct+deblock. (regression in r607)


git-svn-id: svn://svn.videolan.org/x264/trunk@654 df754926-b1dd-0310-bc7b-ec298dee348c
---

diff --git a/common/common.h b/common/common.h
index fadc3664..88879737 100644
--- a/common/common.h
+++ b/common/common.h
@@ -474,6 +474,7 @@ struct x264_t
         int8_t  *skipbp;                    /* block pattern for SKIP or DIRECT (sub)mbs. B-frames + cabac only */
         int8_t  *mb_transform_size;         /* transform_size_8x8_flag of each mb */
         uint8_t *intra_border_backup[2][3]; /* bottom pixels of the previous mb row, used for intra prediction after the framebuffer has been deblocked */
+        uint8_t (*nnz_backup)[16];          /* when using cavlc + 8x8dct, the deblocker uses a modified nnz */
 
         /* current value */
         int     i_type;
diff --git a/common/frame.c b/common/frame.c
index 04c0fd6d..e5a1b911 100644
--- a/common/frame.c
+++ b/common/frame.c
@@ -294,6 +294,48 @@ void x264_frame_expand_border_mod16( x264_t *h, x264_frame_t *frame )
 }
 
 
+/* cavlc + 8x8 transform stores nnz per 16 coeffs for the purpose of
+ * entropy coding, but per 64 coeffs for the purpose of deblocking */
+void munge_cavlc_nnz_row( x264_t *h, int mb_y, uint8_t (*buf)[16] )
+{
+    uint32_t (*src)[6] = (uint32_t(*)[6])h->mb.non_zero_count + mb_y * h->sps->i_mb_width;
+    int8_t *transform = h->mb.mb_transform_size + mb_y * h->sps->i_mb_width;
+    int x;
+    for( x=0; x<h->sps->i_mb_width; x++ )
+    {
+        memcpy( buf+x, src+x, 16 );
+        if( transform[x] )
+        {
+            if( src[x][0] ) src[x][0] = 0x01010101;
+            if( src[x][1] ) src[x][1] = 0x01010101;
+            if( src[x][2] ) src[x][2] = 0x01010101;
+            if( src[x][3] ) src[x][3] = 0x01010101;
+        }
+    }
+}
+
+static void restore_cavlc_nnz_row( x264_t *h, int mb_y, uint8_t (*buf)[16] )
+{
+    uint8_t (*dst)[24] = h->mb.non_zero_count + mb_y * h->sps->i_mb_width;
+    int x;
+    for( x=0; x<h->sps->i_mb_width; x++ )
+        memcpy( dst+x, buf+x, 16 );
+}
+
+static void munge_cavlc_nnz( x264_t *h, int mb_y, uint8_t (*buf)[16], void (*func)(x264_t*, int, uint8_t (*)[16]) )
+{
+    func( h, mb_y, buf );
+    if( mb_y > 0 )
+        func( h, mb_y-1, buf + h->sps->i_mb_width );
+    if( h->sh.b_mbaff )
+    {
+        func( h, mb_y+1, buf + h->sps->i_mb_width * 2 );
+        if( mb_y > 0 )
+            func( h, mb_y-2, buf + h->sps->i_mb_width * 3 );
+    }
+}
+
+
 /* Deblocking filter */
 
 static const int i_alpha_table[52] =
@@ -536,6 +578,9 @@ void x264_frame_deblock_row( x264_t *h, int mb_y )
                          h->fdec->i_stride[1] << b_interlaced,
                          h->fdec->i_stride[2] << b_interlaced };
 
+    if( !h->pps->b_cabac && h->pps->b_transform_8x8_mode )
+        munge_cavlc_nnz( h, mb_y, h->mb.nnz_backup, munge_cavlc_nnz_row );
+
     for( mb_x = 0; mb_x < h->sps->i_mb_width; )
     {
         const int mb_xy  = mb_y * h->mb.i_mb_stride + mb_x;
@@ -557,17 +602,6 @@ void x264_frame_deblock_row( x264_t *h, int mb_y )
 
         x264_prefetch_fenc( h, h->fdec, mb_x, mb_y );
 
-        /* cavlc + 8x8 transform stores nnz per 16 coeffs for the purpose of
-         * entropy coding, but per 64 coeffs for the purpose of deblocking */
-        if( !h->param.b_cabac && b_8x8_transform )
-        {
-            uint32_t *nnz = (uint32_t*)h->mb.non_zero_count[mb_xy];
-            if( nnz[0] ) nnz[0] = 0x01010101;
-            if( nnz[1] ) nnz[1] = 0x01010101;
-            if( nnz[2] ) nnz[2] = 0x01010101;
-            if( nnz[3] ) nnz[3] = 0x01010101;
-        }
-
         /* i_dir == 0 -> vertical edge
          * i_dir == 1 -> horizontal edge */
         for( i_dir = 0; i_dir < 2; i_dir++ )
@@ -694,6 +728,9 @@ void x264_frame_deblock_row( x264_t *h, int mb_y )
             mb_x++;
         mb_y ^= b_interlaced;
     }
+
+    if( !h->pps->b_cabac && h->pps->b_transform_8x8_mode )
+        munge_cavlc_nnz( h, mb_y, h->mb.nnz_backup, restore_cavlc_nnz_row );
 }
 
 void x264_frame_deblock( x264_t *h )
diff --git a/common/macroblock.c b/common/macroblock.c
index 5f3089ec..3ecda66c 100644
--- a/common/macroblock.c
+++ b/common/macroblock.c
@@ -880,6 +880,7 @@ int x264_macroblock_cache_init( x264_t *h )
 
     /* all coeffs */
     CHECKED_MALLOC( h->mb.non_zero_count, i_mb_count * 24 * sizeof(uint8_t) );
+    CHECKED_MALLOC( h->mb.nnz_backup, h->sps->i_mb_width * 4 * 16 * sizeof(uint8_t) );
 
     if( h->param.b_cabac )
     {
@@ -929,6 +930,7 @@ void x264_macroblock_cache_end( x264_t *h )
     }
     x264_free( h->mb.intra4x4_pred_mode );
     x264_free( h->mb.non_zero_count );
+    x264_free( h->mb.nnz_backup );
     x264_free( h->mb.mb_transform_size );
     x264_free( h->mb.skipbp );
     x264_free( h->mb.cbp );