Try 8x8 transform analysis even when sub8x8 partitions are present

Turn off the sub8x8 partitions, try it, and turn them back on if it didn't help.
Small compression improvement with p4x4 on (~0.1-0.5%).
Also update related comments.

diff --git a/common/macroblock.h b/common/macroblock.h
index df2f8d93235378a1485614b0cc6c5c95b18ba3c5..4d932dfff4b5631353dbbda46c4be441ff8747fb 100644 (file)
--- a/common/macroblock.h
+++ b/common/macroblock.h
@@ -420,23 +420,23 @@ static ALWAYS_INLINE int x264_mb_predict_non_zero_code( x264_t *h, int idx )
         i_ret = ( i_ret + 1 ) >> 1;
     return i_ret & 0x7f;
 }
+
+/* intra and skip are disallowed, p8x8 is conditional. */
+static const uint8_t x264_transform_allowed[X264_MBTYPE_MAX] =
+{
+    0,0,0,0,1,2,0,1,1,1,1,1,1,1,1,1,1,1,0
+};
+
 /* x264_mb_transform_8x8_allowed:
  *      check whether any partition is smaller than 8x8 (or at least
  *      might be, according to just partition type.)
  *      doesn't check for cbp */
 static ALWAYS_INLINE int x264_mb_transform_8x8_allowed( x264_t *h )
 {
-    // intra and skip are disallowed
-    // large partitions are allowed
-    // direct and 8x8 are conditional
-    static const uint8_t partition_tab[X264_MBTYPE_MAX] = {
-        0,0,0,0,1,2,0,1,1,1,1,1,1,1,1,1,1,1,0,
-    };
-
     if( !h->pps->b_transform_8x8_mode )
         return 0;
     if( h->mb.i_type != P_8x8 )
-        return partition_tab[h->mb.i_type];
+        return x264_transform_allowed[h->mb.i_type];
     return M32( h->mb.i_sub_partition ) == D_L0_8x8*0x01010101;
 }
 

diff --git a/encoder/analyse.c b/encoder/analyse.c
index ee61a0035071a97aeff0e879fc022b6fce826bbd..b5c7f576eb9995ec8c986908af284f1ec0c80756 100644 (file)
--- a/encoder/analyse.c
+++ b/encoder/analyse.c
@@ -2851,8 +2851,15 @@ static inline void x264_mb_analyse_transform( x264_t *h )
 
 static inline void x264_mb_analyse_transform_rd( x264_t *h, x264_mb_analysis_t *a, int *i_satd, int *i_rd )
 {
-    if( x264_mb_transform_8x8_allowed( h ) && h->param.analyse.b_transform_8x8 )
+    if( h->param.analyse.b_transform_8x8 && h->pps->b_transform_8x8_mode )
     {
+        uint32_t subpart_bak = M32( h->mb.i_sub_partition );
+        /* Try switching the subpartitions to 8x8 so that we can use 8x8 transform mode */
+        if( h->mb.i_type == P_8x8 )
+            M32( h->mb.i_sub_partition ) = D_L0_8x8*0x01010101;
+        else if( !x264_transform_allowed[h->mb.i_type] )
+            return;
+
         x264_analyse_update_cache( h, a );
         h->mb.b_transform_8x8 ^= 1;
         /* FIXME only luma is needed for 4:2:0, but the score for comparison already includes chroma */
@@ -2865,7 +2872,10 @@ static inline void x264_mb_analyse_transform_rd( x264_t *h, x264_mb_analysis_t *
             *i_rd = i_rd8;
         }
         else
+        {
             h->mb.b_transform_8x8 ^= 1;
+            M32( h->mb.i_sub_partition ) = subpart_bak;
+        }
     }
 }