Check minimum chroma QP in addition to luma QP during CQM init

Correctly error out if the implied minimum chroma QP is too low.
Add missing emms to checkasm macroblock_tree_propagate test.

diff --git a/common/set.c b/common/set.c
index 7d4d23fc94ea4e186602878909aa3dc7bcfbe012..1f53cf380add903ab8176ff332cded30bdcefe71 100644 (file)
--- a/common/set.c
+++ b/common/set.c
@@ -78,6 +78,7 @@ int x264_cqm_init( x264_t *h )
                         32 - h->param.analyse.i_luma_deadzone[0],
                         32 - 11, 32 - 21 };
     int max_qp_err = -1;
+    int max_chroma_qp_err = -1;
 
     for( i = 0; i < 6; i++ )
     {
@@ -148,8 +149,10 @@ int x264_cqm_init( x264_t *h )
                 h->  quant4_mf[i_list][q][i] = j = SHIFT(quant4_mf[i_list][q%6][0][i], q/6 - 1);
                 // round to nearest, unless that would cause the deadzone to be negative
                 h->quant4_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j );
-                if( j > 0xffff && q > max_qp_err )
+                if( j > 0xffff && q > max_qp_err && (i_list == CQM_4IY || i_list == CQM_4PY) )
                     max_qp_err = q;
+                if( j > 0xffff && q > max_chroma_qp_err && (i_list == CQM_4IC || i_list == CQM_4PC) )
+                    max_chroma_qp_err = q;
             }
         if( h->param.analyse.b_transform_8x8 )
         for( i_list = 0; i_list < 2; i_list++ )
@@ -165,9 +168,14 @@ int x264_cqm_init( x264_t *h )
 
     if( !h->mb.b_lossless && max_qp_err >= h->param.rc.i_qp_min )
     {
-        x264_log( h, X264_LOG_ERROR, "Quantization overflow.\n" );
-        x264_log( h, X264_LOG_ERROR, "Your CQM is incompatible with QP < %d, but min QP is set to %d\n",
-                  max_qp_err+1, h->param.rc.i_qp_min );
+        x264_log( h, X264_LOG_ERROR, "Quantization overflow.  Your CQM is incompatible with QP < %d,\n", max_qp_err+1 );
+        x264_log( h, X264_LOG_ERROR, "but min QP is set to %d.\n", h->param.rc.i_qp_min );
+        return -1;
+    }
+    if( !h->mb.b_lossless && max_chroma_qp_err >= h->chroma_qp_table[h->param.rc.i_qp_min] )
+    {
+        x264_log( h, X264_LOG_ERROR, "Quantization overflow.  Your CQM is incompatible with QP < %d,\n", max_chroma_qp_err+1 );
+        x264_log( h, X264_LOG_ERROR, "but min chroma QP is implied to be %d.\n", h->chroma_qp_table[h->param.rc.i_qp_min] );
         return -1;
     }
     return 0;

diff --git a/encoder/encoder.c b/encoder/encoder.c
index c87ae905b88477b85486b5ba767780537cdeb33b..841bfe9d58506d52343a839b94ca6dcad72bd104 100644 (file)
--- a/encoder/encoder.c
+++ b/encoder/encoder.c
@@ -762,6 +762,8 @@ x264_t *x264_encoder_open   ( x264_param_t *param )
 
     x264_validate_levels( h, 1 );
 
+    h->chroma_qp_table = i_chroma_qp_table + 12 + h->pps->i_chroma_qp_index_offset;
+
     if( x264_cqm_init( h ) < 0 )
         goto fail;
 
@@ -796,8 +798,6 @@ x264_t *x264_encoder_open   ( x264_param_t *param )
     h->i_ref0 = 0;
     h->i_ref1 = 0;
 
-    h->chroma_qp_table = i_chroma_qp_table + 12 + h->pps->i_chroma_qp_index_offset;
-
     x264_rdo_init( );
 
     /* init CPU functions */

diff --git a/tools/checkasm.c b/tools/checkasm.c
index 291c162162c271a7aa7217f73ed4abe0389df7be..ddb551766a1986f70975e8e4b20634704540bd7f 100644 (file)
--- a/tools/checkasm.c
+++ b/tools/checkasm.c
@@ -515,6 +515,7 @@ static int check_dct( int cpu_ref, int cpu_new )
     memset( h, 0, sizeof(*h) );
     h->pps = h->pps_array;
     x264_param_default( &h->param );
+    h->chroma_qp_table = i_chroma_qp_table + 12;
     h->param.analyse.i_luma_deadzone[0] = 0;
     h->param.analyse.i_luma_deadzone[1] = 0;
     h->param.analyse.b_transform_8x8 = 1;
@@ -978,6 +979,7 @@ static int check_mc( int cpu_ref, int cpu_new )
         uint16_t *inter = intra+400;
         uint16_t *qscale = inter+400;
         uint16_t *rand = (uint16_t*)buf2;
+        x264_emms();
         for( i=0; i<400; i++ )
         {
             intra[i]  = *rand++ & 0x7fff;
@@ -1077,6 +1079,7 @@ static int check_quant( int cpu_ref, int cpu_new )
     memset( h, 0, sizeof(*h) );
     h->pps = h->pps_array;
     x264_param_default( &h->param );
+    h->chroma_qp_table = i_chroma_qp_table + 12;
     h->param.rc.i_qp_min = 26;
     h->param.analyse.b_transform_8x8 = 1;