From: Fiona Glaser <fiona@x264.com>
Date: Mon, 24 Mar 2008 09:25:25 +0000 (-0600)
Subject: higher precision RD lambda
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=9289e80611d89ad4050fa738dec9a530c8f4e3d4;p=libx264

higher precision RD lambda
improves quality at QP<=12.
---

diff --git a/encoder/analyse.c b/encoder/analyse.c
index bde51e82..9113091a 100644
--- a/encoder/analyse.c
+++ b/encoder/analyse.c
@@ -138,17 +138,15 @@ static const int i_qp0_cost_table[52] = {
   40,45,51,57,64,72,81,91   /* 44-51 */
 };
 
-/* pow(lambda,2) * .9 */
+/* lambda2 = pow(lambda,2) * .9 * 256 */
 static const int i_qp0_cost2_table[52] = {
-   1,   1,   1,   1,   1,   1, /*  0-5  */
-   1,   1,   1,   1,   1,   1, /*  6-11 */
-   1,   1,   1,   2,   2,   3, /* 12-17 */
-   4,   5,   6,   7,   9,  11, /* 18-23 */
-  14,  18,  23,  29,  36,  46, /* 24-29 */
-  58,  73,  91, 115, 145, 183, /* 30-35 */
- 230, 290, 366, 461, 581, 731, /* 36-41 */
- 922,1161,1463,1843,2322,2926, /* 42-47 */
-3686,4645,5852,7373
+    14,      18,      22,      28,     36,     45,     57,     72, /*  0 -  7 */
+    91,     115,     145,     182,    230,    290,    365,    460, /*  8 - 15 */
+   580,     731,     921,    1161,   1462,   1843,   2322,   2925, /* 16 - 23 */
+  3686,    4644,    5851,    7372,   9289,  11703,  14745,  18578, /* 24 - 31 */
+ 23407,   29491,   37156,   46814,  58982,  74313,  93628, 117964, /* 32 - 39 */
+148626,  187257,  235929,  297252, 374514, 471859, 594505, 749029, /* 40 - 47 */
+943718, 1189010, 1498059, 1887436                                  /* 48 - 51 */
 };
 
 /* TODO: calculate CABAC costs */
@@ -2385,7 +2383,7 @@ void x264_macroblock_analyse( x264_t *h )
                 i_bskip_cost = ssd_mb( h );
 
                 /* 6 = minimum cavlc cost of a non-skipped MB */
-                if( i_bskip_cost <= 6 * analysis.i_lambda2 )
+                if( i_bskip_cost <= ((6 * analysis.i_lambda2 + 128) >> 8) )
                 {
                     h->mb.i_type = B_SKIP;
                     x264_analyse_update_cache( h, &analysis );
diff --git a/encoder/rdo.c b/encoder/rdo.c
index 7f29af74..e1227314 100644
--- a/encoder/rdo.c
+++ b/encoder/rdo.c
@@ -78,21 +78,21 @@ static int x264_rd_cost_mb( x264_t *h, int i_lambda2 )
 
     if( IS_SKIP( h->mb.i_type ) )
     {
-        i_bits = 1 * i_lambda2;
+        i_bits = (1 * i_lambda2 + 128) >> 8;
     }
     else if( h->param.b_cabac )
     {
         x264_cabac_t cabac_tmp;
         h->mc.memcpy_aligned( &cabac_tmp, &h->cabac, offsetof(x264_cabac_t,i_low) );
         x264_macroblock_size_cabac( h, &cabac_tmp );
-        i_bits = ( cabac_tmp.f8_bits_encoded * i_lambda2 + 128 ) >> 8;
+        i_bits = ( (uint64_t)cabac_tmp.f8_bits_encoded * i_lambda2 + 32768 ) >> 16;
     }
     else
     {
         bs_t bs_tmp = h->out.bs;
         bs_tmp.i_bits_encoded = 0;
         x264_macroblock_size_cavlc( h, &bs_tmp );
-        i_bits = bs_tmp.i_bits_encoded * i_lambda2;
+        i_bits = ( bs_tmp.i_bits_encoded * i_lambda2 + 128 ) >> 8;
     }
 
     h->mb.b_transform_8x8 = b_transform_bak;
@@ -127,11 +127,11 @@ int x264_rd_cost_part( x264_t *h, int i_lambda2, int i8, int i_pixel )
         x264_cabac_t cabac_tmp;
         h->mc.memcpy_aligned( &cabac_tmp, &h->cabac, offsetof(x264_cabac_t,i_low) );
         x264_partition_size_cabac( h, &cabac_tmp, i8, i_pixel );
-        i_bits = ( cabac_tmp.f8_bits_encoded * i_lambda2 + 128 ) >> 8;
+        i_bits = ( (uint64_t)cabac_tmp.f8_bits_encoded * i_lambda2 + 32768 ) >> 16;
     }
     else
     {
-        i_bits = x264_partition_size_cavlc( h, i8, i_pixel ) * i_lambda2;
+        i_bits = ( x264_partition_size_cavlc( h, i8, i_pixel ) * i_lambda2 + 128 ) >> 8;
     }
 
     return i_ssd + i_bits;
@@ -149,11 +149,11 @@ int x264_rd_cost_i8x8( x264_t *h, int i_lambda2, int i8, int i_mode )
         x264_cabac_t cabac_tmp;
         h->mc.memcpy_aligned( &cabac_tmp, &h->cabac, offsetof(x264_cabac_t,i_low) );
         x264_partition_i8x8_size_cabac( h, &cabac_tmp, i8, i_mode );
-        i_bits = ( cabac_tmp.f8_bits_encoded * i_lambda2 + 128 ) >> 8;
+        i_bits = ( (uint64_t)cabac_tmp.f8_bits_encoded * i_lambda2 + 32768 ) >> 16;
     }
     else
     {
-        i_bits = x264_partition_i8x8_size_cavlc( h, i8, i_mode ) * i_lambda2;
+        i_bits = ( x264_partition_i8x8_size_cavlc( h, i8, i_mode ) * i_lambda2 + 128 ) >> 8;
     }
 
     return i_ssd + i_bits;
@@ -170,13 +170,12 @@ int x264_rd_cost_i4x4( x264_t *h, int i_lambda2, int i4, int i_mode )
     {
         x264_cabac_t cabac_tmp;
         h->mc.memcpy_aligned( &cabac_tmp, &h->cabac, offsetof(x264_cabac_t,i_low) );
-        
         x264_partition_i4x4_size_cabac( h, &cabac_tmp, i4, i_mode );
-        i_bits = ( cabac_tmp.f8_bits_encoded * i_lambda2 + 128 ) >> 8;
+        i_bits = ( (uint64_t)cabac_tmp.f8_bits_encoded * i_lambda2 + 32768 ) >> 16;
     }
     else
     {
-        i_bits = x264_partition_i4x4_size_cavlc( h, i4, i_mode ) * i_lambda2;
+        i_bits = ( x264_partition_i4x4_size_cavlc( h, i4, i_mode ) * i_lambda2 + 128 ) >> 8;
     }
 
     return i_ssd + i_bits;
@@ -198,11 +197,11 @@ int x264_rd_cost_i8x8_chroma( x264_t *h, int i_lambda2, int i_mode, int b_dct )
         x264_cabac_t cabac_tmp;
         h->mc.memcpy_aligned( &cabac_tmp, &h->cabac, offsetof(x264_cabac_t,i_low) );
         x264_i8x8_chroma_size_cabac( h, &cabac_tmp );
-        i_bits = ( cabac_tmp.f8_bits_encoded * i_lambda2 + 128 ) >> 8;
+        i_bits = ( (uint64_t)cabac_tmp.f8_bits_encoded * i_lambda2 + 32768 ) >> 16;
     }
     else
     {
-        i_bits = x264_i8x8_chroma_size_cavlc( h ) * i_lambda2;
+        i_bits = ( x264_i8x8_chroma_size_cavlc( h ) * i_lambda2 + 128 ) >> 8;
     }
 
     return i_ssd + i_bits;