fix an arithmetic overflow in trellis at high qp.

git-svn-id: svn://svn.videolan.org/x264/trunk@677 df754926-b1dd-0310-bc7b-ec298dee348c

diff --git a/encoder/rdo.c b/encoder/rdo.c
index b204639f5d3cbee6cdb1ef07b73317fdad3d2bfe..d722c229d7a348e14ea6d770d98900403f20dfe4 100644 (file)
--- a/encoder/rdo.c
+++ b/encoder/rdo.c
@@ -397,8 +397,8 @@ static void quant_trellis_cabac( x264_t *h, int16_t *dct,
         {
             // no need to calculate ssd of 0s: it's the same in all nodes.
             // no need to modify level_tree for ctx=0: it starts with an infinite loop of 0s.
-            const int cost_sig0 = x264_cabac_size_decision_noup( &cabac_state_sig[i], 0 )
-                                * i_lambda2 >> ( CABAC_SIZE_BITS - LAMBDA_BITS );
+            const uint32_t cost_sig0 = x264_cabac_size_decision_noup( &cabac_state_sig[i], 0 )
+                                     * (uint64_t)i_lambda2 >> ( CABAC_SIZE_BITS - LAMBDA_BITS );
             for( j = 1; j < 8; j++ )
             {
                 if( nodes_cur[j].score != TRELLIS_SCORE_MAX )