Fix integer overflow in chroma SSD check

Could cause bad skips at very high quantizers on extreme inputs.

diff --git a/encoder/rdo.c b/encoder/rdo.c
index 9bcfb9d8e605e9638154ae337f4b776f16d8350d..5c607afa8848f9257c8b83cd8ff2414e0745f361 100644 (file)
--- a/encoder/rdo.c
+++ b/encoder/rdo.c
@@ -131,7 +131,7 @@ static inline int ssd_plane( x264_t *h, int size, int p, int x, int y )
 static inline int ssd_mb( x264_t *h )
 {
     int chromassd = ssd_plane(h, PIXEL_8x8, 1, 0, 0) + ssd_plane(h, PIXEL_8x8, 2, 0, 0);
-    chromassd = (chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8;
+    chromassd = ((uint64_t)chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8;
     return ssd_plane(h, PIXEL_16x16, 0, 0, 0) + chromassd;
 }
 
@@ -223,7 +223,7 @@ uint64_t x264_rd_cost_part( x264_t *h, int i_lambda2, int i4, int i_pixel )
 
     chromassd = ssd_plane( h, i_pixel+3, 1, (i8&1)*4, (i8>>1)*4 )
               + ssd_plane( h, i_pixel+3, 2, (i8&1)*4, (i8>>1)*4 );
-    chromassd = (chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8;
+    chromassd = ((uint64_t)chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8;
     i_ssd = ssd_plane( h, i_pixel,   0, (i8&1)*8, (i8>>1)*8 ) + chromassd;
 
     if( h->param.b_cabac )