From 466754b8dc7bdb6b32474d1feac8ff0e1451aefb Mon Sep 17 00:00:00 2001 From: Fiona Glaser Date: Mon, 22 Feb 2010 11:21:51 -0800 Subject: [PATCH] Fix integer overflow in chroma SSD check Could cause bad skips at very high quantizers on extreme inputs. --- encoder/rdo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/encoder/rdo.c b/encoder/rdo.c index 9bcfb9d8..5c607afa 100644 --- a/encoder/rdo.c +++ b/encoder/rdo.c @@ -131,7 +131,7 @@ static inline int ssd_plane( x264_t *h, int size, int p, int x, int y ) static inline int ssd_mb( x264_t *h ) { int chromassd = ssd_plane(h, PIXEL_8x8, 1, 0, 0) + ssd_plane(h, PIXEL_8x8, 2, 0, 0); - chromassd = (chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8; + chromassd = ((uint64_t)chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8; return ssd_plane(h, PIXEL_16x16, 0, 0, 0) + chromassd; } @@ -223,7 +223,7 @@ uint64_t x264_rd_cost_part( x264_t *h, int i_lambda2, int i4, int i_pixel ) chromassd = ssd_plane( h, i_pixel+3, 1, (i8&1)*4, (i8>>1)*4 ) + ssd_plane( h, i_pixel+3, 2, (i8&1)*4, (i8>>1)*4 ); - chromassd = (chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8; + chromassd = ((uint64_t)chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8; i_ssd = ssd_plane( h, i_pixel, 0, (i8&1)*8, (i8>>1)*8 ) + chromassd; if( h->param.b_cabac ) -- 2.40.0