From 53844275e9e19d0af6f6e5bda98d884cf09aad5d Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Thu, 24 Jul 2014 15:31:32 -0700 Subject: [PATCH] Fix potential ioc issue in vp9_get_prob for 4K above sizes This commit turns on the existing vp9_get_prob function using 64 bit in the intermediate step. It fixes the ioc issue for 4K above frame sizes (issue 828). Change-Id: I9f627f3beca2c522f73b38fd2a3e7eefdff01a7c --- test/frame_size_tests.cc | 3 +-- vp9/common/vp9_prob.h | 9 --------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/test/frame_size_tests.cc b/test/frame_size_tests.cc index b05d12e9e..d1ea0967f 100644 --- a/test/frame_size_tests.cc +++ b/test/frame_size_tests.cc @@ -89,8 +89,7 @@ TEST_F(VP9FrameSizeTestsLarge, ValidSizes) { // one for each lag in frames (for 2 pass), and then one for each possible // reference buffer (8) - we can end up with up to 30 buffers of roughly this // size or almost 1 gig of memory. - // TODO(jzern): restore this to at least 4096x4096 after issue #828 is fixed. - video.SetSize(4096, 2160); + video.SetSize(4096, 4096); video.set_limit(2); expected_res_ = VPX_CODEC_OK; ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); diff --git a/vp9/common/vp9_prob.h b/vp9/common/vp9_prob.h index f36148035..3920619d8 100644 --- a/vp9/common/vp9_prob.h +++ b/vp9/common/vp9_prob.h @@ -47,18 +47,9 @@ static INLINE vp9_prob clip_prob(int p) { return (p > 255) ? 255u : (p < 1) ? 1u : p; } -// int64 is not needed for normal frame level calculations. -// However when outputting entropy stats accumulated over many frames -// or even clips we can overflow int math. -#ifdef ENTROPY_STATS static INLINE vp9_prob get_prob(int num, int den) { return (den == 0) ? 128u : clip_prob(((int64_t)num * 256 + (den >> 1)) / den); } -#else -static INLINE vp9_prob get_prob(int num, int den) { - return (den == 0) ? 128u : clip_prob((num * 256 + (den >> 1)) / den); -} -#endif static INLINE vp9_prob get_binary_prob(int n0, int n1) { return get_prob(n0, n0 + n1); -- 2.40.0