From a439b3c97709604a61fa9ff5d8d02aeae633a69c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Matthias=20R=C3=A4ncker?= Date: Thu, 20 Sep 2018 19:57:25 +0200 Subject: [PATCH] better-hw-compatibility: fix out of bounds access MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit With --enable-better-hw-compatibility an access to array element -1 can be observed for VP9/ActiveMapTest.Test/0 ../vp9/encoder/vp9_rdopt.c:3938:53: runtime error: index -1 out of bounds for type 'RefBuffer [3]' There doesn't seem anything that would prevent ref_frame from being 0. If there is no reference frame it can probably be assumed that it isn't scaled. Signed-off-by: Matthias Räncker Change-Id: I0a29cd0ffc9a19742e5e72203d5ec5d0a16eac7a --- vp9/encoder/vp9_rdopt.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 1f1cd40d8..e0424c6d8 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -3935,7 +3935,8 @@ void vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, TileDataEnc *tile_data, #if CONFIG_BETTER_HW_COMPATIBILITY // forbid 8X4 and 4X8 partitions if any reference frame is scaled. if (bsize == BLOCK_8X4 || bsize == BLOCK_4X8) { - int ref_scaled = vp9_is_scaled(&cm->frame_refs[ref_frame - 1].sf); + int ref_scaled = ref_frame > INTRA_FRAME && + vp9_is_scaled(&cm->frame_refs[ref_frame - 1].sf); if (second_ref_frame > INTRA_FRAME) ref_scaled += vp9_is_scaled(&cm->frame_refs[second_ref_frame - 1].sf); if (ref_scaled) continue; -- 2.40.0