]> granicus.if.org Git - libvpx/commitdiff
better-hw-compatibility: fix out of bounds access
authorMatthias Räncker <theonetruecamper@gmx.de>
Thu, 20 Sep 2018 17:57:25 +0000 (19:57 +0200)
committerMatthias Räncker <theonetruecamper@gmx.de>
Fri, 21 Sep 2018 21:35:42 +0000 (23:35 +0200)
With --enable-better-hw-compatibility an access to array element -1
can be observed for VP9/ActiveMapTest.Test/0
../vp9/encoder/vp9_rdopt.c:3938:53: runtime error:
  index -1 out of bounds for type 'RefBuffer [3]'

There doesn't seem anything that would prevent ref_frame from being 0.
If there is no reference frame it can probably be assumed that it
isn't scaled.

Signed-off-by: Matthias Räncker <theonetruecamper@gmx.de>
Change-Id: I0a29cd0ffc9a19742e5e72203d5ec5d0a16eac7a

vp9/encoder/vp9_rdopt.c

index 1f1cd40d8473d56285c340c8c8d0f2620e7ba7eb..e0424c6d83d807d3ca2aa23e273e8b9434c41192 100644 (file)
@@ -3935,7 +3935,8 @@ void vp9_rd_pick_inter_mode_sub8x8(VP9_COMP *cpi, TileDataEnc *tile_data,
 #if CONFIG_BETTER_HW_COMPATIBILITY
     // forbid 8X4 and 4X8 partitions if any reference frame is scaled.
     if (bsize == BLOCK_8X4 || bsize == BLOCK_4X8) {
-      int ref_scaled = vp9_is_scaled(&cm->frame_refs[ref_frame - 1].sf);
+      int ref_scaled = ref_frame > INTRA_FRAME &&
+                       vp9_is_scaled(&cm->frame_refs[ref_frame - 1].sf);
       if (second_ref_frame > INTRA_FRAME)
         ref_scaled += vp9_is_scaled(&cm->frame_refs[second_ref_frame - 1].sf);
       if (ref_scaled) continue;