From: Johann <johannkoenig@google.com>
Date: Fri, 3 Nov 2017 16:49:13 +0000 (-0700)
Subject: fail early on oversize frames
X-Git-Tag: v1.7.0~78^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=5fe82459ec5c1566a2532e51550fb15b380f80de;p=libvpx

fail early on oversize frames

Even though frame_size is calculated in uint64_t, it winds up in an int
size value.

This was exposed with the msan test because the memset is called with
(int)frame_size, leading to a segfault.

Change-Id: I7fd930360dca274adb8f3e43e5e6785204808861
---

diff --git a/vpx_scale/generic/yv12config.c b/vpx_scale/generic/yv12config.c
index a674eac84..9c7ca42c7 100644
--- a/vpx_scale/generic/yv12config.c
+++ b/vpx_scale/generic/yv12config.c
@@ -9,6 +9,7 @@
  */
 
 #include <assert.h>
+#include <limits.h>
 
 #include "vpx_scale/yv12config.h"
 #include "vpx_mem/vpx_mem.h"
@@ -165,6 +166,12 @@ int vpx_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
 
     uint8_t *buf = NULL;
 
+    // frame_size is stored in buffer_alloc_sz, which is an int. If it won't
+    // fit, fail early.
+    if (frame_size > INT_MAX) {
+      return -1;
+    }
+
     if (cb != NULL) {
       const int align_addr_extra_size = 31;
       const uint64_t external_frame_size = frame_size + align_addr_extra_size;
@@ -193,8 +200,6 @@ int vpx_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
       vpx_free(ybf->buffer_alloc);
       ybf->buffer_alloc = NULL;
 
-      if (frame_size != (size_t)frame_size) return -1;
-
       ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, (size_t)frame_size);
       if (!ybf->buffer_alloc) return -1;