fail early on oversize frames

Even though frame_size is calculated in uint64_t, it winds up in an int
size value.

This was exposed with the msan test because the memset is called with
(int)frame_size, leading to a segfault.

Change-Id: I7fd930360dca274adb8f3e43e5e6785204808861

diff --git a/vpx_scale/generic/yv12config.c b/vpx_scale/generic/yv12config.c
index a674eac84ba8b83c6368e10cce141f1f11fdd437..9c7ca42c78b4d398a3689fb83302814193be8000 100644 (file)
--- a/vpx_scale/generic/yv12config.c
+++ b/vpx_scale/generic/yv12config.c
@@ -9,6 +9,7 @@
  */
 
 #include <assert.h>
+#include <limits.h>
 
 #include "vpx_scale/yv12config.h"
 #include "vpx_mem/vpx_mem.h"
@@ -165,6 +166,12 @@ int vpx_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
 
     uint8_t *buf = NULL;
 
+    // frame_size is stored in buffer_alloc_sz, which is an int. If it won't
+    // fit, fail early.
+    if (frame_size > INT_MAX) {
+      return -1;
+    }
+
     if (cb != NULL) {
       const int align_addr_extra_size = 31;
       const uint64_t external_frame_size = frame_size + align_addr_extra_size;
@@ -193,8 +200,6 @@ int vpx_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
       vpx_free(ybf->buffer_alloc);
       ybf->buffer_alloc = NULL;
 
-      if (frame_size != (size_t)frame_size) return -1;
-
       ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, (size_t)frame_size);
       if (!ybf->buffer_alloc) return -1;