Bound the total allocated memory of frame buffer

This CL allows to limit memory consumption of the frame buffer pool. As
the result if compiled with VPX_MAX_ALLOCABLE_MEMORY set codec will fail
if frame resolution requires more memory
This is backported CL aae2183cb58b60d01b8e4e15269ee9f48dd72908 from
aomedia

Tested:
configure --extra-cflags="-DVPX_MAX_ALLOCABLE_MEMORY=536870912"
make
./test_libvpx

BUG=webm:1579

Change-Id: Ic62213b600a7562917d5a339a344ad8db4b6f481

diff --git a/vpx_scale/generic/yv12config.c b/vpx_scale/generic/yv12config.c
index 7d4aa2a9198f10225eb2a4d1c018b06a1e2e0d33..08679a47218507bc9d5fb255396702b9d5dd4d20 100644 (file)
--- a/vpx_scale/generic/yv12config.c
+++ b/vpx_scale/generic/yv12config.c
@@ -15,6 +15,9 @@
 #include "vpx_mem/vpx_mem.h"
 #include "vpx_ports/mem.h"
 
+#if defined(VPX_MAX_ALLOCABLE_MEMORY)
+#include "vp9/common/vp9_onyxc_int.h"
+#endif  // VPX_MAX_ALLOCABLE_MEMORY
 /****************************************************************************
  *  Exports
  ****************************************************************************/
@@ -185,6 +188,13 @@ int vpx_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
 
     uint8_t *buf = NULL;
 
+#if defined(VPX_MAX_ALLOCABLE_MEMORY)
+    // The decoder may allocate REF_FRAMES frame buffers in the frame buffer
+    // pool. Bound the total amount of allocated memory as if these REF_FRAMES
+    // frame buffers were allocated in a single allocation.
+    if (frame_size > VPX_MAX_ALLOCABLE_MEMORY / REF_FRAMES) return -1;
+#endif  // VPX_MAX_ALLOCABLE_MEMORY
+
     // frame_size is stored in buffer_alloc_sz, which is a size_t. If it won't
     // fit, fail early.
     if (frame_size > SIZE_MAX) {