Check size limit in vpx_realloc_frame_buffer.

If CONFIG_SIZE_LIMIT is defined, vpx_realloc_frame_buffer should fail if
width or height is too big.

This carries over commit ebc2714d71a834fc32a19eef0a81f51fbc47db01 of
libaom: https://aomedia-review.googlesource.com/c/aom/+/65521

Change-Id: Id7645c5cefbe1847714695d41f506ff30ea985f6

diff --git a/vpx_scale/generic/yv12config.c b/vpx_scale/generic/yv12config.c
index 220b8be3d12cb7357e14043868e6ce836939f4e0..0fdba7b170dc979132a4a9d0e00aedffb7c9c44e 100644 (file)
--- a/vpx_scale/generic/yv12config.c
+++ b/vpx_scale/generic/yv12config.c
@@ -143,6 +143,10 @@ int vpx_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, int width, int height,
                              vpx_codec_frame_buffer_t *fb,
                              vpx_get_frame_buffer_cb_fn_t cb, void *cb_priv) {
   if (ybf) {
+#if CONFIG_SIZE_LIMIT
+    if (width > DECODE_WIDTH_LIMIT || height > DECODE_HEIGHT_LIMIT) return -1;
+#endif
+
     const int vp9_byte_align = (byte_alignment == 0) ? 1 : byte_alignment;
     const int aligned_width = (width + 7) & ~7;
     const int aligned_height = (height + 7) & ~7;