From 1a01194ab548ca9b5d7dbbdc2350d4c06f54e6d4 Mon Sep 17 00:00:00 2001 From: Jim Bankoski Date: Thu, 17 Jul 2014 06:33:29 -0700 Subject: [PATCH] fail allocation of buffers if size_t < frame_size Change-Id: I25c595e8c197ab0a9955d2373f1a74d42fbd1638 --- vpx_scale/generic/yv12config.c | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/vpx_scale/generic/yv12config.c b/vpx_scale/generic/yv12config.c index 3eaf50ed7..e8003cea4 100644 --- a/vpx_scale/generic/yv12config.c +++ b/vpx_scale/generic/yv12config.c @@ -142,34 +142,39 @@ int vp9_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, const int aligned_width = (width + 7) & ~7; const int aligned_height = (height + 7) & ~7; const int y_stride = ((aligned_width + 2 * border) + 31) & ~31; - const int yplane_size = (aligned_height + 2 * border) * y_stride; + const uint64_t yplane_size = (aligned_height + 2 * border) * + (uint64_t)y_stride; const int uv_width = aligned_width >> ss_x; const int uv_height = aligned_height >> ss_y; const int uv_stride = y_stride >> ss_x; const int uv_border_w = border >> ss_x; const int uv_border_h = border >> ss_y; - const int uvplane_size = (uv_height + 2 * uv_border_h) * uv_stride; + const uint64_t uvplane_size = (uv_height + 2 * uv_border_h) * + (uint64_t)uv_stride; #if CONFIG_ALPHA const int alpha_width = aligned_width; const int alpha_height = aligned_height; const int alpha_stride = y_stride; const int alpha_border_w = border; const int alpha_border_h = border; - const int alpha_plane_size = (alpha_height + 2 * alpha_border_h) * - alpha_stride; - const int frame_size = yplane_size + 2 * uvplane_size + - alpha_plane_size; + const uint64_t alpha_plane_size = (alpha_height + 2 * alpha_border_h) * + (uint64_t)alpha_stride; + const uint64_t frame_size = yplane_size + 2 * uvplane_size + + alpha_plane_size; #else - const int frame_size = yplane_size + 2 * uvplane_size; + const uint64_t frame_size = yplane_size + 2 * uvplane_size; #endif if (cb != NULL) { const int align_addr_extra_size = 31; - const size_t external_frame_size = frame_size + align_addr_extra_size; + const uint64_t external_frame_size = frame_size + align_addr_extra_size; assert(fb != NULL); + if (external_frame_size != (size_t)external_frame_size) + return -1; + // Allocation to hold larger frame, or first allocation. - if (cb(cb_priv, external_frame_size, fb) < 0) + if (cb(cb_priv, (size_t)external_frame_size, fb) < 0) return -1; if (fb->data == NULL || fb->size < external_frame_size) @@ -181,10 +186,15 @@ int vp9_realloc_frame_buffer(YV12_BUFFER_CONFIG *ybf, vpx_memset(fb->data, 0, fb->size); ybf->buffer_alloc = (uint8_t *)yv12_align_addr(fb->data, 32); - } else if (frame_size > ybf->buffer_alloc_sz) { + } else if (frame_size > (size_t)ybf->buffer_alloc_sz) { // Allocation to hold larger frame, or first allocation. vpx_free(ybf->buffer_alloc); - ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, frame_size); + ybf->buffer_alloc = NULL; + + if (frame_size != (size_t)frame_size) + return -1; + + ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, (size_t)frame_size); if (!ybf->buffer_alloc) return -1; -- 2.40.0