From: James Zern <jzern@google.com>
Date: Mon, 8 Nov 2021 20:57:12 +0000 (-0800)
Subject: vp8,calc_pframe_target_size: fix integer overflow
X-Git-Tag: v1.12.0-rc1~126
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=23796337ce5b0a2f58eb7386c9aded5e6a4b84f6;p=libvpx

vp8,calc_pframe_target_size: fix integer overflow

this is similar to the fix for calc_iframe_target_size:
5f345a924 Avoid overflow in calc_iframe_target_size

Bug: chromium:1264506
Change-Id: I2f0e161cf9da59ca0724692d581f1594c8098ebb
---

diff --git a/vp8/encoder/ratectrl.c b/vp8/encoder/ratectrl.c
index d591680ce..3df34009a 100644
--- a/vp8/encoder/ratectrl.c
+++ b/vp8/encoder/ratectrl.c
@@ -781,6 +781,7 @@ static void calc_pframe_target_size(VP8_COMP *cpi) {
         }
       } else {
         int percent_high = 0;
+        int64_t target = cpi->this_frame_target;
 
         if ((cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER) &&
             (cpi->buffer_level > cpi->oxcf.optimal_buffer_level)) {
@@ -798,7 +799,9 @@ static void calc_pframe_target_size(VP8_COMP *cpi) {
           percent_high = 0;
         }
 
-        cpi->this_frame_target += (cpi->this_frame_target * percent_high) / 200;
+        target += (target * percent_high) / 200;
+        target = VPXMIN(target, INT_MAX);
+        cpi->this_frame_target = (int)target;
 
         /* Are we allowing control of active_worst_allowed_q according
          * to buffer level.