libav: a more robust fix for aac encoder crash

(cherry picked from commit acf986e8df2e8084f50d684a0aa98329f8c77cab)

diff --git a/contrib/ffmpeg/A14-aacenc-high-bitrate-crash.patch b/contrib/ffmpeg/A14-aacenc-high-bitrate-crash.patch
index 06a640460db2c064d9234c1c5ae343bd182dcf77..1eb260dae4ffda5792be074451c5935630dff69c 100644 (file)
--- a/contrib/ffmpeg/A14-aacenc-high-bitrate-crash.patch
+++ b/contrib/ffmpeg/A14-aacenc-high-bitrate-crash.patch
@@ -1,28 +1,105 @@
-From 7facbe6793a89ffeda7230d39fce8ccb5ce76208 Mon Sep 17 00:00:00 2001
+From 923c5a378ed4a11817a16a7c1ed3011d7ff72ec8 Mon Sep 17 00:00:00 2001
 From: John Stebbins <stebbins@jetheaddev.com>
-Date: Thu, 23 Feb 2017 15:42:02 -0700
-Subject: [PATCH] aacenc: fix crash when encoding at high bitrates
+Date: Sun, 26 Feb 2017 09:58:14 -0700
+Subject: [PATCH] put_bits: bounds check buffer
 
-Increases the size of the AVPacket being written into.  Although the aac
-spec says the max packet size should be 768 * channel count, aacenc
-overshoots this and requires a larger buffer.
+This prevents invalid writes outside put_bits' buffer.
+
+It also has the side effect of allowing measurement of the required
+size of a buffer without the need to pre-allocate an over-sized buffer.
+
+This fixes a crash in aacenc.c where it could write past the end of the
+allocated packet, which is allocated to be the max size allowed by the
+aac spec.  aacenc.c uses the above feature to check the size
+of encoded data and try again when the size is too large.
 ---
- libavcodec/aacenc.c | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
+ libavcodec/aacenc.c   |  2 +-
+ libavcodec/put_bits.h | 25 ++++++++++++++++++++-----
+ 2 files changed, 21 insertions(+), 6 deletions(-)
 
 diff --git a/libavcodec/aacenc.c b/libavcodec/aacenc.c
-index 9b0e99b..b63affc 100644
+index 9b0e99b..a83b2a8 100644
 --- a/libavcodec/aacenc.c
 +++ b/libavcodec/aacenc.c
-@@ -572,7 +572,7 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
+@@ -638,7 +638,7 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
+             start_ch += chans;
          }
-         start_ch += chans;
-     }
--    if ((ret = ff_alloc_packet(avpkt, 768 * s->channels))) {
-+    if ((ret = ff_alloc_packet(avpkt, 8192 * s->channels))) {
-         av_log(avctx, AV_LOG_ERROR, "Error getting output packet\n");
-         return ret;
-     }
+ 
+-        frame_bits = put_bits_count(&s->pb);
++        frame_bits = put_bits_attempted(&s->pb);
+         if (frame_bits <= 6144 * s->channels - 3) {
+             s->psy.bitres.bits = frame_bits / s->channels;
+             break;
+diff --git a/libavcodec/put_bits.h b/libavcodec/put_bits.h
+index 17666fa..677fecd 100644
+--- a/libavcodec/put_bits.h
++++ b/libavcodec/put_bits.h
+@@ -30,6 +30,7 @@
+ #include <stddef.h>
+ #include <assert.h>
+ 
++#include "libavutil/common.h"
+ #include "libavutil/intreadwrite.h"
+ 
+ typedef struct PutBitContext {
+@@ -62,11 +63,19 @@ static inline void init_put_bits(PutBitContext *s, uint8_t *buffer,
+ }
+ 
+ /**
++ * @return the total number of bits attempted to be written to the bitstream.
++ */
++static inline int put_bits_attempted(PutBitContext *s)
++{
++    return (s->buf_ptr - s->buf) * 8 + 32 - s->bit_left;
++}
++
++/**
+  * @return the total number of bits written to the bitstream.
+  */
+ static inline int put_bits_count(PutBitContext *s)
+ {
+-    return (s->buf_ptr - s->buf) * 8 + 32 - s->bit_left;
++    return FFMIN(s->size_in_bits, put_bits_attempted(s));
+ }
+ 
+ /**
+@@ -89,10 +98,14 @@ static inline void flush_put_bits(PutBitContext *s)
+     while (s->bit_left < 32) {
+         /* XXX: should test end of buffer */
+ #ifdef BITSTREAM_WRITER_LE
+-        *s->buf_ptr++ = s->bit_buf;
++        if (s->buf_ptr < s->buf_end)
++            *s->buf_ptr = s->bit_buf;
++        s->buf_ptr++;
+         s->bit_buf  >>= 8;
+ #else
+-        *s->buf_ptr++ = s->bit_buf >> 24;
++        if (s->buf_ptr < s->buf_end)
++            *s->buf_ptr = s->bit_buf >> 24;
++        s->buf_ptr++;
+         s->bit_buf  <<= 8;
+ #endif
+         s->bit_left  += 8;
+@@ -145,7 +158,8 @@ static inline void put_bits(PutBitContext *s, int n, unsigned int value)
+ #ifdef BITSTREAM_WRITER_LE
+     bit_buf |= value << (32 - bit_left);
+     if (n >= bit_left) {
+-        AV_WL32(s->buf_ptr, bit_buf);
++        if (s->buf_ptr < s->buf_end)
++            AV_WL32(s->buf_ptr, bit_buf);
+         s->buf_ptr += 4;
+         bit_buf     = (bit_left == 32) ? 0 : value >> bit_left;
+         bit_left   += 32;
+@@ -158,7 +172,8 @@ static inline void put_bits(PutBitContext *s, int n, unsigned int value)
+     } else {
+         bit_buf   <<= bit_left;
+         bit_buf    |= value >> (n - bit_left);
+-        AV_WB32(s->buf_ptr, bit_buf);
++        if (s->buf_ptr < s->buf_end)
++            AV_WB32(s->buf_ptr, bit_buf);
+         s->buf_ptr += 4;
+         bit_left   += 32 - n;
+         bit_buf     = value;
 -- 
 2.9.3