From: James Berry Date: Wed, 12 Oct 2011 15:18:50 +0000 (-0400) Subject: Fix: check cx_data buffer prior to write X-Git-Tag: v1.0.0~116^2 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=bc7151131d2baea4c958226b0edfe184a0db8986;p=libvpx Fix: check cx_data buffer prior to write check to make sure that cx_data buffer has enough room before writting to it, prior behavior did not which could result in a crash. Change-Id: I3fab6f2bc4a96d7c675ea81acd39ece121738b28 --- diff --git a/vp8/common/onyx.h b/vp8/common/onyx.h index e911ea0f4..28cbaed98 100644 --- a/vp8/common/onyx.h +++ b/vp8/common/onyx.h @@ -221,7 +221,7 @@ extern "C" // receive a frames worth of data caller can assume that a copy of this frame is made // and not just a copy of the pointer.. int vp8_receive_raw_frame(VP8_PTR comp, unsigned int frame_flags, YV12_BUFFER_CONFIG *sd, int64_t time_stamp, int64_t end_time_stamp); - int vp8_get_compressed_data(VP8_PTR comp, unsigned int *frame_flags, unsigned long *size, unsigned char *dest, int64_t *time_stamp, int64_t *time_end, int flush); + int vp8_get_compressed_data(VP8_PTR comp, unsigned int *frame_flags, unsigned long *size, unsigned char *dest, unsigned char *dest_end, int64_t *time_stamp, int64_t *time_end, int flush); int vp8_get_preview_raw_frame(VP8_PTR comp, YV12_BUFFER_CONFIG *dest, vp8_ppflags_t *flags); int vp8_use_as_reference(VP8_PTR comp, int ref_frame_flags); diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c index cea8e1232..1dda22f1c 100644 --- a/vp8/encoder/bitstream.c +++ b/vp8/encoder/bitstream.c @@ -358,11 +358,12 @@ static void write_partition_size(unsigned char *cx_data, int size) } -static void pack_tokens_into_partitions_c(VP8_COMP *cpi, unsigned char *cx_data, int num_part, int *size) +static void pack_tokens_into_partitions_c(VP8_COMP *cpi, unsigned char *cx_data, unsigned char * cx_data_end, int num_part, int *size) { int i; unsigned char *ptr = cx_data; + unsigned char *ptr_end = cx_data_end; unsigned int shift; vp8_writer *w = &cpi->bc2; *size = 3 * (num_part - 1); @@ -371,7 +372,7 @@ static void pack_tokens_into_partitions_c(VP8_COMP *cpi, unsigned char *cx_data, for (i = 0; i < num_part; i++) { - vp8_start_encode(w, ptr); + vp8_start_encode(w, ptr, ptr_end); { unsigned int split; int count = w->count; @@ -437,7 +438,13 @@ static void pack_tokens_into_partitions_c(VP8_COMP *cpi, unsigned char *cx_data, w->buffer[x] += 1; } + validate_buffer(w->buffer + w->pos, + 1, + cx_data_end, + &cpi->common.error); + w->buffer[w->pos++] = (lowvalue >> (24 - offset)); + lowvalue <<= offset; shift = count; lowvalue &= 0xffffff; @@ -497,7 +504,14 @@ static void pack_tokens_into_partitions_c(VP8_COMP *cpi, unsigned char *cx_data, w->buffer[x] += 1; } - w->buffer[w->pos++] = (lowvalue >> (24 - offset)); + validate_buffer(w->buffer + w->pos, + 1, + cx_data_end, + &cpi->common.error); + + w->buffer[w->pos++] = + (lowvalue >> (24 - offset)); + lowvalue <<= offset; shift = count; lowvalue &= 0xffffff; @@ -543,7 +557,13 @@ static void pack_tokens_into_partitions_c(VP8_COMP *cpi, unsigned char *cx_data, if (!++count) { count = -8; + validate_buffer(w->buffer + w->pos, + 1, + cx_data_end, + &cpi->common.error); + w->buffer[w->pos++] = (lowvalue >> 24); + lowvalue &= 0xffffff; } } @@ -1526,7 +1546,7 @@ static void put_delta_q(vp8_writer *bc, int delta_q) vp8_write_bit(bc, 0); } -void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size) +void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned char * dest_end, unsigned long *size) { int i, j; VP8_HEADER oh; @@ -1536,6 +1556,7 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size) int extra_bytes_packed = 0; unsigned char *cx_data = dest; + unsigned char *cx_data_end = dest_end; const int *mb_feature_data_bits; oh.show_frame = (int) pc->show_frame; @@ -1544,6 +1565,8 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size) oh.first_partition_length_in_bytes = 0; mb_feature_data_bits = vp8_mb_feature_data_bits; + + validate_buffer(cx_data, 3, cx_data_end, &cpi->common.error); cx_data += 3; #if defined(SECTIONBITS_OUTPUT) @@ -1560,6 +1583,8 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size) { int v; + validate_buffer(cx_data, 7, cx_data_end, &cpi->common.error); + // Start / synch code cx_data[0] = 0x9D; cx_data[1] = 0x01; @@ -1573,10 +1598,11 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size) cx_data[5] = v; cx_data[6] = v >> 8; + extra_bytes_packed = 7; cx_data += extra_bytes_packed ; - vp8_start_encode(bc, cx_data); + vp8_start_encode(bc, cx_data, cx_data_end); // signal clr type vp8_write_bit(bc, pc->clr_type); @@ -1584,7 +1610,7 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size) } else - vp8_start_encode(bc, cx_data); + vp8_start_encode(bc, cx_data, cx_data_end); // Signal whether or not Segmentation is enabled @@ -1841,13 +1867,13 @@ void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size) int asize; num_part = 1 << pc->multi_token_partition; - pack_tokens_into_partitions(cpi, cx_data + bc->pos, num_part, &asize); + pack_tokens_into_partitions(cpi, cx_data + bc->pos, cx_data_end, num_part, &asize); *size += asize; } else { - vp8_start_encode(&cpi->bc2, cx_data + bc->pos); + vp8_start_encode(&cpi->bc2, cx_data + bc->pos, cx_data_end); #if CONFIG_MULTITHREAD if (cpi->b_multi_threaded) diff --git a/vp8/encoder/bitstream.h b/vp8/encoder/bitstream.h index f5d148ea4..8a875a5bd 100644 --- a/vp8/encoder/bitstream.h +++ b/vp8/encoder/bitstream.h @@ -27,13 +27,13 @@ void vp8cx_pack_mb_row_tokens_armv5(VP8_COMP *cpi, vp8_writer *w, const vp8_tree_index *); # define pack_tokens(a,b,c) \ vp8cx_pack_tokens_armv5(a,b,c,vp8_coef_encodings,vp8_extra_bits,vp8_coef_tree) -# define pack_tokens_into_partitions(a,b,c,d) \ +# define pack_tokens_into_partitions(a,b,unused,c,d) \ vp8cx_pack_tokens_into_partitions_armv5(a,b,c,d,vp8_coef_encodings,vp8_extra_bits,vp8_coef_tree) # define pack_mb_row_tokens(a,b) \ vp8cx_pack_mb_row_tokens_armv5(a,b,vp8_coef_encodings,vp8_extra_bits,vp8_coef_tree) #else # define pack_tokens(a,b,c) pack_tokens_c(a,b,c) -# define pack_tokens_into_partitions(a,b,c,d) pack_tokens_into_partitions_c(a,b,c,d) +# define pack_tokens_into_partitions(a,b,c,d,e) pack_tokens_into_partitions_c(a,b,c,d,e) # define pack_mb_row_tokens(a,b) pack_mb_row_tokens_c(a,b) #endif #endif diff --git a/vp8/encoder/boolhuff.c b/vp8/encoder/boolhuff.c index 08ae66b8d..d8ff5f933 100644 --- a/vp8/encoder/boolhuff.c +++ b/vp8/encoder/boolhuff.c @@ -40,15 +40,16 @@ const unsigned int vp8_prob_cost[256] = 22, 21, 19, 18, 16, 15, 13, 12, 10, 9, 7, 6, 4, 3, 1, 1 }; -void vp8_start_encode(BOOL_CODER *br, unsigned char *source) +void vp8_start_encode(BOOL_CODER *br, unsigned char *source, unsigned char *source_end) { - br->lowvalue = 0; - br->range = 255; - br->value = 0; - br->count = -24; - br->buffer = source; - br->pos = 0; + br->lowvalue = 0; + br->range = 255; + br->value = 0; + br->count = -24; + br->buffer = source; + br->buffer_end = source_end; + br->pos = 0; } void vp8_stop_encode(BOOL_CODER *br) diff --git a/vp8/encoder/boolhuff.h b/vp8/encoder/boolhuff.h index 04755f532..569b7791f 100644 --- a/vp8/encoder/boolhuff.h +++ b/vp8/encoder/boolhuff.h @@ -20,6 +20,7 @@ #define __INC_BOOLHUFF_H #include "vpx_ports/mem.h" +#include "vpx/internal/vpx_codec_internal.h" typedef struct { @@ -29,13 +30,15 @@ typedef struct int count; unsigned int pos; unsigned char *buffer; + unsigned char *buffer_end; + struct vpx_internal_error_info *error; // Variables used to track bit costs without outputing to the bitstream unsigned int measure_cost; unsigned long bit_counter; } BOOL_CODER; -extern void vp8_start_encode(BOOL_CODER *bc, unsigned char *buffer); +extern void vp8_start_encode(BOOL_CODER *bc, unsigned char *buffer, unsigned char *buffer_end); extern void vp8_encode_value(BOOL_CODER *br, int data, int bits); extern void vp8_stop_encode(BOOL_CODER *bc); @@ -44,7 +47,19 @@ extern const unsigned int vp8_prob_cost[256]; DECLARE_ALIGNED(16, extern const unsigned char, vp8_norm[256]); +static int validate_buffer(const unsigned char *start, + size_t len, + const unsigned char *end, + struct vpx_internal_error_info *error) +{ + if (start + len > start && start + len < end) + return 1; + else + vpx_internal_error(error, VPX_CODEC_CORRUPT_FRAME, + "Truncated packet or corrupt partition "); + return 0; +} static void vp8_encode_bool(BOOL_CODER *br, int bit, int probability) { unsigned int split; @@ -96,7 +111,9 @@ static void vp8_encode_bool(BOOL_CODER *br, int bit, int probability) br->buffer[x] += 1; } + validate_buffer(br->buffer + br->pos, 1, br->buffer_end, br->error); br->buffer[br->pos++] = (lowvalue >> (24 - offset)); + lowvalue <<= offset; shift = count; lowvalue &= 0xffffff; diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c index 43c971480..5a1cf1f9f 100644 --- a/vp8/encoder/onyx_if.c +++ b/vp8/encoder/onyx_if.c @@ -3447,6 +3447,7 @@ static void encode_frame_to_data_rate VP8_COMP *cpi, unsigned long *size, unsigned char *dest, + unsigned char* dest_end, unsigned int *frame_flags ) { @@ -4414,7 +4415,7 @@ static void encode_frame_to_data_rate #endif // build the bitstream - vp8_pack_bitstream(cpi, dest, size); + vp8_pack_bitstream(cpi, dest, dest_end, size); #if CONFIG_MULTITHREAD /* if PSNR packets are generated we have to wait for the lpf */ @@ -4827,13 +4828,13 @@ static void check_gf_quality(VP8_COMP *cpi) } #if !(CONFIG_REALTIME_ONLY) -static void Pass2Encode(VP8_COMP *cpi, unsigned long *size, unsigned char *dest, unsigned int *frame_flags) +static void Pass2Encode(VP8_COMP *cpi, unsigned long *size, unsigned char *dest, unsigned char * dest_end, unsigned int *frame_flags) { if (!cpi->common.refresh_alt_ref_frame) vp8_second_pass(cpi); - encode_frame_to_data_rate(cpi, size, dest, frame_flags); + encode_frame_to_data_rate(cpi, size, dest, dest_end, frame_flags); cpi->twopass.bits_left -= 8 * *size; if (!cpi->common.refresh_alt_ref_frame) @@ -4906,7 +4907,7 @@ static int frame_is_reference(const VP8_COMP *cpi) } -int vp8_get_compressed_data(VP8_PTR ptr, unsigned int *frame_flags, unsigned long *size, unsigned char *dest, int64_t *time_stamp, int64_t *time_end, int flush) +int vp8_get_compressed_data(VP8_PTR ptr, unsigned int *frame_flags, unsigned long *size, unsigned char *dest, unsigned char *dest_end, int64_t *time_stamp, int64_t *time_end, int flush) { #if HAVE_ARMV7 int64_t store_reg[8]; @@ -4921,6 +4922,14 @@ int vp8_get_compressed_data(VP8_PTR ptr, unsigned int *frame_flags, unsigned lon if (!cpi) return -1; + if (setjmp(cpi->common.error.jmp)){ + cpi->common.error.setjmp = 0; + return VPX_CODEC_CORRUPT_FRAME; + } + + cpi->bc.error = &cpi->common.error; + cpi->common.error.setjmp = 1; + #if HAVE_ARMV7 #if CONFIG_RUNTIME_CPU_DETECT if (cm->rtcd.flags & HAS_NEON) @@ -5134,11 +5143,11 @@ int vp8_get_compressed_data(VP8_PTR ptr, unsigned int *frame_flags, unsigned lon } else if (cpi->pass == 2) { - Pass2Encode(cpi, size, dest, frame_flags); + Pass2Encode(cpi, size, dest, dest_end, frame_flags); } else #endif - encode_frame_to_data_rate(cpi, size, dest, frame_flags); + encode_frame_to_data_rate(cpi, size, dest, dest_end, frame_flags); if (cpi->compressor_speed == 2) { @@ -5371,6 +5380,8 @@ int vp8_get_compressed_data(VP8_PTR ptr, unsigned int *frame_flags, unsigned lon } #endif + cpi->common.error.setjmp = 0; + return 0; } diff --git a/vp8/encoder/onyx_int.h b/vp8/encoder/onyx_int.h index 6678c15fb..0686a108a 100644 --- a/vp8/encoder/onyx_int.h +++ b/vp8/encoder/onyx_int.h @@ -683,7 +683,7 @@ void control_data_rate(VP8_COMP *cpi); void vp8_encode_frame(VP8_COMP *cpi); -void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned long *size); +void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned char *dest_end, unsigned long *size); void vp8_activity_masking(VP8_COMP *cpi, MACROBLOCK *x); diff --git a/vp8/vp8_cx_iface.c b/vp8/vp8_cx_iface.c index f8336240c..1be7e337f 100644 --- a/vp8/vp8_cx_iface.c +++ b/vp8/vp8_cx_iface.c @@ -761,6 +761,8 @@ static vpx_codec_err_t vp8e_encode(vpx_codec_alg_priv_t *ctx, int64_t dst_time_stamp, dst_end_time_stamp; unsigned long size, cx_data_sz; unsigned char *cx_data; + unsigned char *cx_data_end; + int comp_data_state = 0; /* Set up internal flags */ if (ctx->base.init_flags & VPX_CODEC_USE_PSNR) @@ -793,11 +795,25 @@ static vpx_codec_err_t vp8e_encode(vpx_codec_alg_priv_t *ctx, cx_data = ctx->cx_data; cx_data_sz = ctx->cx_data_sz; + cx_data_end = ctx->cx_data + cx_data_sz; lib_flags = 0; - while (cx_data_sz >= ctx->cx_data_sz / 2 - && -1 != vp8_get_compressed_data(ctx->cpi, &lib_flags, &size, cx_data, &dst_time_stamp, &dst_end_time_stamp, !img)) + while (cx_data_sz >= ctx->cx_data_sz / 2) { + comp_data_state = vp8_get_compressed_data(ctx->cpi, + &lib_flags, + &size, + cx_data, + cx_data_end, + &dst_time_stamp, + &dst_end_time_stamp, + !img); + + if(comp_data_state == VPX_CODEC_CORRUPT_FRAME) + return VPX_CODEC_CORRUPT_FRAME; + else if(comp_data_state == -1) + break; + if (size) { vpx_codec_pts_t round, delta;