From 8a5cb084e6e40ba0f6e979843a599bae85a24812 Mon Sep 17 00:00:00 2001 From: angiebird Date: Thu, 23 Jan 2020 14:56:01 -0800 Subject: [PATCH] Add coded_frame to EncodeFrameResults This coded_frame represents the raw coded image. Change-Id: Iea439da2f9e84c4507b082d77ebaac49bfd74fff --- vp9/encoder/vp9_encoder.c | 45 +++++++++-- vp9/encoder/vp9_encoder.h | 11 +++ vp9/simple_encode.cc | 162 +++++++++++++++++++++++++++++++------- vp9/simple_encode.h | 14 +++- vp9/vp9_cx_iface.c | 4 +- 5 files changed, 200 insertions(+), 36 deletions(-) diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c index 8120bf563..49514f8e9 100644 --- a/vp9/encoder/vp9_encoder.c +++ b/vp9/encoder/vp9_encoder.c @@ -7102,10 +7102,6 @@ static void setup_tpl_stats(VP9_COMP *cpi) { #endif // CONFIG_NON_GREEDY_MV } -static void init_encode_frame_result(ENCODE_FRAME_RESULT *encode_frame_result) { - encode_frame_result->show_idx = -1; // Actual encoding doesn't happen. -} - #if !CONFIG_REALTIME_ONLY #if CONFIG_RATE_CTRL static void copy_frame_counts(const FRAME_COUNTS *input_counts, @@ -7224,6 +7220,34 @@ static void copy_frame_counts(const FRAME_COUNTS *input_counts, } } } + +static void yv12_buffer_to_image_buffer(const YV12_BUFFER_CONFIG *yv12_buffer, + IMAGE_BUFFER *image_buffer) { + const uint8_t *src_buf_ls[3] = { yv12_buffer->y_buffer, yv12_buffer->u_buffer, + yv12_buffer->v_buffer }; + const int src_stride_ls[3] = { yv12_buffer->y_stride, yv12_buffer->uv_stride, + yv12_buffer->uv_stride }; + const int w_ls[3] = { yv12_buffer->y_crop_width, yv12_buffer->uv_crop_width, + yv12_buffer->uv_crop_width }; + const int h_ls[3] = { yv12_buffer->y_crop_height, yv12_buffer->uv_crop_height, + yv12_buffer->uv_crop_height }; + int plane; + for (plane = 0; plane < 3; ++plane) { + const int src_stride = src_stride_ls[plane]; + const int w = w_ls[plane]; + const int h = h_ls[plane]; + const uint8_t *src_buf = src_buf_ls[plane]; + uint8_t *dst_buf = image_buffer->plane_buffer[plane]; + int r; + assert(image_buffer->plane_width[plane] == w); + assert(image_buffer->plane_height[plane] == h); + for (r = 0; r < h; ++r) { + memcpy(dst_buf, src_buf, sizeof(*src_buf) * w); + src_buf += src_stride; + dst_buf += w; + } + } +} #endif // CONFIG_RATE_CTRL static void update_encode_frame_result( @@ -7249,6 +7273,9 @@ static void update_encode_frame_result( encode_frame_result->sse = psnr.sse[0]; copy_frame_counts(counts, &encode_frame_result->frame_counts); encode_frame_result->partition_info = partition_info; + if (encode_frame_result->coded_frame.allocated) { + yv12_buffer_to_image_buffer(coded_frame, &encode_frame_result->coded_frame); + } #else // CONFIG_RATE_CTRL (void)bit_depth; (void)input_bit_depth; @@ -7262,6 +7289,14 @@ static void update_encode_frame_result( } #endif // !CONFIG_REALTIME_ONLY +void vp9_init_encode_frame_result(ENCODE_FRAME_RESULT *encode_frame_result) { + encode_frame_result->show_idx = -1; // Actual encoding doesn't happen. +#if CONFIG_RATE_CTRL + vp9_zero(encode_frame_result->coded_frame); + encode_frame_result->coded_frame.allocated = 0; +#endif // CONFIG_RATE_CTRL +} + int vp9_get_compressed_data(VP9_COMP *cpi, unsigned int *frame_flags, size_t *size, uint8_t *dest, int64_t *time_stamp, int64_t *time_end, int flush, @@ -7277,7 +7312,6 @@ int vp9_get_compressed_data(VP9_COMP *cpi, unsigned int *frame_flags, int arf_src_index; const int gf_group_index = cpi->twopass.gf_group.index; int i; - init_encode_frame_result(encode_frame_result); if (is_one_pass_cbr_svc(cpi)) { vp9_one_pass_cbr_svc_start_layer(cpi); @@ -7515,6 +7549,7 @@ int vp9_get_compressed_data(VP9_COMP *cpi, unsigned int *frame_flags, cpi->td.mb.fp_src_pred = 0; #if CONFIG_REALTIME_ONLY + (void)encode_frame_result; if (cpi->use_svc) { SvcEncode(cpi, size, dest, frame_flags); } else { diff --git a/vp9/encoder/vp9_encoder.h b/vp9/encoder/vp9_encoder.h index be02494d9..59f0e944c 100644 --- a/vp9/encoder/vp9_encoder.h +++ b/vp9/encoder/vp9_encoder.h @@ -913,6 +913,14 @@ static INLINE void free_motion_vector_info(struct VP9_COMP *cpi) { vpx_free(cpi->motion_vector_info); cpi->motion_vector_info = NULL; } + +// This is the c-version counter part of ImageBuffer +typedef struct IMAGE_BUFFER { + int allocated; + int plane_width[3]; + int plane_height[3]; + uint8_t *plane_buffer[3]; +} IMAGE_BUFFER; #endif // CONFIG_RATE_CTRL typedef struct ENCODE_FRAME_RESULT { @@ -924,10 +932,13 @@ typedef struct ENCODE_FRAME_RESULT { FRAME_COUNTS frame_counts; const PARTITION_INFO *partition_info; const MOTION_VECTOR_INFO *motion_vector_info; + IMAGE_BUFFER coded_frame; #endif // CONFIG_RATE_CTRL int quantize_index; } ENCODE_FRAME_RESULT; +void vp9_init_encode_frame_result(ENCODE_FRAME_RESULT *encode_frame_result); + void vp9_initialize_enc(void); void vp9_update_compressor_with_img_fmt(VP9_COMP *cpi, vpx_img_fmt_t img_fmt); diff --git a/vp9/simple_encode.cc b/vp9/simple_encode.cc index 9ee4853f8..e9c985df5 100644 --- a/vp9/simple_encode.cc +++ b/vp9/simple_encode.cc @@ -21,6 +21,37 @@ namespace vp9 { +static int get_plane_height(vpx_img_fmt_t img_fmt, int frame_height, + int plane) { + assert(plane < 3); + if (plane == 0) { + return frame_height; + } + switch (img_fmt) { + case VPX_IMG_FMT_I420: + case VPX_IMG_FMT_I440: + case VPX_IMG_FMT_YV12: + case VPX_IMG_FMT_I42016: + case VPX_IMG_FMT_I44016: return (frame_height + 1) >> 1; + default: return frame_height; + } +} + +static int get_plane_width(vpx_img_fmt_t img_fmt, int frame_width, int plane) { + assert(plane < 3); + if (plane == 0) { + return frame_width; + } + switch (img_fmt) { + case VPX_IMG_FMT_I420: + case VPX_IMG_FMT_YV12: + case VPX_IMG_FMT_I422: + case VPX_IMG_FMT_I42016: + case VPX_IMG_FMT_I42216: return (frame_width + 1) >> 1; + default: return frame_width; + } +} + // TODO(angiebird): Merge this function with vpx_img_plane_width() static int img_plane_width(const vpx_image_t *img, int plane) { if (plane > 0 && img->x_chroma_shift > 0) @@ -339,6 +370,70 @@ static void update_frame_counts(const FRAME_COUNTS *input_counts, } } +void output_image_buffer(const ImageBuffer &image_buffer, std::FILE *out_file) { + for (int plane = 0; plane < 3; ++plane) { + const int w = image_buffer.plane_width[plane]; + const int h = image_buffer.plane_height[plane]; + const uint8_t *buf = image_buffer.plane_buffer[plane].get(); + fprintf(out_file, "%d %d\n", h, w); + for (int i = 0; i < w * h; ++i) { + fprintf(out_file, "%d ", (int)buf[i]); + } + fprintf(out_file, "\n"); + } +} + +static bool init_image_buffer(ImageBuffer *image_buffer, int frame_width, + int frame_height, vpx_img_fmt_t img_fmt) { + for (int plane = 0; plane < 3; ++plane) { + const int w = get_plane_width(img_fmt, frame_width, plane); + const int h = get_plane_height(img_fmt, frame_height, plane); + image_buffer->plane_width[plane] = w; + image_buffer->plane_height[plane] = h; + image_buffer->plane_buffer[plane].reset(new (std::nothrow) uint8_t[w * h]); + if (image_buffer->plane_buffer[plane].get() == nullptr) { + return false; + } + } + return true; +} + +static void ImageBuffer_to_IMAGE_BUFFER(const ImageBuffer &image_buffer, + IMAGE_BUFFER *image_buffer_c) { + image_buffer_c->allocated = 1; + for (int plane = 0; plane < 3; ++plane) { + image_buffer_c->plane_width[plane] = image_buffer.plane_width[plane]; + image_buffer_c->plane_height[plane] = image_buffer.plane_height[plane]; + image_buffer_c->plane_buffer[plane] = + image_buffer.plane_buffer[plane].get(); + } +} + +static size_t get_max_coding_data_byte_size(int frame_width, int frame_height) { + return frame_width * frame_height * 3; +} + +static bool init_encode_frame_result(EncodeFrameResult *encode_frame_result, + int frame_width, int frame_height, + vpx_img_fmt_t img_fmt) { + const size_t max_coding_data_byte_size = + get_max_coding_data_byte_size(frame_width, frame_height); + + encode_frame_result->coding_data.reset( + new (std::nothrow) uint8_t[max_coding_data_byte_size]); + + encode_frame_result->num_rows_4x4 = get_num_unit_4x4(frame_width); + encode_frame_result->num_cols_4x4 = get_num_unit_4x4(frame_height); + encode_frame_result->partition_info.resize(encode_frame_result->num_rows_4x4 * + encode_frame_result->num_cols_4x4); + + if (encode_frame_result->coding_data.get() == nullptr) { + return false; + } + return init_image_buffer(&encode_frame_result->coded_frame, frame_width, + frame_height, img_fmt); +} + static void update_encode_frame_result( EncodeFrameResult *encode_frame_result, const ENCODE_FRAME_RESULT *encode_frame_info) { @@ -426,8 +521,6 @@ SimpleEncode::SimpleEncode(int frame_width, int frame_height, impl_ptr_ = std::unique_ptr(new EncodeImpl()); frame_width_ = frame_width; frame_height_ = frame_height; - num_rows_4x4_ = get_num_unit_4x4(frame_width); - num_cols_4x4_ = get_num_unit_4x4(frame_height); frame_rate_num_ = frame_rate_num; frame_rate_den_ = frame_rate_den; target_bitrate_ = target_bitrate; @@ -473,6 +566,7 @@ void SimpleEncode::ComputeFirstPassStats() { size_t size; unsigned int frame_flags = 0; ENCODE_FRAME_RESULT encode_frame_info; + vp9_init_encode_frame_result(&encode_frame_info); // TODO(angiebird): Call vp9_first_pass directly vp9_get_compressed_data(cpi, &frame_flags, &size, NULL, &time_stamp, &time_end, flush, &encode_frame_info); @@ -585,32 +679,44 @@ void SimpleEncode::EncodeFrame(EncodeFrameResult *encode_frame_result) { break; } } - assert(encode_frame_result->coding_data.get() == nullptr); - const size_t max_coding_data_byte_size = frame_width_ * frame_height_ * 3; - encode_frame_result->coding_data = std::move( - std::unique_ptr(new uint8_t[max_coding_data_byte_size])); - encode_frame_result->num_rows_4x4 = num_rows_4x4_; - encode_frame_result->num_cols_4x4 = num_cols_4x4_; - encode_frame_result->partition_info.resize(num_rows_4x4_ * num_cols_4x4_); - int64_t time_stamp; - int64_t time_end; - int flush = 1; // Make vp9_get_compressed_data encode a frame - unsigned int frame_flags = 0; - ENCODE_FRAME_RESULT encode_frame_info; - vp9_get_compressed_data(cpi, &frame_flags, - &encode_frame_result->coding_data_byte_size, - encode_frame_result->coding_data.get(), &time_stamp, - &time_end, flush, &encode_frame_info); - // vp9_get_compressed_data is expected to encode a frame every time, so the - // data size should be greater than zero. - assert(encode_frame_result->coding_data_byte_size > 0); - assert(encode_frame_result->coding_data_byte_size < - max_coding_data_byte_size); - - update_encode_frame_result(encode_frame_result, &encode_frame_info); - IncreaseGroupOfPictureIndex(&group_of_picture_); - if (IsGroupOfPictureFinished(group_of_picture_)) { - UpdateGroupOfPicture(impl_ptr_->cpi, &group_of_picture_); + + if (init_encode_frame_result(encode_frame_result, frame_width_, frame_height_, + impl_ptr_->img_fmt)) { + int64_t time_stamp; + int64_t time_end; + int flush = 1; // Make vp9_get_compressed_data encode a frame + unsigned int frame_flags = 0; + ENCODE_FRAME_RESULT encode_frame_info; + vp9_init_encode_frame_result(&encode_frame_info); + ImageBuffer_to_IMAGE_BUFFER(encode_frame_result->coded_frame, + &encode_frame_info.coded_frame); + vp9_get_compressed_data(cpi, &frame_flags, + &encode_frame_result->coding_data_byte_size, + encode_frame_result->coding_data.get(), &time_stamp, + &time_end, flush, &encode_frame_info); + // vp9_get_compressed_data is expected to encode a frame every time, so the + // data size should be greater than zero. + if (encode_frame_result->coding_data_byte_size <= 0) { + fprintf(stderr, "Coding data size <= 0\n"); + abort(); + } + const size_t max_coding_data_byte_size = + get_max_coding_data_byte_size(frame_width_, frame_height_); + if (encode_frame_result->coding_data_byte_size > + max_coding_data_byte_size) { + fprintf(stderr, "Coding data size exceeds the maximum.\n"); + abort(); + } + + update_encode_frame_result(encode_frame_result, &encode_frame_info); + IncreaseGroupOfPictureIndex(&group_of_picture_); + if (IsGroupOfPictureFinished(group_of_picture_)) { + UpdateGroupOfPicture(impl_ptr_->cpi, &group_of_picture_); + } + } else { + // TODO(angiebird): Clean up encode_frame_result. + fprintf(stderr, "init_encode_frame_result() failed.\n"); + this->EndEncode(); } } diff --git a/vp9/simple_encode.h b/vp9/simple_encode.h index b30fa635f..741c49c7e 100644 --- a/vp9/simple_encode.h +++ b/vp9/simple_encode.h @@ -130,6 +130,17 @@ struct FrameCounts { NewMotionVectorContextCounts mv; }; +struct ImageBuffer { + // The image data is stored in raster order, + // i.e. image[plane][r][c] = + // plane_buffer[plane][r * plane_width[plane] + plane_height[plane]]. + std::unique_ptr plane_buffer[3]; + int plane_width[3]; + int plane_height[3]; +}; + +void output_image_buffer(const ImageBuffer &image_buffer, std::FILE *out_file); + struct EncodeFrameResult { int show_idx; FrameType frame_type; @@ -164,6 +175,7 @@ struct EncodeFrameResult { // Horizontal next: |column_start| + |width|, // Vertical next: |row_start| + |height|. std::vector partition_info; + ImageBuffer coded_frame; }; struct GroupOfPicture { @@ -252,8 +264,6 @@ class SimpleEncode { int frame_width_; // frame width in pixels. int frame_height_; // frame height in pixels. - int num_rows_4x4_; // number of row units, in size of 4. - int num_cols_4x4_; // number of column units, in size of 4. int frame_rate_num_; int frame_rate_den_; int target_bitrate_; diff --git a/vp9/vp9_cx_iface.c b/vp9/vp9_cx_iface.c index 3aa4c3ddf..9ec2cf276 100644 --- a/vp9/vp9_cx_iface.c +++ b/vp9/vp9_cx_iface.c @@ -1205,8 +1205,9 @@ static vpx_codec_err_t encoder_encode(vpx_codec_alg_priv_t *ctx, // compute first pass stats if (img) { int ret; - ENCODE_FRAME_RESULT encode_frame_result; vpx_codec_cx_pkt_t fps_pkt; + ENCODE_FRAME_RESULT encode_frame_result; + vp9_init_encode_frame_result(&encode_frame_result); // TODO(angiebird): Call vp9_first_pass directly ret = vp9_get_compressed_data(cpi, &lib_flags, &size, cx_data, &dst_time_stamp, &dst_end_time_stamp, @@ -1229,6 +1230,7 @@ static vpx_codec_err_t encoder_encode(vpx_codec_alg_priv_t *ctx, #endif // !CONFIG_REALTIME_ONLY } else { ENCODE_FRAME_RESULT encode_frame_result; + vp9_init_encode_frame_result(&encode_frame_result); while (cx_data_sz >= ctx->cx_data_sz / 2 && -1 != vp9_get_compressed_data(cpi, &lib_flags, &size, cx_data, &dst_time_stamp, &dst_end_time_stamp, -- 2.40.0