Fix a bug in HBD buffer size computation

The value of use_highbitdepth flag is used for compute the size for
high bit depth buffer allocation, which should take value 0 or 1
depending on if the buffer is used for high bit depth or not.
Previously, the values is set to 8 or 0, this commit fixes the issue
and properly set the value for this flag to 1 or 0.

This cuts the size of highbitdepth buffer memory allocation to 2/9 of
the size prior to the fix.

Change-Id: I401518b5a6147e5d8a973e54f7ca6bc1892065e0

diff --git a/vp10/common/onyxc_int.h b/vp10/common/onyxc_int.h
index 53de852d190d734eaacd519c741e4ea655090ce1..b6051fd812030ba8f16dffeb619e173bce808e5f 100644 (file)
--- a/vp10/common/onyxc_int.h
+++ b/vp10/common/onyxc_int.h
@@ -146,7 +146,8 @@ typedef struct VP10Common {
   int subsampling_y;
 
 #if CONFIG_VP9_HIGHBITDEPTH
-  int use_highbitdepth;  // Marks if we need to use 16bit frame buffers.
+  // Marks if we need to use 16bit frame buffers (1: yes, 0: no).
+  int use_highbitdepth;
 #endif
 
   YV12_BUFFER_CONFIG *frame_to_show;

diff --git a/vp10/encoder/encoder.c b/vp10/encoder/encoder.c
index 2a617c2728ed82e4398a760f917aa99b6e7c88bf..739791ce6f2ba764f5abaec01ad83decc70fcd10 100644 (file)
--- a/vp10/encoder/encoder.c
+++ b/vp10/encoder/encoder.c
@@ -4010,7 +4010,7 @@ int vp10_receive_raw_frame(VP10_COMP *cpi, unsigned int frame_flags,
   const int subsampling_x = sd->subsampling_x;
   const int subsampling_y = sd->subsampling_y;
 #if CONFIG_VP9_HIGHBITDEPTH
-  const int use_highbitdepth = sd->flags & YV12_FLAG_HIGHBITDEPTH;
+  const int use_highbitdepth = (sd->flags & YV12_FLAG_HIGHBITDEPTH) != 0;
   check_initial_width(cpi, use_highbitdepth, subsampling_x, subsampling_y);
 #else
   check_initial_width(cpi, subsampling_x, subsampling_y);