From 7ea59de69c15554931851c12447d380ed58a83c9 Mon Sep 17 00:00:00 2001 From: Debargha Mukherjee Date: Thu, 10 Mar 2016 22:56:24 -0800 Subject: [PATCH] Fix an overflow in highbitdepth loop restoration Change-Id: Ie20cd35a4c96443c0de234d2cf097187a70ec8dd --- vp10/common/restoration.h | 3 +++ vp10/encoder/pickrst.c | 42 +++++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/vp10/common/restoration.h b/vp10/common/restoration.h index 43b140e60..980fe7275 100644 --- a/vp10/common/restoration.h +++ b/vp10/common/restoration.h @@ -42,6 +42,9 @@ extern "C" { #define WIENER_FILT_TAP1_BITS 4 #define WIENER_FILT_TAP2_BITS 5 +#define WIENER_FILT_BITS \ + ((WIENER_FILT_TAP0_BITS + WIENER_FILT_TAP1_BITS + WIENER_FILT_TAP2_BITS) * 2) + #define WIENER_FILT_TAP0_MAXV \ (WIENER_FILT_TAP0_MINV -1 + (1 << WIENER_FILT_TAP0_BITS)) #define WIENER_FILT_TAP1_MAXV \ diff --git a/vp10/encoder/pickrst.c b/vp10/encoder/pickrst.c index 998283674..13f955d50 100644 --- a/vp10/encoder/pickrst.c +++ b/vp10/encoder/pickrst.c @@ -9,6 +9,7 @@ */ #include +#include #include #include @@ -27,12 +28,12 @@ #include "vp10/encoder/picklpf.h" #include "vp10/encoder/pickrst.h" -static int try_restoration_frame(const YV12_BUFFER_CONFIG *sd, - VP10_COMP *const cpi, - RestorationInfo *rsi, - int partial_frame) { +static int64_t try_restoration_frame(const YV12_BUFFER_CONFIG *sd, + VP10_COMP *const cpi, + RestorationInfo *rsi, + int partial_frame) { VP10_COMMON *const cm = &cpi->common; - int filt_err; + int64_t filt_err; vp10_loop_restoration_frame(cm->frame_to_show, cm, rsi, 1, partial_frame); #if CONFIG_VP9_HIGHBITDEPTH @@ -55,7 +56,8 @@ static int search_bilateral_level(const YV12_BUFFER_CONFIG *sd, int filter_level, int partial_frame, double *best_cost_ret) { VP10_COMMON *const cm = &cpi->common; - int i, restoration_best, err; + int i, restoration_best; + int64_t err; double best_cost; double cost; const int restoration_level_bits = vp10_restoration_level_bits(&cpi->common); @@ -400,8 +402,8 @@ static void update_b_sep_sym(double **Mc, double **Hc, double *a, double *b) { } } -static void wiener_decompose_sep_sym(double *M, double *H, - double *a, double *b) { +static int wiener_decompose_sep_sym(double *M, double *H, + double *a, double *b) { static const double init_filt[RESTORATION_WIN] = { 0.035623, -0.127154, 0.211436, 0.760190, 0.211436, -0.127154, 0.035623, }; @@ -424,6 +426,7 @@ static void wiener_decompose_sep_sym(double *M, double *H, update_b_sep_sym(Mc, Hc, a, b); iter++; } + return 1; } #define CLIP(x, lo, hi) ((x) < (lo) ? (lo) : (x) > (hi) ? (hi) : (x)) @@ -448,7 +451,8 @@ static int search_wiener_filter(const YV12_BUFFER_CONFIG *src, double *best_cost_ret) { VP10_COMMON *const cm = &cpi->common; RestorationInfo rsi; - int err, bits; + int64_t err; + int bits; double cost_wiener, cost_norestore; MACROBLOCK *x = &cpi->td.mb; double M[RESTORATION_WIN2]; @@ -485,7 +489,10 @@ static int search_wiener_filter(const YV12_BUFFER_CONFIG *src, compute_stats(dgd->y_buffer, src->y_buffer, width, height, dgd_stride, src_stride, M, H); - wiener_decompose_sep_sym(M, H, vfilterd, hfilterd); + if (!wiener_decompose_sep_sym(M, H, vfilterd, hfilterd)) { + *best_cost_ret = DBL_MAX; + return 0; + } quantize_sym_filter(vfilterd, vfilter); quantize_sym_filter(hfilterd, hfilter); @@ -493,7 +500,7 @@ static int search_wiener_filter(const YV12_BUFFER_CONFIG *src, memcpy(rsi.vfilter, vfilter, sizeof(rsi.vfilter)); memcpy(rsi.hfilter, hfilter, sizeof(rsi.hfilter)); err = try_restoration_frame(src, cpi, &rsi, partial_frame); - bits = 22; + bits = WIENER_FILT_BITS; cost_wiener = RDCOST_DBL(x->rdmult, x->rddiv, (bits << 2), err); vpx_yv12_copy_y(&cpi->last_frame_uf, cm->frame_to_show); @@ -511,10 +518,10 @@ void vp10_pick_filter_restoration( const YV12_BUFFER_CONFIG *sd, VP10_COMP *cpi, LPF_PICK_METHOD method) { VP10_COMMON *const cm = &cpi->common; struct loopfilter *const lf = &cm->lf; - int wiener_success; - double cost_bilateral = 1e12; - double cost_wiener = 1e12; - double cost_norestore = 1e12; + int wiener_success = 0; + double cost_bilateral = DBL_MAX; + double cost_wiener = DBL_MAX; + double cost_norestore = DBL_MAX; lf->sharpness_level = cm->frame_type == KEY_FRAME ? 0 : cpi->oxcf.sharpness; @@ -577,8 +584,6 @@ void vp10_pick_filter_restoration( wiener_success = search_wiener_filter( sd, cpi, lf->filter_level, method == LPF_PICK_FROM_SUBIMAGE, cm->rst_info.vfilter, cm->rst_info.hfilter, &cost_wiener); - // printf("Costs %g %g (%d) %g\n", - // cost_norestore, cost_bilateral, lf->filter_level, cost_wiener); if (cost_bilateral < cost_wiener) { lf->filter_level = blf_filter_level; if (cm->rst_info.restoration_level != -1) @@ -591,5 +596,8 @@ void vp10_pick_filter_restoration( else cm->rst_info.restoration_type = RESTORE_NONE; } + // printf("[%d] Costs %g %g (%d) %g (%d)\n", cm->rst_info.restoration_type, + // cost_norestore, cost_bilateral, lf->filter_level, cost_wiener, + // wiener_success); } } -- 2.40.0