From 658e85425218483878fa76bb274b8da530b62b87 Mon Sep 17 00:00:00 2001 From: Jerome Jiang Date: Mon, 5 Jun 2017 11:09:05 -0700 Subject: [PATCH] Merge skin detection code in vp8/9. BUG=webm:1438 Change-Id: Ie3dc034c7dbb498a0b088a767b1936ddeed4df14 --- vp8/common/skin_detection.c | 73 +---------------------------- vp8/common/skin_detection.h | 11 ++--- vp8/encoder/onyx_if.c | 2 +- vp9/encoder/vp9_encoder.c | 3 +- vp9/encoder/vp9_skin_detection.c | 73 +---------------------------- vp9/encoder/vp9_skin_detection.h | 6 +-- vpx_dsp/skin_detection.c | 79 ++++++++++++++++++++++++++++++++ vpx_dsp/skin_detection.h | 26 +++++++++++ vpx_dsp/vpx_dsp.mk | 4 ++ 9 files changed, 120 insertions(+), 157 deletions(-) create mode 100644 vpx_dsp/skin_detection.c create mode 100644 vpx_dsp/skin_detection.h diff --git a/vp8/common/skin_detection.c b/vp8/common/skin_detection.c index 4fd2da81e..ed8f983a8 100644 --- a/vp8/common/skin_detection.c +++ b/vp8/common/skin_detection.c @@ -13,77 +13,6 @@ #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_mem/vpx_mem.h" -#define MODEL_MODE 1 - -// Fixed-point skin color model parameters. -static const int skin_mean[5][2] = { { 7463, 9614 }, - { 6400, 10240 }, - { 7040, 10240 }, - { 8320, 9280 }, - { 6800, 9614 } }; -static const int skin_inv_cov[4] = { 4107, 1663, 1663, 2157 }; // q16 -static const int skin_threshold[6] = { 1570636, 1400000, 800000, - 800000, 800000, 800000 }; // q18 - -// Thresholds on luminance. -static const int y_low = 40; -static const int y_high = 220; - -// Evaluates the Mahalanobis distance measure for the input CbCr values. -static int evaluate_skin_color_difference(const int cb, const int cr, - const int idx) { - const int cb_q6 = cb << 6; - const int cr_q6 = cr << 6; - const int cb_diff_q12 = - (cb_q6 - skin_mean[idx][0]) * (cb_q6 - skin_mean[idx][0]); - const int cbcr_diff_q12 = - (cb_q6 - skin_mean[idx][0]) * (cr_q6 - skin_mean[idx][1]); - const int cr_diff_q12 = - (cr_q6 - skin_mean[idx][1]) * (cr_q6 - skin_mean[idx][1]); - const int cb_diff_q2 = (cb_diff_q12 + (1 << 9)) >> 10; - const int cbcr_diff_q2 = (cbcr_diff_q12 + (1 << 9)) >> 10; - const int cr_diff_q2 = (cr_diff_q12 + (1 << 9)) >> 10; - const int skin_diff = - skin_inv_cov[0] * cb_diff_q2 + skin_inv_cov[1] * cbcr_diff_q2 + - skin_inv_cov[2] * cbcr_diff_q2 + skin_inv_cov[3] * cr_diff_q2; - return skin_diff; -} - -// Checks if the input yCbCr values corresponds to skin color. -int skin_pixel(int y, int cb, int cr, int motion) { - if (y < y_low || y > y_high) { - return 0; - } else { - if (MODEL_MODE == 0) { - return (evaluate_skin_color_difference(cb, cr, 0) < skin_threshold[0]); - } else { - int i = 0; - // Exit on grey. - if (cb == 128 && cr == 128) return 0; - // Exit on very strong cb. - if (cb > 150 && cr < 110) return 0; - for (; i < 5; ++i) { - int skin_color_diff = evaluate_skin_color_difference(cb, cr, i); - if (skin_color_diff < skin_threshold[i + 1]) { - if (y < 60 && skin_color_diff > 3 * (skin_threshold[i + 1] >> 2)) { - return 0; - } else if (motion == 0 && - skin_color_diff > (skin_threshold[i + 1] >> 1)) { - return 0; - } else { - return 1; - } - } - // Exit if difference is much large than the threshold. - if (skin_color_diff > (skin_threshold[i + 1] << 3)) { - return 0; - } - } - return 0; - } - } -} - int compute_skin_block(const uint8_t *y, const uint8_t *u, const uint8_t *v, int stride, int strideuv, int consec_zeromv, int curr_motion_magn) { @@ -103,7 +32,7 @@ int compute_skin_block(const uint8_t *y, const uint8_t *u, const uint8_t *v, v[4 * strideuv + 3] + v[4 * strideuv + 4]) >> 2; if (consec_zeromv > 25 && curr_motion_magn == 0) motion = 0; - return skin_pixel(ysource, usource, vsource, motion); + return vpx_skin_pixel(ysource, usource, vsource, motion); } } diff --git a/vp8/common/skin_detection.h b/vp8/common/skin_detection.h index 4cb22a79a..1fe233212 100644 --- a/vp8/common/skin_detection.h +++ b/vp8/common/skin_detection.h @@ -8,11 +8,12 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef VP8_ENCODER_SKIN_DETECTION_H_ -#define VP8_ENCODER_SKIN_DETECTION_H_ +#ifndef VP8_COMMON_SKIN_DETECTION_H_ +#define VP8_COMMON_SKIN_DETECTION_H_ #include "vp8/encoder/onyx_int.h" #include "vpx/vpx_integer.h" +#include "vpx_dsp/skin_detection.h" #include "vpx_scale/yv12config.h" #ifdef __cplusplus @@ -21,10 +22,6 @@ extern "C" { struct VP8_COMP; -// #define OUTPUT_YUV_SKINMAP - -int skin_pixel(int y, int cb, int cr, int motion); - int compute_skin_block(const uint8_t *y, const uint8_t *u, const uint8_t *v, int stride, int strideuv, int consec_zeromv, int curr_motion_magn); @@ -39,4 +36,4 @@ extern void vp8_write_yuv_frame(FILE *f, YV12_BUFFER_CONFIG *s); } // extern "C" #endif -#endif // VP8_ENCODER_SKIN_DETECTION_H_ +#endif // VP8_COMMON_SKIN_DETECTION_H_ diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c index 9227f750d..9a9f3d1f8 100644 --- a/vp8/encoder/onyx_if.c +++ b/vp8/encoder/onyx_if.c @@ -89,7 +89,7 @@ FILE *yuv_file; FILE *yuv_denoised_file; #endif #ifdef OUTPUT_YUV_SKINMAP -FILE *yuv_skinmap_file = NULL; +static FILE *yuv_skinmap_file = NULL; #endif #if 0 diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c index 00db9d57d..c7793488f 100644 --- a/vp9/encoder/vp9_encoder.c +++ b/vp9/encoder/vp9_encoder.c @@ -80,7 +80,7 @@ FILE *yuv_denoised_file = NULL; #endif #ifdef OUTPUT_YUV_SKINMAP -FILE *yuv_skinmap_file = NULL; +static FILE *yuv_skinmap_file = NULL; #endif #ifdef OUTPUT_YUV_REC FILE *yuv_rec_file; @@ -2586,6 +2586,7 @@ int vp9_update_entropy(VP9_COMP *cpi, int update) { // as YUV 420. We simply use the top-left pixels of the UV buffers, since we do // not denoise the UV channels at this time. If ever we implement UV channel // denoising we will have to modify this. +// TODO(jianj): Remove the duplicated one in vp8 and move it to vpx_util. void vp9_write_yuv_frame_420(YV12_BUFFER_CONFIG *s, FILE *f) { uint8_t *src = s->y_buffer; int h = s->y_crop_height; diff --git a/vp9/encoder/vp9_skin_detection.c b/vp9/encoder/vp9_skin_detection.c index 3f3d48fb9..4cdac723a 100644 --- a/vp9/encoder/vp9_skin_detection.c +++ b/vp9/encoder/vp9_skin_detection.c @@ -15,75 +15,6 @@ #include "vp9/encoder/vp9_encoder.h" #include "vp9/encoder/vp9_skin_detection.h" -#define MODEL_MODE 1 - -// Fixed-point skin color model parameters. -static const int skin_mean[5][2] = { { 7463, 9614 }, - { 6400, 10240 }, - { 7040, 10240 }, - { 8320, 9280 }, - { 6800, 9614 } }; -static const int skin_inv_cov[4] = { 4107, 1663, 1663, 2157 }; // q16 -static const int skin_threshold[6] = { 1570636, 1400000, 800000, - 800000, 800000, 800000 }; // q18 - -// Thresholds on luminance. -static const int y_low = 40; -static const int y_high = 220; - -// Evaluates the Mahalanobis distance measure for the input CbCr values. -static int evaluate_skin_color_difference(int cb, int cr, int idx) { - const int cb_q6 = cb << 6; - const int cr_q6 = cr << 6; - const int cb_diff_q12 = - (cb_q6 - skin_mean[idx][0]) * (cb_q6 - skin_mean[idx][0]); - const int cbcr_diff_q12 = - (cb_q6 - skin_mean[idx][0]) * (cr_q6 - skin_mean[idx][1]); - const int cr_diff_q12 = - (cr_q6 - skin_mean[idx][1]) * (cr_q6 - skin_mean[idx][1]); - const int cb_diff_q2 = (cb_diff_q12 + (1 << 9)) >> 10; - const int cbcr_diff_q2 = (cbcr_diff_q12 + (1 << 9)) >> 10; - const int cr_diff_q2 = (cr_diff_q12 + (1 << 9)) >> 10; - const int skin_diff = - skin_inv_cov[0] * cb_diff_q2 + skin_inv_cov[1] * cbcr_diff_q2 + - skin_inv_cov[2] * cbcr_diff_q2 + skin_inv_cov[3] * cr_diff_q2; - return skin_diff; -} - -int vp9_skin_pixel(const uint8_t y, const uint8_t cb, const uint8_t cr, - int motion) { - if (y < y_low || y > y_high) { - return 0; - } else { - if (MODEL_MODE == 0) { - return (evaluate_skin_color_difference(cb, cr, 0) < skin_threshold[0]); - } else { - int i = 0; - // Exit on grey. - if (cb == 128 && cr == 128) return 0; - // Exit on very strong cb. - if (cb > 150 && cr < 110) return 0; - for (; i < 5; i++) { - int skin_color_diff = evaluate_skin_color_difference(cb, cr, i); - if (skin_color_diff < skin_threshold[i + 1]) { - if (y < 60 && skin_color_diff > 3 * (skin_threshold[i + 1] >> 2)) - return 0; - else if (motion == 0 && - skin_color_diff > (skin_threshold[i + 1] >> 1)) - return 0; - else - return 1; - } - // Exit if difference is much large than the threshold. - if (skin_color_diff > (skin_threshold[i + 1] << 3)) { - return 0; - } - } - return 0; - } - } -} - int vp9_compute_skin_block(const uint8_t *y, const uint8_t *u, const uint8_t *v, int stride, int strideuv, int bsize, int consec_zeromv, int curr_motion_magn) { @@ -101,7 +32,7 @@ int vp9_compute_skin_block(const uint8_t *y, const uint8_t *u, const uint8_t *v, const uint8_t usource = u[uv_height_shift * strideuv + uv_width_shift]; const uint8_t vsource = v[uv_height_shift * strideuv + uv_width_shift]; if (consec_zeromv > 25 && curr_motion_magn == 0) motion = 0; - return vp9_skin_pixel(ysource, usource, vsource, motion); + return vpx_skin_pixel(ysource, usource, vsource, motion); } } @@ -159,7 +90,7 @@ void vp9_compute_skin_map(VP9_COMP *const cpi, FILE *yuv_skinmap_file) { ysource = (ysource + ysource2 + ysource3 + ysource4) >> 2; usource = (usource + usource2 + usource3 + usource4) >> 2; vsource = (vsource + vsource2 + vsource3 + vsource4) >> 2; - is_skin = vp9_skin_pixel(ysource, usource, vsource, 1); + is_skin = vpx_skin_pixel(ysource, usource, vsource, 1); } else { int block_size = BLOCK_8X8; int consec_zeromv = 0; diff --git a/vp9/encoder/vp9_skin_detection.h b/vp9/encoder/vp9_skin_detection.h index c77382dbd..1c6a081f4 100644 --- a/vp9/encoder/vp9_skin_detection.h +++ b/vp9/encoder/vp9_skin_detection.h @@ -12,6 +12,7 @@ #define VP9_ENCODER_VP9_SKIN_MAP_H_ #include "vp9/common/vp9_blockd.h" +#include "vpx_dsp/skin_detection.h" #ifdef __cplusplus extern "C" { @@ -19,11 +20,6 @@ extern "C" { struct VP9_COMP; -// #define OUTPUT_YUV_SKINMAP - -int vp9_skin_pixel(const uint8_t y, const uint8_t cb, const uint8_t cr, - int motion); - int vp9_compute_skin_block(const uint8_t *y, const uint8_t *u, const uint8_t *v, int stride, int strideuv, int bsize, int consec_zeromv, int curr_motion_magn); diff --git a/vpx_dsp/skin_detection.c b/vpx_dsp/skin_detection.c new file mode 100644 index 000000000..bbbb6c3a1 --- /dev/null +++ b/vpx_dsp/skin_detection.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2017 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "vpx_dsp/skin_detection.h" + +#define MODEL_MODE 1 + +// Fixed-point skin color model parameters. +static const int skin_mean[5][2] = { { 7463, 9614 }, + { 6400, 10240 }, + { 7040, 10240 }, + { 8320, 9280 }, + { 6800, 9614 } }; +static const int skin_inv_cov[4] = { 4107, 1663, 1663, 2157 }; // q16 +static const int skin_threshold[6] = { 1570636, 1400000, 800000, + 800000, 800000, 800000 }; // q18 +// Thresholds on luminance. +static const int y_low = 40; +static const int y_high = 220; + +// Evaluates the Mahalanobis distance measure for the input CbCr values. +static int vpx_evaluate_skin_color_difference(const int cb, const int cr, + const int idx) { + const int cb_q6 = cb << 6; + const int cr_q6 = cr << 6; + const int cb_diff_q12 = + (cb_q6 - skin_mean[idx][0]) * (cb_q6 - skin_mean[idx][0]); + const int cbcr_diff_q12 = + (cb_q6 - skin_mean[idx][0]) * (cr_q6 - skin_mean[idx][1]); + const int cr_diff_q12 = + (cr_q6 - skin_mean[idx][1]) * (cr_q6 - skin_mean[idx][1]); + const int cb_diff_q2 = (cb_diff_q12 + (1 << 9)) >> 10; + const int cbcr_diff_q2 = (cbcr_diff_q12 + (1 << 9)) >> 10; + const int cr_diff_q2 = (cr_diff_q12 + (1 << 9)) >> 10; + const int skin_diff = + skin_inv_cov[0] * cb_diff_q2 + skin_inv_cov[1] * cbcr_diff_q2 + + skin_inv_cov[2] * cbcr_diff_q2 + skin_inv_cov[3] * cr_diff_q2; + return skin_diff; +} + +// Checks if the input yCbCr values corresponds to skin color. +int vpx_skin_pixel(const int y, const int cb, const int cr, int motion) { + if (y < y_low || y > y_high) { + return 0; + } else if (MODEL_MODE == 0) { + return (vpx_evaluate_skin_color_difference(cb, cr, 0) < skin_threshold[0]); + } else { + int i = 0; + // Exit on grey. + if (cb == 128 && cr == 128) return 0; + // Exit on very strong cb. + if (cb > 150 && cr < 110) return 0; + for (; i < 5; ++i) { + int skin_color_diff = vpx_evaluate_skin_color_difference(cb, cr, i); + if (skin_color_diff < skin_threshold[i + 1]) { + if (y < 60 && skin_color_diff > 3 * (skin_threshold[i + 1] >> 2)) { + return 0; + } else if (motion == 0 && + skin_color_diff > (skin_threshold[i + 1] >> 1)) { + return 0; + } else { + return 1; + } + } + // Exit if difference is much large than the threshold. + if (skin_color_diff > (skin_threshold[i + 1] << 3)) { + return 0; + } + } + return 0; + } +} diff --git a/vpx_dsp/skin_detection.h b/vpx_dsp/skin_detection.h new file mode 100644 index 000000000..0f87bb521 --- /dev/null +++ b/vpx_dsp/skin_detection.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_DSP_SKIN_DETECTION_H_ +#define VPX_DSP_SKIN_DETECTION_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// #define OUTPUT_YUV_SKINMAP + +int vpx_skin_pixel(const int y, const int cb, const int cr, int motion); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // VPX_DSP_SKIN_DETECTION_H_ diff --git a/vpx_dsp/vpx_dsp.mk b/vpx_dsp/vpx_dsp.mk index da057c883..ec124e852 100644 --- a/vpx_dsp/vpx_dsp.mk +++ b/vpx_dsp/vpx_dsp.mk @@ -286,6 +286,10 @@ DSP_SRCS-$(HAVE_VSX) += ppc/hadamard_vsx.c endif # CONFIG_VP9_ENCODER +# skin detection +DSP_SRCS-yes += skin_detection.h +DSP_SRCS-yes += skin_detection.c + ifeq ($(CONFIG_ENCODERS),yes) DSP_SRCS-yes += sad.c DSP_SRCS-yes += subtract.c -- 2.40.0