From ad1c2c8e383cb0f23ba8a0ba2ae211ad9f5eba62 Mon Sep 17 00:00:00 2001 From: Anton Mitrofanov Date: Tue, 28 Jun 2011 21:39:09 +0400 Subject: [PATCH] Fix resize filter crash with YUVJ* input pixfmt --- filters/video/resize.c | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/filters/video/resize.c b/filters/video/resize.c index 9a4ccc80..cc51cd8c 100644 --- a/filters/video/resize.c +++ b/filters/video/resize.c @@ -339,6 +339,27 @@ static int handle_opts( const char **optlist, char **opts, video_info_t *info, r return 0; } +static int handle_jpeg( int *format ) +{ + switch( *format ) + { + case PIX_FMT_YUVJ420P: + *format = PIX_FMT_YUV420P; + return 1; + case PIX_FMT_YUVJ422P: + *format = PIX_FMT_YUV422P; + return 1; + case PIX_FMT_YUVJ444P: + *format = PIX_FMT_YUV444P; + return 1; + case PIX_FMT_YUVJ440P: + *format = PIX_FMT_YUV440P; + return 1; + default: + return 0; + } +} + static int x264_init_sws_context( resizer_hnd_t *h ) { if( !h->ctx ) @@ -348,22 +369,28 @@ static int x264_init_sws_context( resizer_hnd_t *h ) return -1; /* set flags that will not change */ + int dst_format = h->dst.pix_fmt; + int dst_range = handle_jpeg( &dst_format ); av_set_int( h->ctx, "sws_flags", h->ctx_flags ); av_set_int( h->ctx, "dstw", h->dst.width ); av_set_int( h->ctx, "dsth", h->dst.height ); - av_set_int( h->ctx, "dst_format", h->dst.pix_fmt ); - av_set_int( h->ctx, "dst_range", 0 ); /* FIXME: use the correct full range value */ + av_set_int( h->ctx, "dst_format", dst_format ); + av_set_int( h->ctx, "dst_range", dst_range ); /* FIXME: use the correct full range value */ } + int src_format = h->scale.pix_fmt; + int src_range = handle_jpeg( &src_format ); av_set_int( h->ctx, "srcw", h->scale.width ); av_set_int( h->ctx, "srch", h->scale.height ); - av_set_int( h->ctx, "src_format", h->scale.pix_fmt ); - av_set_int( h->ctx, "src_range", 0 ); /* FIXME: use the correct full range value */ + av_set_int( h->ctx, "src_format", src_format ); + av_set_int( h->ctx, "src_range", src_range ); /* FIXME: use the correct full range value */ /* FIXME: use the correct full range values * FIXME: use the correct matrix coefficients (only YUV -> RGB conversions are supported) */ - sws_setColorspaceDetails( h->ctx, sws_getCoefficients( SWS_CS_DEFAULT ), 0, - sws_getCoefficients( SWS_CS_DEFAULT ), 0, 0, 1<<16, 1<<16 ); + sws_setColorspaceDetails( h->ctx, + sws_getCoefficients( SWS_CS_DEFAULT ), src_range, + sws_getCoefficients( SWS_CS_DEFAULT ), av_get_int( h->ctx, "dst_range", NULL ), + 0, 1<<16, 1<<16 ); return sws_init_context( h->ctx, NULL, NULL ) < 0; } -- 2.40.0