From: Darold Date: Wed, 10 Oct 2012 18:00:41 +0000 (+0200) Subject: Fix detection of compressed log files and allow automatic detection and uncompress... X-Git-Tag: v3.2~131 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=f58bcb44ffa5c3587dc5bcf41cffcbdb56baec63;p=pgbadger Fix detection of compressed log files and allow automatic detection and uncompress of .gz, .bz2 and .zip files. --- diff --git a/README b/README index fe5c500..517851f 100644 --- a/README +++ b/README @@ -50,7 +50,7 @@ SYNOPSIS -w | --watch-mode : only report errors just like logwatch could do. -x | --extension : output format. Values: text or html. Default: html -z | --zcat exec_path : set the full path to the zcat program. Use it if - zcat is not on your path or you want to use gzcat. + zcat or bzcat or unzip is not on your path. --pie-limit num : pie data lower than num% will show a sum instead. --exclude-query regex : any query matching the given regex will be excluded from the report. For example: "^(VACUUM|COMMIT)" @@ -112,8 +112,8 @@ DESCRIPTION features, such as zooming. pgBadger is able to autodetect your log file format (syslog, stderr or - csvlog). It is designed to parse huge log files, as well as gzip - compressed file. See a complete list of features below. + csvlog). It is designed to parse huge log files, as well as gzip, zip or + bzip2 compressed files. See a complete list of features below. FEATURE pgBadger reports everything about your SQL queries: @@ -150,15 +150,22 @@ REQUIREMENT This module is optional, if you don't have PostgreSQL log in the CSV format you don't need to install it. - Under Windows OS you may not be able to use gzipped log files unless you - have a zcat like utility that could uncompress the log file and send - content to stdout. If you have such an utility or in other OSes you want - to use other compression utility like bzip2 or Zip, use the --zcat - comand line option as follow: + Compressed log file format is autodetected following the file exention. + If pgbadger find a gz extention it will use the zcat utility, with a bz2 + extention it will use bzcat and if the file extention is zip then the + unzip utility will be used. - --zcat="unzip -p" or --zcat="gunzip -c" or --zcat="bzip2 -dc" + If those utilities are not found in the PATH environment variable then + use the --zcat command line otption to change this path. For exemple: - the last example can also be used like this: --zcat="bzcat" + --zcat="/usr/local/bin/gunzip -c" or --zcat="/usr/local/bin/bzip2 -dc" + --zcat="C:\tools\unzip -p" + + By default pgBadger will use the zcat, bzcat and unzip utilities + following the file extension. If you use the default autodetection + compress format you can mixed gz, bz2 or zip files. Specifying a custom + value to --zcat option will removed this feature of mixed compressed + format. POSTGRESQL CONFIGURATION You must enable some configuration directives in your postgresql.conf diff --git a/doc/pgBadger.pod b/doc/pgBadger.pod index 739f46a..8e02089 100644 --- a/doc/pgBadger.pod +++ b/doc/pgBadger.pod @@ -52,7 +52,7 @@ Options: -w | --watch-mode : only report errors just like logwatch could do. -x | --extension : output format. Values: text or html. Default: html -z | --zcat exec_path : set the full path to the zcat program. Use it if - zcat is not on your path or you want to use gzcat. + zcat or bzcat or unzip is not on your path. --pie-limit num : pie data lower than num% will show a sum instead. --exclude-query regex : any query matching the given regex will be excluded from the report. For example: "^(VACUUM|COMMIT)" @@ -105,7 +105,7 @@ By the way, we would like to thank Guillaume Smet for all the work he has done o pgBadger is written in pure Perl language. It uses a javascript library to draw graphs so that you don't need additional Perl modules or any other package to install. Furthermore, this library gives us additional features, such as zooming. -pgBadger is able to autodetect your log file format (syslog, stderr or csvlog). It is designed to parse huge log files, as well as gzip compressed file. See a complete list of features below. +pgBadger is able to autodetect your log file format (syslog, stderr or csvlog). It is designed to parse huge log files, as well as gzip, zip or bzip2 compressed files. See a complete list of features below. =head1 FEATURE @@ -139,14 +139,19 @@ If you planned to parse PostgreSQL CSV log files you might need some Perl Module This module is optional, if you don't have PostgreSQL log in the CSV format you don't need to install it. -Under Windows OS you may not be able to use gzipped log files unless you have a -zcat like utility that could uncompress the log file and send content to stdout. -If you have such an utility or in other OSes you want to use other compression -utility like bzip2 or Zip, use the --zcat comand line option as follow: +Compressed log file format is autodetected following the file exention. If pgbadger find a gz extention +it will use the zcat utility, with a bz2 extention it will use bzcat and if the file extention is zip +then the unzip utility will be used. - --zcat="unzip -p" or --zcat="gunzip -c" or --zcat="bzip2 -dc" +If those utilities are not found in the PATH environment variable then use the --zcat command line otption +to change this path. For exemple: -the last example can also be used like this: --zcat="bzcat" + --zcat="/usr/local/bin/gunzip -c" or --zcat="/usr/local/bin/bzip2 -dc" + --zcat="C:\tools\unzip -p" + +By default pgBadger will use the zcat, bzcat and unzip utilities following the file extension. +If you use the default autodetection compress format you can mixed gz, bz2 or zip files. Specifying +a custom value to --zcat option will removed this feature of mixed compressed format. =head1 POSTGRESQL CONFIGURATION diff --git a/pgbadger b/pgbadger index bb068d9..5a95952 100755 --- a/pgbadger +++ b/pgbadger @@ -45,6 +45,8 @@ my $JQGRAPH = 1; # Command line options my $zcat = 'zcat'; +my $bzcat = 'bzcat'; +my $ucat = 'unzip -p'; my $uncompress_size = "gunzip -l %f | grep -E '^\\s*[0-9]+' | awk '{print \$2}'"; my $format = ''; my $outfile = ''; @@ -456,10 +458,16 @@ foreach my $logfile (@log_files) { require Text::CSV; my $csv = Text::CSV->new({binary => 1, eol => $/}); my $io = undef; - if ($logfile !~ /\.gz/) { + if ($logfile !~ /\.(gz|bz2|zip)/i) { open($io, "<", $logfile) or die "FATAL: cannot read csvlog file $logfile. $!\n"; } else { - open($io, "$zcat $logfile |") or die "FATAL: cannot open pipe to $zcat $logfile. $!\n"; + my $uncompress = $zcat; + if (($logfile =~ /\.bz2/i) && ($zcat =~ /^zcat$/)) { + $uncompress = $bzcat; + } elsif (($logfile =~ /\.zip/i) && ($zcat =~ /^zcat$/)) { + $uncompress = $ucat; + } + open($io, "$uncompress $logfile |") or die "FATAL: cannot open pipe to $uncompress $logfile. $!\n"; # Real size of the file is unknow, try to find it my $cmd_file_size = $uncompress_size; $cmd_file_size =~ s/\%f/$logfile/g; @@ -539,19 +547,26 @@ foreach my $logfile (@log_files) { # Open log file for reading my $lfile = new IO::File; - if ($logfile !~ /\.gz/) { + if ($logfile !~ /\.(gz|bz2|zip)/i) { $lfile->open($logfile) || die "FATAL: cannot read log file $logfile. $!\n"; } else { + my $uncompress = $zcat; + if (($logfile =~ /\.bz2/i) && ($zcat =~ /^zcat$/)) { + $uncompress = $bzcat; + } elsif (($logfile =~ /\.zip/i) && ($zcat =~ /^zcat$/)) { + $uncompress = $ucat; + } + + # Open a pipe to zcat program for compressed log + $lfile->open("$uncompress $logfile |") || die "FATAL: cannot read from pipe to $uncompress $logfile. $!\n"; + # Real size of the file is unknow, try to find it my $cmd_file_size = $uncompress_size; $cmd_file_size =~ s/\%f/$logfile/g; $totalsize = `$cmd_file_size`; chomp($totalsize); $totalsize ||= 0; - - # Open a pipe to zcat program for compressed log - $lfile->open("$zcat $logfile |") || die "FATAL: cannot read from pipe to $zcat $logfile. $!\n"; } my $time_pattern = qr/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})/; @@ -828,7 +843,7 @@ Options: -w | --watch-mode : only report events/errors just like logwatch could do. -x | --extension : output format. Values: text or html. Default: html -z | --zcat exec_path : set the full path to the zcat program. Use it if - zcat is not on your path or you want to use gzcat. + zcat or bzcat or unzip is not on your path. --pie-limit num : pie data lower than num% will show a sum instead. --exclude-query regex : any query matching the given regex will be excluded from the report. For example: "^(VACUUM|COMMIT)" @@ -3455,8 +3470,6 @@ sub parse_query $cur_info{$t_pid}{ident} = $prefix_vars{'t_ident'}; $cur_info{$t_pid}{query} = $prefix_vars{'t_query'}; $cur_info{$t_pid}{duration} = $t_duration; -print SDTERR "3449: $prefix_vars{'t_duration'} = $t_duration\n"; - $cur_info{$t_pid}{pid} = $prefix_vars{'t_pid'}; $cur_info{$t_pid}{session} = $prefix_vars{'t_session_line'}; $cur_info{$t_pid}{loglevel} = $prefix_vars{'t_loglevel'}; @@ -3665,12 +3678,18 @@ sub autodetect_format my $nline = 0; my $fmt = ''; my $tfile = new IO::File; - if ($file !~ /\.gz/) { + if ($file !~ /\.(gz|bz2|zip)/i) { $tfile->open($file) || die "FATAL: cannot read logfile $file. $!\n"; } else { + my $uncompress = $zcat; + if (($file =~ /\.bz2/i) && ($zcat =~ /^zcat$/)) { + $uncompress = $bzcat; + } elsif (($file =~ /\.zip/i) && ($zcat =~ /^zcat$/)) { + $uncompress = $ucat; + } # Open a pipe to zcat program for compressed log - $tfile->open("$zcat $file |") || die "FATAL: cannot read from pipe to $zcat $file. $!\n"; + $tfile->open("$uncompress $file |") || die "FATAL: cannot read from pipe to $uncompress $file. $!\n"; } my $duration = 'duration:'; if ($error_only || ($disable_hourly && $disable_query)) {