From: Gilles Darold Date: Thu, 30 Aug 2018 09:11:13 +0000 (+0200) Subject: List of changes in this commit: X-Git-Tag: v10.0~13 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=20712f8339f90a51d87b1dbc2507b664a20a2818;p=pgbadger List of changes in this commit: - Fix parsing of vacuum / analyze system usage for PostgreSQL 10. Thanks to Achilleas Mantzios for the patch. - Fix Temporary File Activity table. - Remove dependency to git during install. - Add support to auto_explain json output format. Thanks to dmius for the report. - Fix auto_explain parser and queries that was counted twice. Thanks to zam6ak for the report. - Add support to %q placeholder in log_line_prefix. - Fix checkpoint regex to match PostgreSQL 10 log messages. Thanks to Edmund Horner for the patch. - Update description of -f | --format option by adding information about jsonlog format. - Fix wrong long name for option -J that should be --Jobs intead of --job_per_file. Thanks to Chad Trabant for the report. - Add jsonlog input format. Some users are using the jsonlog format of Michael Paquier extension, with -f jsonlog pgbadger will be able to parse the log. - Fix query normalisation to not duplicate with bind queries. Normalisation of values are now tranformed into a single ? and no more 0 for numbers, two single quote for string. Thanks to vadv for the report. - Fix log level count. Thanks to Jean-Christophe Arnu for the report. - Make pgbadger more compliant with B::Lint bare sub name. - Made perlcritic happy. - Add --prettify-json command line option to prettify JSON output. Default output is all in single line. - Fix Events distribution report. - Fix bug with --prefix when log_line_prefix contain multiple %%. Thanks to svb007 for the report. - Add --log-timezone +/-XX command line option to set the number of hours from GMT of the timezone that must be used to adjust date/time read from log file before beeing parsed. Using this option make more difficult log search with a date/time because the time will not be the same in the log. Note that you might still need to adjust the graph timezone using -Z when the client has not the same timezone. Thanks to xdexter for the feature request. - Apply timezone to bar chart in pgBouncer reports. - Apply timezone to bar chart in Top queries reports. - Apply timezone to bar chart in Most frequent errors/events report. - Remove INDEXES from the keyword list and add BUFFERS to this list. - Fix normalization of query using cursors. --- diff --git a/pgbadger b/pgbadger index 06f8914..7b59970 100755 --- a/pgbadger +++ b/pgbadger @@ -11,14 +11,14 @@ # # You should enable SQL query logging with log_min_duration_statement >= 0 # With stderr output -# Log line prefix should be: log_line_prefix = '%t [%p]: [%l-1] ' -# Log line prefix should be: log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d ' -# Log line prefix should be: log_line_prefix = '%t [%p]: [%l-1] db=%d,user=%u ' +# Log line prefix should be: log_line_prefix = '%t [%p]: ' +# Log line prefix should be: log_line_prefix = '%t [%p]: user=%u,db=%d ' +# Log line prefix should be: log_line_prefix = '%t [%p]: db=%d,user=%u ' # If you need report per client Ip adresses you can add client=%h or remote=%h # pgbadger will also recognized the following form: -# log_line_prefix = '%t [%p]: [%l-1] db=%d,user=%u,client=%h ' +# log_line_prefix = '%t [%p]: db=%d,user=%u,client=%h ' # or -# log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,remote=%h ' +# log_line_prefix = '%t [%p]: user=%u,db=%d,remote=%h ' # With syslog output # Log line prefix should be: log_line_prefix = 'db=%d,user=%u ' # @@ -39,7 +39,7 @@ use IO::File; use Benchmark; use File::Basename; use Storable qw(store_fd fd_retrieve); -use Time::Local 'timegm_nocheck'; +use Time::Local qw(timegm_nocheck timelocal_nocheck); use POSIX qw(locale_h sys_wait_h _exit strftime); setlocale(LC_NUMERIC, ''); setlocale(LC_ALL, 'C'); @@ -297,7 +297,7 @@ my $last_parsed = ''; my $report_title = ''; my $log_line_prefix = ''; my $compiled_prefix = ''; -my $project_url = 'http://darold.github.com/pgbadger/'; +my $project_url = 'http://pgbadger.darold.net/'; my $t_min = 0; my $t_max = 0; my $remove_comment = 0; @@ -321,12 +321,13 @@ my $log_duration = 0; my $logfile_list = ''; my $enable_checksum = 0; my $timezone = 0; -my $log_timezone = 0; my $pgbouncer_only = 0; my $rebuild = 0; my $week_start_monday = 0; my $use_sessionid_as_pid = 0; my $dump_normalized_only = 0; +my $log_timezone = 0; +my $json_prettify = 0; my $NUMPROGRESS = 10000; my @DIMENSIONS = (800, 300); @@ -334,6 +335,8 @@ my $RESRC_URL = ''; my $img_format = 'png'; my @log_files = (); my %prefix_vars = (); +my $q_prefix = ''; +my @prefix_q_params = (); my $remote_host = ''; my $ssh_command = ''; @@ -487,7 +490,8 @@ my $result = GetOptions( 'pgbouncer-only!' => \$pgbouncer_only, 'start-monday!' => \$week_start_monday, 'normalized-only!' => \$dump_normalized_only, - "log-timezone=s" => \$log_timezone, + 'log-timezone=i' => \$log_timezone, + 'prettify-json!' => \$json_prettify, ); die "FATAL: use pgbadger --help\n" if (not $result); @@ -527,12 +531,13 @@ if (-e "$PID_FILE") { } # Create pid file -unless(open(OUT, ">$PID_FILE")) { +if (open(my $out, '>', $PID_FILE)) { + print $out $$; + close($out); +} else { print "FATAL: can't create pid file $PID_FILE, $!\n"; exit 3; } -print OUT $$; -close(OUT); # Rewrite some command line arguments as lists &compute_arg_list(); @@ -602,11 +607,12 @@ if ($logfile_list) { if (!-e $logfile_list) { localdie("FATAL: logfile list $logfile_list must exist!\n"); } - if (not open(IN, $logfile_list)) { + my $in = undef; + if (not open($in, "<", $logfile_list)) { localdie("FATAL: can not read logfile list $logfile_list, $!.\n"); } - my @files = ; - close(IN); + my @files = <$in>; + close($in); foreach my $file (@files) { chomp($file); $file =~ s/\r//; @@ -725,10 +731,8 @@ $top ||= 20; # Set timezone $timezone = ((0-$timezone)*3600); -# Set timezone for logs $log_timezone = ((0-$log_timezone)*3600); - # Set the default extension and output format if (!$extension) { if ($outfile =~ /\.bin/i) { @@ -869,7 +873,7 @@ if ($error_only && $disable_error) { my $regex_prefix_dbname = qr/(?:db|database)=([^,]*)/; my $regex_prefix_dbuser = qr/(?:user|usr)=([^,]*)/; my $regex_prefix_dbclient = qr/(?:client|remote|ip|host)=([^,\(]*)/; -my $regex_prefix_dbappname = qr/(?:app|application|appname)=([^,]*)/; +my $regex_prefix_dbappname = qr/(?:app|application)=([^,]*)/; # Set pattern to look for query type my $action_regex = qr/^[\s\(]*(DELETE|INSERT|UPDATE|SELECT|COPY|WITH|CREATE|DROP|ALTER|TRUNCATE|BEGIN|COMMIT|ROLLBACK|START|END|SAVEPOINT)/is; @@ -877,12 +881,12 @@ my $action_regex = qr/^[\s\(]*(DELETE|INSERT|UPDATE|SELECT|COPY|WITH|CREATE|DROP # Loading excluded query from file if any if ($exclude_file) { - open(IN, "$exclude_file") or localdie("FATAL: can't read file $exclude_file: $!\n"); - my @exclq = ; - close(IN); + open(my $in, '<', $exclude_file) or localdie("FATAL: can't read file $exclude_file: $!\n"); + my @exclq = <$in>; + close($in); chomp(@exclq); - map {s/\r//;} @exclq; foreach my $r (@exclq) { + $r =~ s/\r//; &check_regex($r, '--exclude-file'); } push(@exclude_query, @exclq); @@ -911,12 +915,12 @@ if ($#include_time >= 0) { # Loading included query from file if any if ($include_file) { - open(IN, "$include_file") or localdie("FATAL: can't read file $include_file: $!\n"); - my @exclq = ; - close(IN); + open(my $in, '<', $include_file) or localdie("FATAL: can't read file $include_file: $!\n"); + my @exclq = <$in>; + close($in); chomp(@exclq); - map {s/\r//;} @exclq; foreach my $r (@exclq) { + $r =~ s/\r//; &check_regex($r, '--include-file'); } push(@include_query, @exclq); @@ -970,8 +974,8 @@ my %abbr_month = ( # Keywords variable my @pg_keywords = qw( - ALL ANALYSE ANALYZE AND ANY ARRAY AS ASC ASYMMETRIC AUTHORIZATION BERNOULLI BINARY BOTH CASE - CAST CHECK COLLATE COLLATION COLUMN CONCURRENTLY CONSTRAINT CREATE CROSS CUBE + ALL ANALYSE ANALYZE AND ANY ARRAY AS ASC ASYMMETRIC AUTHORIZATION BERNOULLI BINARY BOTH BUFFERS + CASE CAST CHECK COLLATE COLLATION COLUMN CONCURRENTLY CONSTRAINT CREATE CROSS CUBE CURRENT_DATE CURRENT_ROLE CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER DEFAULT DEFERRABLE DESC DISTINCT DO ELSE END EXCEPT FALSE FETCH FOR FOREIGN FREEZE FROM FULL GRANT GROUP GROUPING HAVING ILIKE IN INITIALLY INNER INTERSECT INTO IS ISNULL JOIN LEADING @@ -1011,7 +1015,7 @@ my @KEYWORDS1 = qw( DEFERRED DEFINER DELIMITER DELIMITERS DICTIONARY DISABLE DISCARD DOCUMENT DOMAIN DOUBLE EACH ENABLE ENCODING ENCRYPTED ENUM ESCAPE EXCLUDE EXCLUDING EXCLUSIVE EXECUTE EXTENSION EXTERNAL FIRST FLOAT FOLLOWING FORCE FORWARD FUNCTIONS GLOBAL GRANTED HANDLER HEADER HOLD - HOUR IDENTITY IMMEDIATE IMMUTABLE IMPLICIT INCLUDING INCREMENT INDEXES INHERITS INLINE INOUT INPUT + HOUR IDENTITY IMMEDIATE IMMUTABLE IMPLICIT INCLUDING INCREMENT INHERITS INLINE INOUT INPUT INSENSITIVE INSTEAD INT INTEGER INVOKER ISOLATION LABEL LARGE LAST LC_COLLATE LC_CTYPE LEAKPROOF LEVEL LISTEN LOCATION LOOP MAPPING MATCH MAXVALUE MINUTE MINVALUE MODE MONTH MOVE NAMES NATIONAL NCHAR NEXT NO NONE NOTHING NOTIFY NOWAIT NULLS OBJECT OF OFF OIDS OPERATOR OPTIONS @@ -1060,7 +1064,6 @@ my %SYMBOLS = ( '\/' => '/', '!=' => '!=' ); my @BRACKETS = ('(', ')'); -map {$_ = quotemeta($_)} @BRACKETS; # Inbounds of query times histogram my @histogram_query_time = (0, 1, 5, 10, 25, 50, 100, 500, 1000, 10000); @@ -1201,9 +1204,9 @@ if ($incremental) { # Reading last line parsed if ($last_parsed && -e $last_parsed) { - if (open(IN, "$last_parsed")) { - my @content = ; - close(IN); + if (open(my $in, '<', $last_parsed)) { + my @content = <$in>; + close($in); foreach my $line (@content) { chomp($line); next if (!$line); @@ -1614,8 +1617,8 @@ if ( ($#given_log_files >= 0) && (($queue_size > 1) || ($job_per_file > 1)) ) { # Get last line parsed from all process if ($last_parsed) { - if (open(IN, "$tmp_last_parsed") ) { - while (my $line = ) { + if (open(my $in, '<', $tmp_last_parsed) ) { + while (my $line = <$in>) { chomp($line); $line =~ s/\r//; my ($d, $p, $l, @o) = split(/\t/, $line); @@ -1636,29 +1639,29 @@ if ($last_parsed) { } } } - close(IN); + close($in); } unlink("$tmp_last_parsed"); } # Save last line parsed if ($last_parsed && ($last_line{datetime} || $pgb_last_line{datetime}) && ($last_line{orig} || $pgb_last_line{orig}) ) { - if (open(OUT, ">$last_parsed")) { + if (open(my $out, '>', $last_parsed)) { if ($last_line{datetime}) { $last_line{current_pos} ||= 0; - print OUT "$last_line{datetime}\t$last_line{current_pos}\t$last_line{orig}\n"; + print $out "$last_line{datetime}\t$last_line{current_pos}\t$last_line{orig}\n"; } elsif ($saved_last_line{datetime}) { $saved_last_line{current_pos} ||= 0; - print OUT "$saved_last_line{datetime}\t$saved_last_line{current_pos}\t$saved_last_line{orig}\n"; + print $out "$saved_last_line{datetime}\t$saved_last_line{current_pos}\t$saved_last_line{orig}\n"; } if ($pgb_last_line{datetime}) { $pgb_last_line{current_pos} ||= 0; - print OUT "pgbouncer\t$pgb_last_line{datetime}\t$pgb_last_line{current_pos}\t$pgb_last_line{orig}\n"; + print $out "pgbouncer\t$pgb_last_line{datetime}\t$pgb_last_line{current_pos}\t$pgb_last_line{orig}\n"; } elsif ($pgb_saved_last_line{datetime}) { $pgb_saved_last_line{current_pos} ||= 0; - print OUT "pgbouncer\t$pgb_saved_last_line{datetime}\t$pgb_saved_last_line{current_pos}\t$pgb_saved_last_line{orig}\n"; + print $out "pgbouncer\t$pgb_saved_last_line{datetime}\t$pgb_saved_last_line{current_pos}\t$pgb_saved_last_line{orig}\n"; } - close(OUT); + close($out); } else { &logmsg('ERROR', "can't save last parsed line into $last_parsed, $!"); } @@ -1677,6 +1680,10 @@ if (!$incremental && ($#given_log_files >= 0) ) { &logmsg('LOG', "Ok, generating $extension report..."); + # Some message have been temporary stored as ERROR but + # they are LOGestore them to the right log level. + &restore_log_type_count(); + if ($extension ne 'tsung') { $fh = new IO::File ">$outfile"; if (not defined $fh) { @@ -1720,13 +1727,13 @@ if (!$incremental && ($#given_log_files >= 0) ) { # Look for directory where report must be generated my @build_directories = (); if (-e "$last_parsed.tmp") { - if (open(IN, "$last_parsed.tmp")) { - while (my $l = ) { + if (open(my $in, '<', "$last_parsed.tmp")) { + while (my $l = <$in>) { chomp($l); $l =~ s/\r//; push(@build_directories, $l) if (!grep(/^$l$/, @build_directories)); } - close(IN); + close($in); unlink("$last_parsed.tmp"); } else { &logmsg('ERROR', "can't read file $last_parsed.tmp, $!"); @@ -1779,9 +1786,9 @@ Options: -D | --dns-resolv : client ip addresses are replaced by their DNS name. Be warned that this can really slow down pgBadger. -e | --end datetime : end date/time for the data to be parsed in log. - -f | --format logtype : possible values: syslog, syslog2, stderr, csv and - pgbouncer. Use this option when pgBadger is not - able to auto-detect the log format. + -f | --format logtype : possible values: syslog, syslog2, stderr, jsonlog, + cvs and pgbouncer. Use this option when pgBadger is + not able to auto-detect the log format. -G | --nograph : disable graphs on HTML output. Enabled by default. -h | --help : show this message and exit. -i | --ident name : programname used as syslog ident. Default: postgres @@ -1891,16 +1898,22 @@ Options: --journalctl command : command to use to replace PostgreSQL logfile by a call to journalctl. Basically it might be: journalctl -u postgresql-9.5 - --pid-file PATH : set the path of the pid file to manage - concurrent execution of pgBadger. + --pid-dir path : set the path where the pid file must be stored. + Default /tmp + --pid-file file : set the name of the pid file to manage concurrent + execution of pgBadger. Default: pgbadger.pid --rebuild : used to rebuild all html reports in incremental output directories where there is binary data files. --pgbouncer-only : only show PgBouncer related menu in the header. --start-monday : in incremental mode, calendar's weeks start on sunday. Use this option to start on monday. --normalized-only : only dump all normalized query to out.txt - --log-timezone +/-XX : Set the number of hours from GMT of the timezone - when parsing logs. + --log-timezone +/-XX : Set the number of hours from GMT of the timezone + that must be used to adjust date/time read from + log file before beeing parsed. Using this option + make more difficult log search with a date/time. + --prettify-json : use it if you want json output to be prettified. + pgBadger is able to parse a remote log file using a passwordless ssh connection. Use the -r or --remote-host to set the host ip address or hostname. There's also @@ -1930,7 +1943,7 @@ Examples: /pglog/postgresql-2012-08-21* perl pgbadger --prefix '%m %u@%d %p %r %a : ' /pglog/postgresql.log # Log line prefix with syslog log output - perl pgbadger --prefix 'user=%u,db=%d,client=%h,app=%a' \ + perl pgbadger --prefix 'user=%u,db=%d,client=%h,appname=%a' \ /pglog/postgresql-2012-08-21* # Use my 8 CPUs to parse my 10GB file faster, much faster perl pgbadger -j 8 /pglog/postgresql-9.1-main.log @@ -2015,6 +2028,7 @@ sub set_parser_regex my $fmt = shift; @prefix_params = (); + @prefix_q_params = (); if ($fmt eq 'pgbouncer') { @@ -2029,8 +2043,11 @@ sub set_parser_regex } elsif ($log_line_prefix) { # Build parameters name that will be extracted from the prefix regexp - my $llp = ''; - ($llp, @prefix_params) = &build_log_line_prefix_regex($log_line_prefix); + my %res = &build_log_line_prefix_regex($log_line_prefix); + my $llp = $res{'llp'}; + @prefix_params = @{ $res{'param_list'} }; + $q_prefix = $res{'q_prefix'}; + @prefix_q_params = @{ $res{'q_param_list'} }; if ($fmt eq 'syslog') { $llp = @@ -2055,7 +2072,6 @@ sub set_parser_regex $llp = '^' . $llp . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT|LOCATION):\s+(?:[0-9A-Z]{5}:\s+)?(.*)'; $compiled_prefix = qr/$llp/; push(@prefix_params, 't_loglevel', 't_query'); - } } elsif ($fmt eq 'syslog') { @@ -2078,14 +2094,14 @@ sub set_parser_regex } elsif ($fmt eq 'stderr') { $compiled_prefix = - qr/^(\d{10}\.\d{3}|\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})[\.\d]*(?: [A-Z\+\-\d]{3,6})?\s\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT|LOCATION):\s+(?:[0-9A-Z]{5}:\s+)?(.*)/; + qr/^(\d{10}\.\d{3}|\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})[\.\d]*(?: [A-Z\+\-\d]{3,6})?\s\[([0-9a-f\.]+)\]:\s\[(\d+)\-\d+\]\s*(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT|LOCATION):\s+(?:[0-9A-Z]{5}:\s+)?(.*)/; push(@prefix_params, 't_timestamp', 't_pid', 't_session_line', 't_logprefix', 't_loglevel', 't_query'); } elsif ($fmt eq 'default') { $fmt = 'stderr'; $compiled_prefix = - qr/^(\d{10}\.\d{3}|\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})[\.\d]*(?: [A-Z\+\-\d]{3,6})?\s\[(\d+)\]\s(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT|LOCATION):\s+(?:[0-9A-Z]{5}:\s+)?(.*)/; + qr/^(\d{10}\.\d{3}|\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2})[\.\d]*(?: [A-Z\+\-\d]{3,6})?\s\[([0-9a-f\.]+)\][:]*\s(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT|LOCATION):\s+(?:[0-9A-Z]{5}:\s+)?(.*)/; push(@prefix_params, 't_timestamp', 't_pid', 't_logprefix', 't_loglevel', 't_query'); } @@ -2108,16 +2124,16 @@ sub build_incremental_reports my @build_directories = @_; my %weeks_directories = (); - foreach $incr_date (sort @build_directories) { + foreach my $bpath (sort @build_directories) { - $last_incr_date = $incr_date; + $incr_date = $bpath; + $last_incr_date = $bpath; # Set the path to binary files - my $bpath = $incr_date; $bpath =~ s/\-/\//g; - $incr_date =~ /^(\d+)-(\d+)\-(\d+)$/; # Get the week number following the date + $incr_date =~ /^(\d+)-(\d+)\-(\d+)$/; my $wn = &get_week_number($1, $2, $3); $weeks_directories{$wn} = "$1-$2" if ($rebuild || !exists $weeks_directories{$wn}); @@ -2160,8 +2176,8 @@ sub build_incremental_reports my $wdir = ''; # Load data per day - foreach $incr_date (@wdays) { - my $bpath = $incr_date; + foreach my $bpath (@wdays) { + $incr_date = $bpath; $bpath =~ s/\-/\//g; $incr_date =~ /^(\d+)\-(\d+)\-(\d+)$/; $wdir = "$1/week-$wn"; @@ -2207,7 +2223,9 @@ sub build_incremental_reports } my $date = localtime(time); my @tmpjscode = @jscode; - map { s/EDIT_URI/\./; } @tmpjscode; + for (my $i = 0; $i <= $#tmpjscode; $i++) { + $tmpjscode[$i] =~ s/EDIT_URI/\./; + } my $local_title = 'Global Index on incremental reports'; if ($report_title) { $local_title = 'Global Index - ' . $report_title; @@ -2485,66 +2503,6 @@ sub update_progress_bar } } -sub apply_tz_offset -{ - # Apply timezone offset to datetime string - # Parsing regex is set using $pattern - my ($datetime, $offset, $pattern, $format) = @_; - my ($y, $m, $d, $h, $mi, $s) = (0, 0, 0, 0, 0, 0, 0); - - $format = "%Y-%m-%d %H:%M:%S" unless defined $format; - $pattern = qr/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})/ unless defined $pattern; - # $datetime parsing - ($y, $m, $d, $h, $mi, $s) = ($datetime =~ $pattern); - - if ($offset == 0) - { - # If no tz offset, return input and parsed datetime - return ($datetime, $y, $m, $d, $h, $mi, $s); - } - my $t = timegm_nocheck($s, $mi, $h, $d, $m - 1, $y); - # Apply offset - $t += ($offset); - my @gmtime = CORE::gmtime($t); - ($y, $m, $d, $h, $mi, $s) = split(/:/, strftime("%Y:%m:%d:%H:%M:%S", @gmtime)); - return (strftime($format, @gmtime), $y, $m, $d, $h, $mi, $s); -} - -sub apply_tz_offset_chronos -{ - # Apply timezone offset to chronos structure and returns a copy - my ($chronosref, $timezone) = @_; - - my %new_chronos = (); - my %chronos = %{$chronosref}; - if ($timezone == 0) - { - # If no timezone offset, just return original chronos - return %chronos; - } - my ($nc_t, $nc_y, $nc_m, $nc_d, $nc_h, $nc_mi, $nc_s); - foreach my $d (sort keys %chronos) { - foreach my $h (sort keys %{ $chronos{$d} }) { - # Apply timezone offset to $d $h:00:00 - # not going to the minute - ( - $nc_t, - $nc_y, - $nc_m, - $nc_d, - $nc_h, - $nc_mi, - $nc_s - ) = apply_tz_offset("$d $h:00:00", $timezone, qr/(\d{4})(\d{2})(\d{2}) (\d{2}):(\d{2}):(\d{2})/); - my $nc_date = "$nc_y$nc_m$nc_d"; - # Copy original chronos subset into new chronos at the right time (after TZ - # offset application). - $new_chronos{$nc_date}{$nc_h} = $chronos{$d}{$h}; - } - } - return %new_chronos -} - #### # Main function called per each parser process @@ -2557,7 +2515,6 @@ sub process_file my $old_errors_count = 0; my $getout = 0; $start_offset ||= 0; - my $time_pattern = qr/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})/; $0 = 'pgbadger parser'; @@ -2627,6 +2584,7 @@ sub process_file # Parse pgbouncer logfile if ($fmt eq 'pgbouncer') { + my $time_pattern = qr/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})/; my $cur_pid = ''; my @matches = (); my $has_exclusion = 0; @@ -2671,16 +2629,9 @@ sub process_file $prefix_vars{$pgb_prefix_parse1[$i]} = $matches[$i]; } - # Apply log timezone offset - ( - $prefix_vars{'t_timestamp'}, - $prefix_vars{'t_year'}, - $prefix_vars{'t_month'}, - $prefix_vars{'t_day'}, - $prefix_vars{'t_hour'}, - $prefix_vars{'t_min'}, - $prefix_vars{'t_sec'} - ) = apply_tz_offset($prefix_vars{'t_timestamp'}, $log_timezone, $time_pattern); + # Get time detailed information + ($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, + $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}) = ($prefix_vars{'t_timestamp'} =~ $time_pattern); # Skip unwanted lines my $res = &skip_unwanted_line(); @@ -2694,6 +2645,15 @@ sub process_file # Jump to the last line parsed if required next if (!&check_incremental_position($fmt, $prefix_vars{'t_timestamp'}, $line)); + # Store the current timestamp of the log line + &store_current_timestamp($prefix_vars{'t_timestamp'}); + + # Override timestamp when we have to adjust datetime to the log timezone + if ($log_timezone) { + ($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}) = change_timezone($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}); + $prefix_vars{'t_timestamp'} = "$prefix_vars{'t_year'}-$prefix_vars{'t_month'}-$prefix_vars{'t_day'} $prefix_vars{'t_hour'}:$prefix_vars{'t_min'}:$prefix_vars{'t_sec'}"; + } + # Extract other information from the line @matches = ($line =~ $pgbouncer_log_parse2); if ($#matches >= 0) { @@ -2712,9 +2672,6 @@ sub process_file } } - # Store the current timestamp of the log line - &store_current_timestamp($prefix_vars{'t_timestamp'}); - # Check if the log line should be excluded from the report if (&validate_log_line($prefix_vars{'t_pid'})) { $prefix_vars{'t_host'} = 'stderr'; # this unused variable is used to store format information when log format is not syslog @@ -2769,21 +2726,19 @@ sub process_file # Extract the date if ($row->[0] =~ m/^(\d+)-(\d+)-(\d+)\s+(\d+):(\d+):(\d+)\.(\d+)/) { - # Remove newline characters from queries - map { s/[\r\n]+/ /gs; } @$row; - + $prefix_vars{'t_year'} = $1; + $prefix_vars{'t_month'} = $2; + $prefix_vars{'t_day'} = $3; + $prefix_vars{'t_hour'} = $4; + $prefix_vars{'t_min'} = $5; + $prefix_vars{'t_sec'} = $6; my $milli = $7 || 0; + $prefix_vars{'t_timestamp'} = "$prefix_vars{'t_year'}-$prefix_vars{'t_month'}-$prefix_vars{'t_day'} $prefix_vars{'t_hour'}:$prefix_vars{'t_min'}:$prefix_vars{'t_sec'}"; - # Apply log timezone offset - ( - $prefix_vars{'t_timestamp'}, - $prefix_vars{'t_year'}, - $prefix_vars{'t_month'}, - $prefix_vars{'t_day'}, - $prefix_vars{'t_hour'}, - $prefix_vars{'t_min'}, - $prefix_vars{'t_sec'} - ) = apply_tz_offset("$1-$2-$3 $4:$5:$6", $log_timezone, $time_pattern); + # Remove newline characters from queries + for (my $i = 0; $i <= $#$row; $i++) { + $row->[$i] =~ s/[\r\n]+/ /gs; + } # Skip unwanted lines my $res = &skip_unwanted_line(); @@ -2800,6 +2755,12 @@ sub process_file # Store the current timestamp of the log line &store_current_timestamp($prefix_vars{'t_timestamp'}); + # Update current timestamp with the timezone wanted + if ($log_timezone) { + ($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}) = change_timezone($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}); + $prefix_vars{'t_timestamp'} = "$prefix_vars{'t_year'}-$prefix_vars{'t_month'}-$prefix_vars{'t_day'} $prefix_vars{'t_hour'}:$prefix_vars{'t_min'}:$prefix_vars{'t_sec'}"; + } + # Set query parameters as global variables $prefix_vars{'t_dbuser'} = $row->[1] || ''; $prefix_vars{'t_dbname'} = $row->[2] || ''; @@ -2840,11 +2801,14 @@ sub process_file } } + elsif ($fmt eq 'binary') { + &load_stats($lfile); } else { # Format is not CSV. + my $time_pattern = qr/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})/; my $cur_pid = ''; my @matches = (); my $goon = 0; @@ -2884,15 +2848,59 @@ sub process_file %prefix_vars = (); + # Parse jsonlog lines + if ($fmt =~ /jsonlog/) { + + %prefix_vars = parse_jsonlog_input($line); + + # Skip unwanted lines + my $res = &skip_unwanted_line(); + next if ($res == 1); + + # Jump to the last line parsed if required + next if (!&check_incremental_position($fmt, $prefix_vars{'t_timestamp'}, $line)); + + # Store the current timestamp of the log line + &store_current_timestamp($prefix_vars{'t_timestamp'}); + + # Update current timestamp with the timezone wanted + if ($log_timezone) { + ($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}) = change_timezone($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}); + $prefix_vars{'t_timestamp'} = "$prefix_vars{'t_year'}-$prefix_vars{'t_month'}-$prefix_vars{'t_day'} $prefix_vars{'t_hour'}:$prefix_vars{'t_min'}:$prefix_vars{'t_sec'}"; + } + + # Check if the log line should be excluded from the report + if (&validate_log_line($prefix_vars{'t_pid'})) { + + # Parse the query now + &parse_query($fmt); + + # The information can be saved immediately with csvlog + &store_queries($prefix_vars{'t_pid'}); + delete $cur_info{$prefix_vars{'t_pid'}}; + } + # Parse syslog lines - if ($fmt =~ /syslog/) { + } elsif ($fmt =~ /syslog/) { @matches = ($line =~ $compiled_prefix); + my $q_match = 0; + if ($#matches < 0 && $q_prefix) { + @matches = ($line =~ $q_prefix); + $q_match = 1; + } + if ($#matches >= 0) { - for (my $i = 0 ; $i <= $#prefix_params ; $i++) { - $prefix_vars{$prefix_params[$i]} = $matches[$i]; + if (!$q_match) { + for (my $i = 0 ; $i <= $#prefix_params ; $i++) { + $prefix_vars{$prefix_params[$i]} = $matches[$i]; + } + } else { + for (my $i = 0 ; $i <= $#prefix_q_params ; $i++) { + $prefix_vars{$prefix_q_params[$i]} = $matches[$i]; + } } # skip non postgresql lines @@ -2914,18 +2922,6 @@ sub process_file } $prefix_vars{'t_timestamp'} = "$prefix_vars{'t_year'}-$prefix_vars{'t_month'}-$prefix_vars{'t_day'} $prefix_vars{'t_hour'}:$prefix_vars{'t_min'}:$prefix_vars{'t_sec'}"; - - # Apply log timezone offset - ( - $prefix_vars{'t_timestamp'}, - $prefix_vars{'t_year'}, - $prefix_vars{'t_month'}, - $prefix_vars{'t_day'}, - $prefix_vars{'t_hour'}, - $prefix_vars{'t_min'}, - $prefix_vars{'t_sec'} - ) = apply_tz_offset($prefix_vars{'t_timestamp'}, $log_timezone, $time_pattern); - if ($prefix_vars{'t_hostport'} && !$prefix_vars{'t_client'}) { $prefix_vars{'t_client'} = $prefix_vars{'t_hostport'}; # Remove the port part @@ -2949,6 +2945,12 @@ sub process_file # Store the current timestamp of the log line &store_current_timestamp($prefix_vars{'t_timestamp'}); + # Update current timestamp with the timezone wanted + if ($log_timezone) { + ($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}) = change_timezone($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}); + $prefix_vars{'t_timestamp'} = "$prefix_vars{'t_year'}-$prefix_vars{'t_month'}-$prefix_vars{'t_day'} $prefix_vars{'t_hour'}:$prefix_vars{'t_min'}:$prefix_vars{'t_sec'}"; + } + # Extract information from log line prefix if (!$log_line_prefix) { &parse_log_prefix($prefix_vars{'t_logprefix'}); @@ -2997,27 +2999,23 @@ sub process_file } elsif ($fmt eq 'stderr') { @matches = ($line =~ $compiled_prefix); + + my $q_match = 0; + if ($#matches < 0 && $q_prefix) { + @matches = ($line =~ $q_prefix); + $q_match = 1; + } + if ($#matches >= 0) { - # Store auto explain plan when switching to an other log entry - foreach my $p (keys %cur_plan_info) { - if (exists $cur_plan_info{$p}{plan}) { - # Extract the query part from the plan - my $key = 'query'; - my @plan = split("\n", $cur_plan_info{$p}{plan}); - foreach my $l (@plan) { - $key = 'plan' if ($l =~ /\(cost=\d+.*rows=\d+/); - $cur_info{$p}{$key} .= "$l\n"; - } - $cur_info{$p}{query} =~ s/^\s*Query Text:\s+//s; - delete $cur_plan_info{$p}; - &store_queries($p); - delete $cur_info{$p}; + if (!$q_match) { + for (my $i = 0 ; $i <= $#prefix_params ; $i++) { + $prefix_vars{$prefix_params[$i]} = $matches[$i]; + } + } else { + for (my $i = 0 ; $i <= $#prefix_q_params ; $i++) { + $prefix_vars{$prefix_q_params[$i]} = $matches[$i]; } - } - - for (my $i = 0 ; $i <= $#prefix_params ; $i++) { - $prefix_vars{$prefix_params[$i]} = $matches[$i]; } $prefix_vars{'t_pid'} = $prefix_vars{'t_session_id'} if ($use_sessionid_as_pid); @@ -3037,19 +3035,10 @@ sub process_file my $ms = $1; $prefix_vars{'t_epoch'} = $prefix_vars{'t_timestamp'}; $prefix_vars{'t_timestamp'} = strftime("%Y-%m-%d %H:%M:%S", CORE::localtime($prefix_vars{'t_timestamp'})); - $prefix_vars{'t_timestamp'} .= $ms; + $prefix_vars{'t_timestamp'} .= $ms; } - - # Apply log timezone offset - ( - $prefix_vars{'t_timestamp'}, - $prefix_vars{'t_year'}, - $prefix_vars{'t_month'}, - $prefix_vars{'t_day'}, - $prefix_vars{'t_hour'}, - $prefix_vars{'t_min'}, - $prefix_vars{'t_sec'} - ) = apply_tz_offset($prefix_vars{'t_timestamp'}, $log_timezone, $time_pattern); + ($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, + $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}) = ($prefix_vars{'t_timestamp'} =~ $time_pattern); if ($prefix_vars{'t_hostport'} && !$prefix_vars{'t_client'}) { $prefix_vars{'t_client'} = $prefix_vars{'t_hostport'}; @@ -3073,6 +3062,12 @@ sub process_file # Store the current timestamp of the log line &store_current_timestamp($prefix_vars{'t_timestamp'}); + # Update current timestamp with the timezone wanted + if ($log_timezone) { + ($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}) = change_timezone($prefix_vars{'t_year'}, $prefix_vars{'t_month'}, $prefix_vars{'t_day'}, $prefix_vars{'t_hour'}, $prefix_vars{'t_min'}, $prefix_vars{'t_sec'}); + $prefix_vars{'t_timestamp'} = "$prefix_vars{'t_year'}-$prefix_vars{'t_month'}-$prefix_vars{'t_day'} $prefix_vars{'t_hour'}:$prefix_vars{'t_min'}:$prefix_vars{'t_sec'}"; + } + # Extract information from log line prefix if (!$log_line_prefix) { &parse_log_prefix($prefix_vars{'t_logprefix'}); @@ -3101,7 +3096,7 @@ sub process_file # Some log line may be written by applications next if ($line =~ /\bLOG: /); - # Parse orphan lines to append inforamtion to the right place + # Parse orphan lines to append information to the right place &parse_orphan_line($cur_pid, $line); } else { @@ -3112,6 +3107,7 @@ sub process_file } last if (($stop_offset > 0) && ($current_offset >= $stop_offset)); } + if ($last_parsed) { $last_line{current_pos} = $current_offset; } @@ -3180,10 +3176,10 @@ sub process_file $bpath =~ s/\-/\//g; # Mark the directory as needing index update - if (open(OUT, ">>$last_parsed.tmp")) { - flock(OUT, 2) || return $getout; - print OUT "$incr_date\n"; - close(OUT); + if (open(my $out, '>>', "$last_parsed.tmp")) { + flock($out, 2) || return $getout; + print $out "$incr_date\n"; + close($out); } else { &logmsg('ERROR', "can't save last parsed line into $last_parsed.tmp, $!"); } @@ -3208,18 +3204,18 @@ sub process_file # Save last line into temporary file if ($last_parsed && (scalar keys %last_line || scalar keys %pgb_last_line)) { - if (open(OUT, ">>$tmp_last_parsed")) { - flock(OUT, 2) || return $getout; + if (open(my $out, '>>', "$tmp_last_parsed")) { + flock($out, 2) || return $getout; if ($fmt eq 'pgbouncer') { $pgb_last_line{current_pos} ||= 0; &logmsg('DEBUG', "Saving pgbouncer last parsed line into $tmp_last_parsed ($pgb_last_line{datetime}\t$pgb_last_line{current_pos})"); - print OUT "pgbouncer\t$pgb_last_line{datetime}\t$pgb_last_line{current_pos}\t$pgb_last_line{orig}\n"; + print $out "pgbouncer\t$pgb_last_line{datetime}\t$pgb_last_line{current_pos}\t$pgb_last_line{orig}\n"; } else { $last_line{current_pos} ||= 0; &logmsg('DEBUG', "Saving last parsed line into $tmp_last_parsed ($last_line{datetime}\t$last_line{current_pos})"); - print OUT "$last_line{datetime}\t$last_line{current_pos}\t$last_line{orig}\n"; + print $out "$last_line{datetime}\t$last_line{current_pos}\t$last_line{orig}\n"; } - close(OUT); + close($out); } else { &logmsg('ERROR', "can't save last parsed line into $tmp_last_parsed, $!"); } @@ -3234,6 +3230,87 @@ sub process_file return $getout; } +sub unescape_jsonlog +{ + my $str = shift; + + while ($str =~ s/([^\\])\\"/$1"/g) {}; + while ($str =~ s/([^\\])\\t/$1\t/g) {}; + while ($str =~ s/\\r\\n/\n/gs) {}; + while ($str =~ s/([^\\])\\r/$1\n/gs) {}; + while ($str =~ s/([^\\])\\n/$1\n/gs) {}; + + return $str; +} + +sub parse_jsonlog_input +{ + my $str = shift; + + my %infos = (); + + # Extract the date + if ($str =~ m/\{"timestamp":"(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)\.(\d+)/) { + + $infos{'t_year'} = $1; + $infos{'t_month'} = $2; + $infos{'t_day'} = $3; + $infos{'t_hour'} = $4; + $infos{'t_min'} = $5; + $infos{'t_sec'} = $6; + my $milli = $7 || 0; + $infos{'t_timestamp'} = "$infos{'t_year'}-$infos{'t_month'}-$infos{'t_day'} $infos{'t_hour'}:$infos{'t_min'}:$infos{'t_sec'}"; + } + + # Set query parameters as global variables + if ($str =~ m/"user":"(.*?)"(?:,"|\})/) { + $infos{'t_dbuser'} = $1; + } + if ($str =~ m/"dbname":"(.*?)"(?:,"|\})/) { + $infos{'t_dbname'} = $1; + } + if ($str =~ m/"application_name":"(.*?)"(?:,"|\})/) { + $infos{'t_appname'} = $1; + } + if ($str =~ m/"remote_host":"(.*?)"(?:,"|\})/) { + $infos{'t_client'} = $1; + $infos{'t_client'} =~ s/:.*//; + $infos{'t_client'} = _gethostbyaddr($infos{'t_client'}) if ($dns_resolv); + } + $infos{'t_host'} = 'jsonlog'; # this unused variable is used to store format information when log format is not syslog + if ($str =~ m/"pid":"(.*?)"(?:,"|\})/) { + $infos{'t_pid'} = $1; + } + if ($str =~ m/"error_severity":"(.*?)"(?:,"|\})/) { + $infos{'t_loglevel'} = $1; + } + if ($str =~ m/"state_code":"(.*?)"(?:,"|\})/) { + $infos{'t_sqlstate'} = $1; + } + if ($str =~ m/"message":"(.*?)"(?:,"|\})/) { + $infos{'t_query'} = unescape_jsonlog($1); + } elsif ($str =~ m/"statement":"(.*?)"(?:,"|\})/) { + $infos{'t_query'} = unescape_jsonlog($1); + } + + # Set ERROR additional information + if ($str =~ m/"(?:detail_log|detail)":"(.*?)"(?:,"|\})/) { + $infos{'t_detail'} = unescape_jsonlog($1); + } + if ($str =~ m/"hint":"(.*?)"(?:,"|\})/) { + $infos{'t_hint'} = unescape_jsonlog($1); + } + if ($str =~ m/"context":"(.*?)"(?:,"|\})/) { + $infos{'t_context'} = unescape_jsonlog($1); + } + if ($str =~ m/"(?:statement|internal_query)":"(.*?)"(?:,"|\})/) { + $infos{'t_statement'} = unescape_jsonlog($1); + } + + return %infos; +} + + sub parse_orphan_line { my ($cur_pid, $line) = @_; @@ -3243,7 +3320,7 @@ sub parse_orphan_line if ($line =~ /^\t?(pages|tuples): (\d+) removed, (\d+) remain/) { $autovacuum_info{tables}{$cur_info{$cur_pid}{vacuum}}{$1}{removed} += $2; } - if ($line =~ m#^\t?system usage: CPU .* sec elapsed (.*) sec#) { + if ($line =~ m#^\t?system usage: CPU .* (?:sec|s,) elapsed (.*) s#) { if ($1 > $autovacuum_info{peak}{system_usage}{elapsed}) { $autovacuum_info{peak}{system_usage}{elapsed} = $1; $autovacuum_info{peak}{system_usage}{table} = $cur_info{$cur_pid}{vacuum}; @@ -3572,10 +3649,10 @@ sub check_incremental_position $bpath =~ s/\-/\//g; # Mark this directory as needing a reindex - if (open(OUT, ">>$last_parsed.tmp")) { - flock(OUT, 2) || return 1; - print OUT "$incr_date\n"; - close(OUT); + if (open(my $out, '>>' , "$last_parsed.tmp")) { + flock($out, 2) || return 1; + print $out "$incr_date\n"; + close($out); } else { &logmsg('ERROR', "can't save last parsed line into $last_parsed.tmp, $!"); } @@ -3632,9 +3709,20 @@ sub normalize_query # Remove comments $orig_query =~ s/\/\*(.*?)\*\///gs; + # Keep case on object name between doublequote + my %objnames = (); + my $i = 0; + while ($orig_query =~ s/("[^"]+")/%%OBJNAME$i%%/) { + $objnames{$i} = $1; + $i++; + } # Set the entire query lowercase $orig_query = lc($orig_query); + # Restore object name + while ($orig_query =~ s/\%\%objname(\d+)\%\%/$objnames{$1}/gs) {}; + %objnames = (); + # Remove extra space, new line and tab characters by a single space $orig_query =~ s/\s+/ /gs; @@ -3648,17 +3736,17 @@ sub normalize_query # Remove string content $orig_query =~ s/\\'//gs; - $orig_query =~ s/'[^']*'/''/gs; - $orig_query =~ s/''('')+/''/gs; + $orig_query =~ s/'[^']*'/\?/gs; + $orig_query =~ s/\?(\?)+/\?/gs; # Remove NULL parameters - $orig_query =~ s/=\s*NULL/=''/gs; + $orig_query =~ s/=\s*NULL/= \?/gs; # Remove numbers - $orig_query =~ s/([^a-z0-9_\$\-])-?\d+/${1}0/gs; + $orig_query =~ s/([^a-z0-9_\$\-])-?\d+/$1\?/gs; # Remove hexadecimal numbers - $orig_query =~ s/([^a-z_\$-])0x[0-9a-f]{1,10}/${1}0x/gs; + $orig_query =~ s/([^a-z_\$-])0x[0-9a-f]{1,10}/$1\?/gs; # Remove bind parameters $orig_query =~ s/\$\d+/\?/gs; @@ -3667,7 +3755,12 @@ sub normalize_query $orig_query =~ s/\bin\s*\([\'0x,\s\?]*\)/in (...)/gs; # Remove curor names in CURSOR and IN clauses - $orig_query =~ s/\b(declare|in)\s+"[^"]+"/$1 "..."/gs; + $orig_query =~ s/\b(declare|in|deallocate|close)\s+"[^"]+"/$1 "..."/gs; + + # Normalise cursor name + $orig_query =~ s/\bdeclare\s+[^"\s]+\s+cursor/declare "..." cursor/gs; + $orig_query =~ s/\b(fetch\s+next\s+from)\s+[^\s]+/$1 "..."/gs; + $orig_query =~ s/\b(deallocate|close)\s+[^"\s]+/$1 "..."/gs; return $orig_query; } @@ -3933,6 +4026,27 @@ sub pgb_set_top_error_sample } } +sub get_log_limit +{ + $overall_stat{'first_log_ts'} =~ /^(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)/; + my ($t_y, $t_mo, $t_d, $t_h, $t_mi, $t_s); + if (!$log_timezone) { + ($t_y, $t_mo, $t_d, $t_h, $t_mi, $t_s) = ($1, $2, $3, $4, $5, $6); + } else { + ($t_y, $t_mo, $t_d, $t_h, $t_mi, $t_s) = change_timezone($1, $2, $3, $4, $5, $6); + } + my $t_log_min = "$t_y-$t_mo-$t_d $t_h:$t_mi:$t_s"; + $overall_stat{'last_log_ts'} =~ /^(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)/; + if (!$log_timezone) { + ($t_y, $t_mo, $t_d, $t_h, $t_mi, $t_s) = ($1, $2, $3, $4, $5, $6); + } else { + ($t_y, $t_mo, $t_d, $t_h, $t_mi, $t_s) = change_timezone($1, $2, $3, $4, $5, $6); + } + my $t_log_max = "$t_y-$t_mo-$t_d $t_h:$t_mi:$t_s"; + + return ($t_log_min, $t_log_max); +} + sub dump_as_text { @@ -3946,6 +4060,8 @@ sub dump_as_text if ($#log_files > 0) { $logfile_str .= ', ..., ' . $log_files[-1]; } + # Set logs limits + my ($t_log_min, $t_log_max) = get_log_limit(); print $fh qq{ pgBadger :: $report_title @@ -3954,7 +4070,7 @@ pgBadger :: $report_title Generated on $curdate Log file: $logfile_str Parsed $fmt_nlines log entries in $total_time -Log start from $overall_stat{'first_log_ts'} to $overall_stat{'last_log_ts'} +Log start from $t_log_min to $t_log_max }; @@ -4243,7 +4359,8 @@ Report not supported by text format foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$k}{samples}}) { last if ($j > $sample); my $ttl = $top_locked_info[$i]->[1] || ''; - my $db = " - $normalyzed_info{$k}{samples}{$d}{date} - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); + my $db = ''; + $db .= " - $normalyzed_info{$k}{samples}{$d}{date} - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); $db .= ", user: $normalyzed_info{$k}{samples}{$d}{user}" if ($normalyzed_info{$k}{samples}{$d}{user}); $db .= ", remote: $normalyzed_info{$k}{samples}{$d}{remote}" if ($normalyzed_info{$k}{samples}{$d}{remote}); $db .= ", app: $normalyzed_info{$k}{samples}{$d}{app}" if ($normalyzed_info{$k}{samples}{$d}{app}); @@ -4262,7 +4379,8 @@ Report not supported by text format print $fh "Rank Wait time (s) Query\n"; for (my $i = 0 ; $i <= $#top_locked_info ; $i++) { my $ttl = $top_locked_info[$i]->[1] || ''; - my $db = " - database: $top_locked_info[$i]->[3]" if ($top_locked_info[$i]->[3]); + my $db = ''; + $db .= " - database: $top_locked_info[$i]->[3]" if ($top_locked_info[$i]->[3]); $db .= ", user: $top_locked_info[$i]->[4]" if ($top_locked_info[$i]->[4]); $db .= ", remote: $top_locked_info[$i]->[5]" if ($top_locked_info[$i]->[5]); $db .= ", app: $top_locked_info[$i]->[6]" if ($top_locked_info[$i]->[6]); @@ -4304,7 +4422,8 @@ Report not supported by text format my $j = 1; foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$k}{samples}}) { last if ($j > $sample); - my $db = "$normalyzed_info{$k}{samples}{$d}{date} - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); + my $db = ''; + $db .= "$normalyzed_info{$k}{samples}{$d}{date} - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); $db .= ", user: $normalyzed_info{$k}{samples}{$d}{user}" if ($normalyzed_info{$k}{samples}{$d}{user}); $db .= ", remote: $normalyzed_info{$k}{samples}{$d}{remote}" if ($normalyzed_info{$k}{samples}{$d}{remote}); $db .= ", app: $normalyzed_info{$k}{samples}{$d}{app}" if ($normalyzed_info{$k}{samples}{$d}{app}); @@ -4325,7 +4444,8 @@ Report not supported by text format print $fh "Rank Size Query\n"; for (my $i = 0 ; $i <= $#top_tempfile_info ; $i++) { my $ttl = $top_tempfile_info[$i]->[1] || ''; - my $db = " - database: $top_tempfile_info[$i]->[3]" if ($top_tempfile_info[$i]->[3]); + my $db = ''; + $db .= " - database: $top_tempfile_info[$i]->[3]" if ($top_tempfile_info[$i]->[3]); $db .= ", user: $top_tempfile_info[$i]->[4]" if ($top_tempfile_info[$i]->[4]); $db .= ", remote: $top_tempfile_info[$i]->[5]" if ($top_tempfile_info[$i]->[5]); $db .= ", app: $top_tempfile_info[$i]->[6]" if ($top_tempfile_info[$i]->[6]); @@ -4361,7 +4481,8 @@ Report not supported by text format my $j = 1; foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$k}{samples}}) { last if ($j > $sample); - my $db = "$normalyzed_info{$k}{samples}{$d}{date} - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); + my $db = ''; + $db .= "$normalyzed_info{$k}{samples}{$d}{date} - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); $db .= ", user: $normalyzed_info{$k}{samples}{$d}{user}" if ($normalyzed_info{$k}{samples}{$d}{user}); $db .= ", remote: $normalyzed_info{$k}{samples}{$d}{remote}" if ($normalyzed_info{$k}{samples}{$d}{remote}); $db .= ", app: $normalyzed_info{$k}{samples}{$d}{app}" if ($normalyzed_info{$k}{samples}{$d}{app}); @@ -4382,7 +4503,8 @@ Report not supported by text format print $fh "Rank Times cancelled Query\n"; for (my $i = 0 ; $i <= $#top_cancelled_info ; $i++) { my $ttl = $top_cancelled_info[$i]->[1] || ''; - my $db = " - database: $top_cancelled_info[$i]->[3]" if ($top_cancelled_info[$i]->[3]); + my $db = ''; + $db .= " - database: $top_cancelled_info[$i]->[3]" if ($top_cancelled_info[$i]->[3]); $db .= ", user: $top_cancelled_info[$i]->[4]" if ($top_cancelled_info[$i]->[4]); $db .= ", remote: $top_cancelled_info[$i]->[5]" if ($top_cancelled_info[$i]->[5]); $db .= ", app: $top_cancelled_info[$i]->[6]" if ($top_cancelled_info[$i]->[6]); @@ -4398,7 +4520,8 @@ Report not supported by text format print $fh "\n- Slowest queries ------------------------------------------------------\n\n"; print $fh "Rank Duration (s) Query\n"; for (my $i = 0 ; $i <= $#top_slowest ; $i++) { - my $db = " database: $top_slowest[$i]->[3]" if ($top_slowest[$i]->[3]); + my $db = ''; + $db .= " database: $top_slowest[$i]->[3]" if ($top_slowest[$i]->[3]); $db .= ", user: $top_slowest[$i]->[4]" if ($top_slowest[$i]->[4]); $db .= ", remote: $top_slowest[$i]->[5]" if ($top_slowest[$i]->[5]); $db .= ", app: $top_slowest[$i]->[6]" if ($top_slowest[$i]->[6]); @@ -4433,7 +4556,8 @@ Report not supported by text format my $j = 1; foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$k}{samples}}) { last if ($j > $sample); - my $db = " - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); + my $db = ''; + $db .= " - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); $db .= ", user: $normalyzed_info{$k}{samples}{$d}{user}" if ($normalyzed_info{$k}{samples}{$d}{user}); $db .= ", remote: $normalyzed_info{$k}{samples}{$d}{remote}" if ($normalyzed_info{$k}{samples}{$d}{remote}); $db .= ", app: $normalyzed_info{$k}{samples}{$d}{app}" if ($normalyzed_info{$k}{samples}{$d}{app}); @@ -4471,7 +4595,8 @@ Report not supported by text format my $i = 1; foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$k}{samples}}) { last if ($i > $sample); - my $db = " - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); + my $db = ''; + $db .= " - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); $db .= ", user: $normalyzed_info{$k}{samples}{$d}{user}" if ($normalyzed_info{$k}{samples}{$d}{user}); $db .= ", remote: $normalyzed_info{$k}{samples}{$d}{remote}" if ($normalyzed_info{$k}{samples}{$d}{remote}); $db .= ", app: $normalyzed_info{$k}{samples}{$d}{app}" if ($normalyzed_info{$k}{samples}{$d}{app}); @@ -4509,7 +4634,8 @@ Report not supported by text format my $i = 1; foreach my $d (sort {$b <=> $a} keys %{$normalyzed_info{$k}{samples}}) { last if ($i > $sample); - my $db = " - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); + my $db = ''; + $db .= " - database: $normalyzed_info{$k}{samples}{$d}{db}" if ($normalyzed_info{$k}{samples}{$d}{db}); $db .= ", user: $normalyzed_info{$k}{samples}{$d}{user}" if ($normalyzed_info{$k}{samples}{$d}{user}); $db .= ", remote: $normalyzed_info{$k}{samples}{$d}{remote}" if ($normalyzed_info{$k}{samples}{$d}{remote}); $db .= ", app: $normalyzed_info{$k}{samples}{$d}{app}" if ($normalyzed_info{$k}{samples}{$d}{app}); @@ -4625,6 +4751,8 @@ sub dump_error_as_text } $report_title ||= 'PostgreSQL Log Analyzer'; + # Set logs limits + my ($t_log_min, $t_log_max) = get_log_limit(); print $fh qq{ pgBadger :: $report_title @@ -4633,7 +4761,7 @@ pgBadger :: $report_title Generated on $curdate Log file: $logfile_str Parsed $fmt_nlines log entries in $total_time -Log start from $overall_stat{'first_log_ts'} to $overall_stat{'last_log_ts'} +Log start from $t_log_min to $t_log_max }; &show_error_as_text(); @@ -4815,7 +4943,9 @@ sub html_header my $global_info = &print_global_information(); my @tmpjscode = @jscode; - map { s/EDIT_URI/$uri/; } @tmpjscode; + for (my $i = 0; $i <= $#tmpjscode; $i++) { + $tmpjscode[$i] =~ s/EDIT_URI/\./; + } my $local_title = 'PostgreSQL Log Analyzer'; if ($report_title) { @@ -5000,7 +5130,8 @@ sub html_header } } if (!$disable_error && !$pgbouncer_only) { - my $sqlstate_report = '
  • Error class distribution
    • ' if (scalar keys %errors_code > 0); + my $sqlstate_report = ''; + $sqlstate_report = '
  • Error class distribution
    • ' if (scalar keys %errors_code > 0); print $fh qq{