From: Darold Gilles Date: Sat, 12 Apr 2014 09:07:45 +0000 (+0200) Subject: Histogram granularity can be adjusted using the -A command line option. By default... X-Git-Tag: v5.1~22 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=171759277619969cddbd41e0bd2ca16fd1fad568;p=pgbadger Histogram granularity can be adjusted using the -A command line option. By default they will report the mean of each top queries/error occuring per hour, but you can specify the granularity down to the minute. --- diff --git a/README b/README index fd91a3c..02f62a2 100644 --- a/README +++ b/README @@ -16,6 +16,8 @@ SYNOPSIS -a | --average minutes : number of minutes to build the average graphs of queries and connections. + -A | --histo-avg minutes: number of minutes to build the histogram graphs + of queries. Default 60 minutes. -b | --begin datetime : start date/time for the data to be parsed in log. -c | --dbclient host : only report on entries for the given client host. -C | --nocomment : remove comments like /* ... */ from queries. @@ -214,6 +216,10 @@ FEATURE You can also have incremental reports with one report per day and a cumulative report per week. + Histogram granularity can be adjusted using the -A command line option. + By default they will report the mean of each top queries/error occuring + per hour, but you can specify the granularity down to the minute. + REQUIREMENT pgBadger comes as a single Perl script - you do not need anything other than a modern Perl distribution. Charts are rendered using a Javascript diff --git a/doc/pgBadger.pod b/doc/pgBadger.pod index 817b0b4..85ac27d 100644 --- a/doc/pgBadger.pod +++ b/doc/pgBadger.pod @@ -18,6 +18,8 @@ Options: -a | --average minutes : number of minutes to build the average graphs of queries and connections. + -A | --histo-avg minutes: number of minutes to build the histogram graphs + of queries. Default 60 minutes. -b | --begin datetime : start date/time for the data to be parsed in log. -c | --dbclient host : only report on entries for the given client host. -C | --nocomment : remove comments like /* ... */ from queries. @@ -195,9 +197,15 @@ There's also some pie reports of distribution about: Connections per database/user/client. Autovacuum and autoanalyze per table. -All charts are zoomable and can be saved as PNG images. SQL queries reported are highlighted and beautified automatically. +All charts are zoomable and can be saved as PNG images. SQL queries reported are +highlighted and beautified automatically. -You can also have incremental reports with one report per day and a cumulative report per week. +You can also have incremental reports with one report per day and a cumulative +report per week. + +Histogram granularity can be adjusted using the -A command line option. By default +they will report the mean of each top queries/error occuring per hour, but you can +specify the granularity down to the minute. =head1 REQUIREMENT diff --git a/pgbadger b/pgbadger index 47586c8..542d050 100644 --- a/pgbadger +++ b/pgbadger @@ -142,6 +142,7 @@ my $disable_temporary = 0; my $disable_checkpoint = 0; my $disable_autovacuum = 0; my $avg_minutes = 5; +my $histo_avg_minutes = 60; my $last_parsed = ''; my $report_title = 'PostgreSQL log analyzer'; my $log_line_prefix = ''; @@ -217,6 +218,7 @@ $| = 1; # get the command line parameters my $result = GetOptions( "a|average=i" => \$avg_minutes, + "A|histo-average=i" => \$histo_avg_minutes, "b|begin=s" => \$from, "c|dbclient=s" => \@dbclient, "C|nocomment!" => \$remove_comment, @@ -308,10 +310,17 @@ $progress = 0 if ($quiet); $avg_minutes ||= 5; $avg_minutes = 60 if ($avg_minutes > 60); $avg_minutes = 1 if ($avg_minutes < 1); +$histo_avg_minutes ||= 60; +$histo_avg_minutes = 60 if ($histo_avg_minutes > 60); +$histo_avg_minutes = 1 if ($histo_avg_minutes < 1); my @avgs = (); for (my $i = 0 ; $i < 60 ; $i += $avg_minutes) { push(@avgs, sprintf("%02d", $i)); } +my @histo_avgs = (); +for (my $i = 0 ; $i < 60 ; $i += $histo_avg_minutes) { + push(@histo_avgs, sprintf("%02d", $i)); +} # Set error like log level regex my $parse_regex = qr/^(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|HINT|STATEMENT|CONTEXT)/; @@ -1221,7 +1230,9 @@ Arguments: Options: -a | --average minutes : number of minutes to build the average graphs of - queries and connections. + queries and connections. Default 5 minutes. + -A | --histo-avg minutes: number of minutes to build the histogram graphs + of queries. Default 60 minutes. -b | --begin datetime : start date/time for the data to be parsed in log. -c | --dbclient host : only report on entries for the given client host. -C | --nocomment : remove comments like /* ... */ from queries. @@ -6426,28 +6437,45 @@ sub print_time_consuming foreach my $h (sort keys %{$normalyzed_info{$k}{chronos}{$d}}) { $normalyzed_info{$k}{chronos}{$d}{$h}{average} = $normalyzed_info{$k}{chronos}{$d}{$h}{duration} / ($normalyzed_info{$k}{chronos}{$d}{$h}{count} || 1); - $hourly_count{"$h"} += $normalyzed_info{$k}{chronos}{$d}{$h}{count}; - $hourly_duration{"$h"} += $normalyzed_info{$k}{chronos}{$d}{$h}{duration}; $details .= "$zday$h" . &comma_numbers($normalyzed_info{$k}{chronos}{$d}{$h}{count}) . "" . &convert_time($normalyzed_info{$k}{chronos}{$d}{$h}{duration}) . "" . &convert_time($normalyzed_info{$k}{chronos}{$d}{$h}{average}) . ""; $zday = ""; + foreach my $m (sort keys %{$normalyzed_info{$k}{chronos}{$d}{$h}{min}}) { + my $rd = &average_per_minutes($m, $histo_avg_minutes); + $hourly_count{"$h:$rd"} += $normalyzed_info{$k}{chronos}{$d}{$h}{min}{$m}; + $hourly_duration{"$h:$rd"} += ($normalyzed_info{$k}{chronos}{$d}{$h}{min_duration}{$m} || 0); + } + foreach my $rd (@histo_avgs) { + next if (!exists $hourly_count{"$h:$rd"}); + $details .= "$zday$h:$rd" . + &comma_numbers($hourly_count{"$h:$rd"}) . "" . + &convert_time($hourly_duration{"$h:$rd"}) . "" . + &convert_time($hourly_duration{"$h:$rd"}/($hourly_count{"$h:$rd"}||1)) . ""; + } } } # Set graph dataset my %graph_data = (); + # we need a start date for flotr2 graph, we don't care of it as we just display HH:MM + my $etime = timegm_nocheck(0, 0, 0, 18, 4, 80) * 1000; + my $ctime = 0; foreach my $h ("00" .. "23") { - $graph_data{count} .= "[$h, " . (int($hourly_count{"$h"}/$days) || 0) . "],"; - $graph_data{duration} .= "[$h, " . (int($hourly_duration{"$h"} / ($hourly_count{"$h"} || 1)) || 0) . "],"; + foreach my $rd (@histo_avgs) { + $ctime = (($h*3600)+($rd*60))*1000 + $etime; + $graph_data{count} .= "[$ctime, " . (int($hourly_count{"$h:$rd"}/$days) || 0) . "],"; + $graph_data{duration} .= "[$ctime, " . (int($hourly_duration{"$h:$rd"} / ($hourly_count{"$h:$rd"} || 1)) || 0) . "],"; + } } $graph_data{count} =~ s/,$//; $graph_data{duration} =~ s/,$//; %hourly_count = (); %hourly_duration = (); + $ctime = timegm_nocheck(0, 0, 0, 19, 4, 80) * 1000; my $query_histo = - &flotr2_histograph($graphid++, 'timeconsuming_graph_'.$rank, $graph_data{count}, $graph_data{duration}); + &flotr2_histograph($graphid++, 'timeconsuming_graph_'.$rank, $graph_data{count}, $graph_data{duration}, $etime, $ctime); print $fh qq{ @@ -6567,28 +6595,45 @@ sub print_most_frequent foreach my $h (sort keys %{$normalyzed_info{$k}{chronos}{$d}}) { $normalyzed_info{$k}{chronos}{$d}{$h}{average} = $normalyzed_info{$k}{chronos}{$d}{$h}{duration} / $normalyzed_info{$k}{chronos}{$d}{$h}{count}; - $hourly_count{"$h"} += $normalyzed_info{$k}{chronos}{$d}{$h}{count}; - $hourly_duration{"$h"} += $normalyzed_info{$k}{chronos}{$d}{$h}{duration}; $details .= "$zday$h" . &comma_numbers($normalyzed_info{$k}{chronos}{$d}{$h}{count}) . "" . &convert_time($normalyzed_info{$k}{chronos}{$d}{$h}{duration}) . "" . &convert_time($normalyzed_info{$k}{chronos}{$d}{$h}{average}) . ""; $zday = ""; + foreach my $m (sort keys %{$normalyzed_info{$k}{chronos}{$d}{$h}{min}}) { + my $rd = &average_per_minutes($m, $histo_avg_minutes); + $hourly_count{"$h:$rd"} += $normalyzed_info{$k}{chronos}{$d}{$h}{min}{$m}; + $hourly_duration{"$h:$rd"} += ($normalyzed_info{$k}{chronos}{$d}{$h}{min_duration}{$m} || 0); + } + foreach my $rd (@histo_avgs) { + next if (!exists $hourly_count{"$h:$rd"}); + $details .= "$zday$h:$rd" . + &comma_numbers($hourly_count{"$h:$rd"}) . "" . + &convert_time($hourly_duration{"$h:$rd"}) . "" . + &convert_time($hourly_duration{"$h:$rd"}/($hourly_count{"$h:$rd"}||1)) . ""; + } } } # Set graph dataset my %graph_data = (); - foreach my $h ("00" .. "23") { - $graph_data{count} .= "[$h, " . (int($hourly_count{"$h"}/$days) || 0) . "],"; - $graph_data{duration} .= "[$h, " . (int($hourly_duration{"$h"} / ($hourly_count{"$h"} || 1)) || 0) . "],"; + # we need a start date for flotr2 graph, we don't care of it as we just display HH:MM + my $etime = timegm_nocheck(0, 0, 0, 18, 4, 80) * 1000; + my $ctime = 0; + foreach my $h ("00" .. "23") { + foreach my $rd (@histo_avgs) { + $ctime = (($h*3600)+($rd*60))*1000 + $etime; + $graph_data{count} .= "[$ctime, " . (int($hourly_count{"$h:$rd"}/$days) || 0) . "],"; + $graph_data{duration} .= "[$ctime, " . (int($hourly_duration{"$h:$rd"} / ($hourly_count{"$h:$rd"} || 1)) || 0) . "],"; + } } $graph_data{count} =~ s/,$//; $graph_data{duration} =~ s/,$//; %hourly_count = (); %hourly_duration = (); + $ctime = timegm_nocheck(0, 0, 0, 19, 4, 80) * 1000; my $query_histo = - &flotr2_histograph($graphid++, 'mostfrequent_graph_'.$rank, $graph_data{count}, $graph_data{duration}); + &flotr2_histograph($graphid++, 'mostfrequent_graph_'.$rank, $graph_data{count}, $graph_data{duration}, $etime, $ctime); print $fh qq{ @@ -6709,28 +6754,45 @@ sub print_slowest_queries foreach my $h (sort keys %{$normalyzed_info{$k}{chronos}{$d}}) { $normalyzed_info{$k}{chronos}{$d}{$h}{average} = $normalyzed_info{$k}{chronos}{$d}{$h}{duration} / $normalyzed_info{$k}{chronos}{$d}{$h}{count}; - $hourly_count{"$h"} += $normalyzed_info{$k}{chronos}{$d}{$h}{count}; - $hourly_duration{"$h"} += $normalyzed_info{$k}{chronos}{$d}{$h}{duration}; $details .= "$zday$h" . &comma_numbers($normalyzed_info{$k}{chronos}{$d}{$h}{count}) . "" . &convert_time($normalyzed_info{$k}{chronos}{$d}{$h}{duration}) . "" . &convert_time($normalyzed_info{$k}{chronos}{$d}{$h}{average}) . ""; $zday = ""; + foreach my $m (sort keys %{$normalyzed_info{$k}{chronos}{$d}{$h}{min}}) { + my $rd = &average_per_minutes($m, $histo_avg_minutes); + $hourly_count{"$h:$rd"} += $normalyzed_info{$k}{chronos}{$d}{$h}{min}{$m}; + $hourly_duration{"$h:$rd"} += ($normalyzed_info{$k}{chronos}{$d}{$h}{min_duration}{$m} || 0); + } + foreach my $rd (@histo_avgs) { + next if (!exists $hourly_count{"$h:$rd"}); + $details .= "$zday$h:$rd" . + &comma_numbers($hourly_count{"$h:$rd"}) . "" . + &convert_time($hourly_duration{"$h:$rd"}) . "" . + &convert_time($hourly_duration{"$h:$rd"}/($hourly_count{"$h:$rd"}||1)) . ""; + } } } # Set graph dataset my %graph_data = (); + # we need a start date for flotr2 graph, we don't care of it as we just display HH:MM + my $etime = timegm_nocheck(0, 0, 0, 18, 4, 80) * 1000; + my $ctime = 0; foreach my $h ("00" .. "23") { - $graph_data{count} .= "[$h, " . (int($hourly_count{"$h"}/$days) || 0) . "],"; - $graph_data{duration} .= "[$h, " . (int($hourly_duration{"$h"} / ($hourly_count{"$h"} || 1)) || 0) . "],"; + foreach my $rd (@histo_avgs) { + $ctime = (($h*3600)+($rd*60))*1000 + $etime; + $graph_data{count} .= "[$ctime, " . (int($hourly_count{"$h:$rd"}/$days) || 0) . "],"; + $graph_data{duration} .= "[$ctime, " . (int($hourly_duration{"$h:$rd"} / ($hourly_count{"$h:$rd"} || 1)) || 0) . "],"; + } } $graph_data{count} =~ s/,$//; $graph_data{duration} =~ s/,$//; %hourly_count = (); %hourly_duration = (); + $ctime = timegm_nocheck(0, 0, 0, 19, 4, 80) * 1000; my $query_histo = - &flotr2_histograph($graphid++, 'normalizedslowest_graph_'.$rank, $graph_data{count}, $graph_data{duration}); + &flotr2_histograph($graphid++, 'normalizedslowest_graph_'.$rank, $graph_data{count}, $graph_data{duration}, $etime, $ctime); print $fh qq{ @@ -7185,20 +7247,35 @@ sub show_error_as_html foreach my $h (sort keys %{$error_info{$k}{chronos}{$d}}) { $details .= "$zday$h" . &comma_numbers($error_info{$k}{chronos}{$d}{$h}{count}) . ""; - $hourly_count{"$h"} += $error_info{$k}{chronos}{$d}{$h}{count}; $zday = ""; + foreach my $m (sort keys %{$error_info{$k}{chronos}{$d}{$h}{min}}) { + my $rd = &average_per_minutes($m, $histo_avg_minutes); + $hourly_count{"$h:$rd"} += $error_info{$k}{chronos}{$d}{$h}{min}{$m}; + } + foreach my $rd (@histo_avgs) { + next if (!exists $hourly_count{"$h:$rd"}); + $details .= "$zday$h:$rd" . + &comma_numbers($hourly_count{"$h:$rd"}) . ""; + } } } # Set graph dataset my %graph_data = (); + # we need a start date for flotr2 graph, we don't care of it as we just display HH:MM + my $etime = timegm_nocheck(0, 0, 0, 18, 4, 80) * 1000; + my $ctime = 0; foreach my $h ("00" .. "23") { - $graph_data{count} .= "[$h, " . (int($hourly_count{"$h"}/$days) || 0) . "],"; + foreach my $rd (@histo_avgs) { + $ctime = (($h*3600)+($rd*60))*1000 + $etime; + $graph_data{count} .= "[$ctime, " . (int($hourly_count{"$h:$rd"}/$days) || 0) . "],"; + } } - $graph_data{count} =~ s/,$//; + $graph_data{count} =~ s/,$//; %hourly_count = (); + $ctime = timegm_nocheck(0, 0, 0, 19, 4, 80) * 1000; my $error_histo = - &flotr2_histograph($graphid++, 'error_graph_'.$rank, $graph_data{count}); + &flotr2_histograph($graphid++, 'error_graph_'.$rank, $graph_data{count}, '', $etime, $ctime); # Escape HTML code in error message $msg = &escape_html($msg); @@ -7495,6 +7572,9 @@ sub load_stats foreach my $day (keys %{ $_error_info{$q}{chronos} }) { foreach my $hour (keys %{$_error_info{$q}{chronos}{$day}}) { $error_info{$q}{chronos}{$day}{$hour}{count} += $_error_info{$q}{chronos}{$day}{$hour}{count}; + foreach my $min (keys %{$_error_info{$q}{chronos}{$day}{$hour}{min}}) { + $error_info{$q}{chronos}{$day}{$hour}{min}{$min} += $_error_info{$q}{chronos}{$day}{$hour}{min}{$min}; + } } } } @@ -7625,6 +7705,14 @@ sub load_stats $_normalyzed_info{$stmt}{chronos}{$day}{$hour}{count}; $normalyzed_info{$stmt}{chronos}{$day}{$hour}{duration} += $_normalyzed_info{$stmt}{chronos}{$day}{$hour}{duration}; + foreach my $min (keys %{$_normalyzed_info{$stmt}{chronos}{$day}{$hour}{min}} ) { + $normalyzed_info{$stmt}{chronos}{$day}{$hour}{min}{$min} += + $_normalyzed_info{$stmt}{chronos}{$day}{$hour}{min}{$min}; + } + foreach my $min (keys %{$_normalyzed_info{$stmt}{chronos}{$day}{$hour}{min_duration}} ) { + $normalyzed_info{$stmt}{chronos}{$day}{$hour}{min_duration}{$min} += + $_normalyzed_info{$stmt}{chronos}{$day}{$hour}{min_duration}{$min}; + } } } @@ -8655,6 +8743,7 @@ sub store_queries # Stores normalized error count per time $error_info{$normalized_error}{chronos}{"$cur_day_str"}{"$cur_hour_str"}{count}++; + $error_info{$normalized_error}{chronos}{"$cur_day_str"}{"$cur_hour_str"}{min}{$cur_info{$t_pid}{min}}++; # Stores normalized query samples if ($sample > 0) { @@ -8766,6 +8855,7 @@ sub store_queries # Store normalized query count and duration per time $normalyzed_info{$normalized}{chronos}{"$cur_day_str"}{"$cur_hour_str"}{count}++; + $normalyzed_info{$normalized}{chronos}{"$cur_day_str"}{"$cur_hour_str"}{min}{$cur_info{$t_pid}{min}}++; if ($cur_info{$t_pid}{duration}) { # Update top slowest queries statistics @@ -8773,6 +8863,7 @@ sub store_queries # Store normalized query total duration $normalyzed_info{$normalized}{duration} += $cur_info{$t_pid}{duration}; + $normalyzed_info{$normalized}{chronos}{"$cur_day_str"}{"$cur_hour_str"}{min_duration}{$cur_info{$t_pid}{min}} += $cur_info{$t_pid}{duration}; # Store min / max duration if (!exists $normalyzed_info{$normalized}{min} || ($normalyzed_info{$normalized}{min} > $cur_info{$t_pid}{duration})) { $normalyzed_info{$normalized}{min} = $cur_info{$t_pid}{duration}; @@ -8876,18 +8967,18 @@ sub average_per_minutes my $val = shift; my $idx = shift; - my @avgs = (); + my @lavgs = (); for (my $i = 0 ; $i < 60 ; $i += $idx) { - push(@avgs, sprintf("%02d", $i)); + push(@lavgs, sprintf("%02d", $i)); } - for (my $i = 0 ; $i <= $#avgs ; $i++) { - if ($val == $avgs[$i]) { - return "$avgs[$i]"; - } elsif ($i == $#avgs) { - return "$avgs[$i]"; - } elsif (($val > $avgs[$i]) && ($val < $avgs[$i + 1])) { - return "$avgs[$i]"; + for (my $i = 0 ; $i <= $#lavgs ; $i++) { + if ($val == $lavgs[$i]) { + return "$lavgs[$i]"; + } elsif ($i == $#lavgs) { + return "$lavgs[$i]"; + } elsif (($val > $lavgs[$i]) && ($val < $lavgs[$i + 1])) { + return "$lavgs[$i]"; } } return $val; @@ -9277,7 +9368,7 @@ EOF sub flotr2_histograph { - my ($buttonid, $divid, $data1, $data2) = @_; + my ($buttonid, $divid, $data1, $data2, $min, $max) = @_; if (!$data1) { return qq{ @@ -9308,9 +9399,11 @@ sub flotr2_histograph $dateTracker_dataopts =~ s/,$//; $dateTracker_dataopts = "[$dateTracker_dataopts]"; - my $yaxis2 = "y2axis: { mode: \"normal\", title: \"Duration\", min: 0, color: \"#8dbd0f\", tickFormatter: function(val){ return pretty_print_number(val,'duration') }, },"; + my $yaxis2 = "y2axis: { mode: \"normal\", noTicks: 4, title: \"Duration\", min: 0, color: \"#8dbd0f\", tickFormatter: function(val){ return pretty_print_number(val,'duration') }, },"; $yaxis2 = '' if (!$data2); + my $maxticks = (23*3600)+($histo_avgs[-1]*60); + return <