From: Darold Gilles Date: Fri, 12 Oct 2012 16:27:29 +0000 (+0200) Subject: Add new --enable-log_min_duration option to force pgbadger to use lines generated... X-Git-Tag: v3.2~123 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=cbe2cfaae17df24c7ad35a3db22c5af8e2e6d0e3;p=pgbadger Add new --enable-log_min_duration option to force pgbadger to use lines generated by the log_min_duration_statement even if the log_duration format is autodetected. Useful if you use both but do not log all queries. Thanks to Vincent Laborie for the feature request. --- diff --git a/README b/README index 7c64191..fb909c1 100644 --- a/README +++ b/README @@ -70,6 +70,8 @@ SYNOPSIS --disable-checkpoint : do not generate checkpoint report. --enable-log_duration : force pgbadger to use log_duration even if log_min_duration_statement format is autodetected. + --enable-log_min_duration: force pgbadger to use log_min_duration even if + log_duration format is autodetected. Examples: @@ -179,7 +181,7 @@ POSTGRESQL CONFIGURATION log_min_duration_statement = 0 Note that pgBadger is not compatible with statements logs provided by - log_statement and log_duration. + log_statement and log_duration. See next chapter for more information. With 'stderr' log format, log_line_prefix must be at least: @@ -219,6 +221,34 @@ POSTGRESQL CONFIGURATION but this is not only recommanded by pgbadger. +log_min_duration_statement versus log_duration + If you want full statistics reports from your log file you must set + log_min_duration_statement = 0. If you just want to report duration and + number of queries and don't want all detail about queries, set + log_min_duration_statement to -1 and enable log_duration. If you want to + report only queries that tooks more than 5 seconds for example but still + want to report all queries duration and number of queries you will need + to generate 2 reports. One using the log_min_duration_statement and the + second using the log_duration. Proceed as follow: + + pgbadger --enable-log_duration /var/log/postgresql.log + + to generate hourly statistics about the number of queries and duration + stats. To generate detailled reports about queries use the following + command: + + pgbadger --enable-log_min_duration /var/log/postgresql.log + + Note that enabling log_statement will not help at all and enabling those + two options in the same command will report an error. + + Use those options if you don't want to log each queries to preserve your + I/O but still want to know the slowest queries upper a certain time and + still have a report on the number of queries and their duration. + Otherwize if you don't have too much performance lost with a + log_min_duration_statement set to 0, do not enable log_duration in your + postgresql.conf file. + INSTALLATION Download the tarball from github and unpack the archive as follow: diff --git a/doc/pgBadger.pod b/doc/pgBadger.pod index 3f9b3d0..7f871ff 100644 --- a/doc/pgBadger.pod +++ b/doc/pgBadger.pod @@ -72,6 +72,8 @@ Options: --disable-checkpoint : do not generate checkpoint report. --enable-log_duration : force pgbadger to use log_duration even if log_min_duration_statement format is autodetected. + --enable-log_min_duration: force pgbadger to use log_min_duration even if + log_duration format is autodetected. Examples: @@ -165,6 +167,7 @@ You must first enable SQL query logging to have something to parse: log_min_duration_statement = 0 Note that pgBadger is not compatible with statements logs provided by log_statement and log_duration. +See next chapter for more information. With 'stderr' log format, log_line_prefix must be at least: @@ -202,6 +205,29 @@ Of course your log messages should be in english without locale support: but this is not only recommanded by pgbadger. +=head1 log_min_duration_statement versus log_duration + +If you want full statistics reports from your log file you must set log_min_duration_statement = 0. +If you just want to report duration and number of queries and don't want all detail about queries, +set log_min_duration_statement to -1 and enable log_duration. If you want to report only queries that +tooks more than 5 seconds for example but still want to report all queries duration and number of +queries you will need to generate 2 reports. One using the log_min_duration_statement and the second +using the log_duration. Proceed as follow: + + pgbadger --enable-log_duration /var/log/postgresql.log + +to generate hourly statistics about the number of queries and duration stats. To generate detailled +reports about queries use the following command: + + pgbadger --enable-log_min_duration /var/log/postgresql.log + +Note that enabling log_statement will not help at all and enabling those two options in the same +command will report an error. + +Use those options if you don't want to log each queries to preserve your I/O but still want to know +the slowest queries upper a certain time and still have a report on the number of queries and their +duration. Otherwize if you don't have too much performance lost with a log_min_duration_statement +set to 0, do not enable log_duration in your postgresql.conf file. =head1 INSTALLATION diff --git a/pgbadger b/pgbadger index 94cd0ee..580d7d2 100755 --- a/pgbadger +++ b/pgbadger @@ -95,6 +95,7 @@ my $t_min_hour = 0; my $t_max_hour = 0; my $log_duration = 0; my $enable_log_duration = 0; +my $enable_log_min_duration = 0; my $NUMPROGRESS = 10000; my @DIMENSIONS = (800, 300); @@ -154,6 +155,7 @@ my $result = GetOptions( "disable-temporary!" => \$disable_temporary, "disable-checkpoint!" => \$disable_checkpoint, "enable-log_duration!" => \$enable_log_duration, + "enable-log_min_duration!" => \$enable_log_min_duration, ); if ($ver) { @@ -196,6 +198,12 @@ $format ||= &autodetect_format($log_files[0]); $log_duration ||= &autodetect_duration($log_files[0]); $log_duration = 1 if ($enable_log_duration); +$log_duration = 0 if ($enable_log_min_duration); + +if ($enable_log_min_duration && $enable_log_duration) { + print STDERR "FATAL: you must choose between reports based on log_duration or log_min_duration_statement.\n\n"; + &usage(); +} # Set default top query $top ||= 20; @@ -881,6 +889,8 @@ Options: --disable-checkpoint : do not generate checkpoint report. --enable-log_duration : force pgbadger to use log_duration even if log_min_duration_statement format is autodetected + --enable-log_min_duration: force pgbadger to use log_min_duration even if + log_duration format is autodetected. Examples: @@ -1145,7 +1155,7 @@ Report not supported by text format }; } - if (!$disable_type && !$log_duration) { + if (!$disable_type && (!$log_duration || $enable_log_min_duration)) { # INSERT/DELETE/UPDATE/SELECT repartition my $totala = $overall_stat{'SELECT'} + $overall_stat{'INSERT'} + $overall_stat{'UPDATE'} + $overall_stat{'DELETE'}; @@ -1248,7 +1258,7 @@ Report not supported by text format } # Show top informations - if (!$disable_query && !$log_duration) { + if (!$disable_query && (!$log_duration || $enable_log_min_duration)) { print $fh "\n- Slowest queries ------------------------------------------------------\n\n"; print $fh "Rank Duration (s) Query\n"; for (my $i = 0 ; $i <= $#top_slowest ; $i++) { @@ -1672,10 +1682,10 @@ EOF if (!$disable_hourly && $overall_stat{'queries_number'}) { print $fh qq{Hourly statistics | }; } - if (!$disable_type && !$log_duration) { + if (!$disable_type && (!$log_duration || $enable_log_min_duration)) { print $fh qq{Queries by type | }; } - if (!$disable_query && !$log_duration) { + if (!$disable_query && (!$log_duration || $enable_log_min_duration)) { print $fh qq{ Slowest queries | Queries that took up the most time (N) | @@ -1738,7 +1748,7 @@ sub html_footer if (!$disable_hourly && $overall_stat{'queries_number'}) { print $fh qq{
  • Hourly statistics
    • }; } - if (!$disable_type && !$log_duration) { + if (!$disable_type && (!$log_duration || $enable_log_min_duration)) { print $fh qq{
  • Queries by type
    • }; } if (!$disable_lock && scalar keys %lock_info > 0) { @@ -1766,7 +1776,7 @@ sub html_footer print $fh qq{
  • Connections per host
  • }; } } - if (!$disable_query && !$log_duration) { + if (!$disable_query && (!$log_duration || $enable_log_min_duration)) { print $fh qq{Slowest queries
  • Queries that took up the most time (N)
  • Most frequent queries (N)
  • Slowest queries (N)
    • }; } @@ -2123,7 +2133,7 @@ sub dump_as_html $d1 = ''; $d2 = ''; - if (!$disable_query && !$log_duration) { + if (!$disable_query && (!$log_duration || $enable_log_min_duration)) { # Select queries foreach my $tm (sort {$a <=> $b} keys %per_hour_info) { $tm =~ /(\d{4})(\d{2})(\d{2})/; @@ -2346,7 +2356,7 @@ qq{Wrote buffersAddedRemovedRecycledWrit } # INSERT/DELETE/UPDATE/SELECT repartition - if (!$disable_type && !$log_duration) { + if (!$disable_type && (!$log_duration || $enable_log_min_duration)) { print $fh qq{

    Queries by type ^

    @@ -2721,7 +2731,7 @@ qq{
    Wrote buffersAddedRemovedRecycledWrit } # Show top informations - if (!$disable_query && !$log_duration) { + if (!$disable_query && (!$log_duration || $enable_log_min_duration)) { print $fh qq{

    Slowest queries ^

    @@ -3441,14 +3451,15 @@ sub parse_query $t_duration = $1; $t_action = 'statement'; $prefix_vars{'t_query'} = 'No log_min_duration'; - } else { + } elsif (!$enable_log_min_duration) { if (exists $cur_info{$t_pid} && ($prefix_vars{'t_session_line'} != $cur_info{$t_pid}{session})) { &store_queries($t_pid); delete $cur_info{$t_pid}; } return; } - } else { + } + if (!$log_duration || $enable_log_min_duration) { if ($prefix_vars{'t_query'} =~ s/duration: ([0-9\.]+) ms (query|statement): //is) { $t_duration = $1; $t_action = $2;