my $logfile_list = '';
my $enable_checksum = 0;
my $timezone = 0;
+my $rebuild = 0;
+
my $NUMPROGRESS = 10000;
my @DIMENSIONS = (800, 300);
'enable-checksum!' => \$enable_checksum,
'journalctl=s' => \$journalctl_cmd,
'pid-dir=s' => \$PID_DIR,
+ 'rebuild!' => \$rebuild,
);
die "FATAL: use pgbadger --help\n" if (not $result);
}
# Logfile is a mandatory parameter when journalctl command is not set.
-if ( ($#log_files < 0) && !$journalctl_cmd) {
+if ( !$rebuild && ($#log_files < 0) && !$journalctl_cmd) {
print STDERR "FATAL: you must give a log file as command line parameter.\n\n";
&usage();
}
# Set default format, with multiple files format will be autodetected each time.
# This default format will be used when the autodetection fail.
my $frmt = '';
-if (!$remote_host) {
- if ($journalctl_cmd) {
- $frmt = 'syslog2';
- } else {
- $frmt = &autodetect_format($log_files[0]);
+if (!$rebuild) {
+ if (!$remote_host) {
+ if ($journalctl_cmd) {
+ $frmt = 'syslog2';
+ } else {
+ $frmt = &autodetect_format($log_files[0]);
+ }
+ } elsif (!$format) {
+ if ($journalctl_cmd) {
+ $frmt = 'syslog2';
+ } else {
+ localdie("FATAL: you must give a log file format (-f or --format) when using remote connection.\n\n");
+ }
}
-} elsif (!$format) {
- if ($journalctl_cmd) {
- $frmt = 'syslog2';
- } else {
- localdie("FATAL: you must give a log file format (-f or --format) when using remote connection.\n\n");
+} else {
+ if (!$incremental) {
+ print STDERR "WARNING: --rebuild require incremental mode, activating it.\n"
}
+ $incremental = 1;
}
$format ||= $frmt;
my @top_cancelled_info = ();
my %drawn_graphs = ();
+# Global output filehandle
+my $fh = undef;
+
my $t0 = Benchmark->new;
# Write resources files from __DATA__ section if they have not been already copied
# Set default output format
$extension = 'binary';
+ if ($rebuild) {
+
+ # Look for directory where report must be generated again
+ my @build_directories = ();
+
+ # Find directories that shoud be rebuilt
+ unless(opendir(DIR, "$outdir")) {
+ localdie("Error: can't opendir $outdir: $!");
+ }
+ my @dyears = grep { $_ =~ /^\d+$/ } readdir(DIR);
+ closedir DIR;
+ foreach my $y (sort { $a <=> $b } @dyears) {
+ unless(opendir(DIR, "$outdir/$y")) {
+ localdie("Error: can't opendir $outdir/$y: $!");
+ }
+ my @dmonths = grep { $_ =~ /^\d+$/ } readdir(DIR);
+ closedir DIR;
+ foreach my $m (sort { $a <=> $b } @dmonths) {
+ unless(opendir(DIR, "$outdir/$y/$m")) {
+ localdie("Error: can't opendir $outdir/$y/$m: $!");
+ }
+ my @ddays = grep { $_ =~ /^\d+$/ } readdir(DIR);
+ closedir DIR;
+ foreach my $d (sort { $a <=> $b } @ddays) {
+ unless(opendir(DIR, "$outdir/$y/$m/$d")) {
+ localdie("Error: can't opendir $outdir/$y/$m/$d: $!");
+ }
+ my @binfiles = grep { $_ =~ /\.bin$/ } readdir(DIR);
+ closedir DIR;
+ push(@build_directories, "$y-$m-$d") if ($#binfiles >= 0);
+ }
+ }
+ }
+
+ &build_incremental_reports(@build_directories);
+
+ my $t2 = Benchmark->new;
+ my $td = timediff($t2, $t0);
+ &logmsg('DEBUG', "rebuilding reports took: " . timestr($td));
+
+ # Remove pidfile
+ unlink("$PID_DIR/pgbadger.pid");
+
+ exit 0;
+ }
+
} else {
# Extra files for resources are not allowed without incremental mode
my $wn = &get_week_number($last_year, $last_month, $last_day);
# Get the days of the current week where binary files must be preserved
my @wdays = &get_wdays_per_month($wn - 1, "$last_year-$last_month");
+
# Find obsolete dir days that shoud be cleaned
unless(opendir(DIR, "$outdir")) {
localdie("Error: can't opendir $outdir: $!");
my $td = timediff($t1, $t0);
&logmsg('DEBUG', "the log statistics gathering took:" . timestr($td));
-# Global output filehandle
-my $fh = undef;
if (!$incremental && ($#given_log_files >= 0) ) {
} elsif (!$incremental || !$noreport) {
- # Build a report per day
- my %weeks_directories = ();
+ # Look for directory where report must be generated
my @build_directories = ();
if (open(IN, "$last_parsed.tmp")) {
while (my $l = <IN>) {
&logmsg('WARNING', "can't read file $last_parsed.tmp, $!");
&logmsg('HINT', "maybe there's no new entries in your log since last run.");
}
- foreach $incr_date (sort @build_directories) {
-
- $last_incr_date = $incr_date;
-
- # Set the path to binary files
- my $bpath = $incr_date;
- $bpath =~ s/\-/\//g;
- $incr_date =~ /^(\d+)-(\d+)\-(\d+)$/;
-
- # Get the week number following the date
- my $wn = &get_week_number($1, $2, $3);
- $weeks_directories{$wn} = "$1-$2" if (!exists $weeks_directories{$wn});
-
- # First clear previous stored statistics
- &init_stats_vars();
-
- # Load all data gathered by all the different processes
- unless(opendir(DIR, "$outdir/$bpath")) {
- localdie("Error: can't opendir $outdir/$bpath: $!");
- }
- my @mfiles = grep { !/^\./ && ($_ =~ /\.bin$/) } readdir(DIR);
- closedir DIR;
- foreach my $f (@mfiles) {
- my $fht = new IO::File;
- $fht->open("< $outdir/$bpath/$f") or localdie("FATAL: can't open file $outdir/$bpath/$f, $!\n");
- &load_stats($fht);
- $fht->close();
- }
-
- &logmsg('LOG', "Ok, generating HTML daily report into $outdir/$bpath/...");
-
- $fh = new IO::File ">$outdir/$bpath/$outfile";
- if (not defined $fh) {
- localdie("FATAL: can't write to $outdir/$bpath/$outfile, $!\n");
- }
- # Create instance to prettify SQL query
- if (!$noprettify) {
- $sql_prettified = SQL::Beautify->new(keywords => \@beautify_pg_keywords);
- }
- &dump_as_html('../../..');
- $fh->close;
- }
-
- # Build a report per week
- foreach my $wn (sort { $a <=> $b } keys %weeks_directories) {
- &init_stats_vars();
-
- # Get all days of the current week
- my @wdays = &get_wdays_per_month($wn - 1, $weeks_directories{$wn});
- my $wdir = '';
-
- # Load data per day
- foreach $incr_date (@wdays) {
- my $bpath = $incr_date;
- $bpath =~ s/\-/\//g;
- $incr_date =~ /^(\d+)\-(\d+)\-(\d+)$/;
- $wdir = "$1/week-$wn";
-
- # Load all data gathered by all the differents processes
- if (-e "$outdir/$bpath") {
- unless(opendir(DIR, "$outdir/$bpath")) {
- localdie("Error: can't opendir $outdir/$bpath: $!");
- }
- my @mfiles = grep { !/^\./ && ($_ =~ /\.bin$/) } readdir(DIR);
- closedir DIR;
- foreach my $f (@mfiles) {
- my $fht = new IO::File;
- $fht->open("< $outdir/$bpath/$f") or localdie("FATAL: can't open file $outdir/$bpath/$f, $!\n");
- &load_stats($fht);
- $fht->close();
- }
- }
- }
-
- &logmsg('LOG', "Ok, generating HTML weekly report into $outdir/$wdir/...");
- if (!-d "$outdir/$wdir") {
- mkdir("$outdir/$wdir");
- }
- $fh = new IO::File ">$outdir/$wdir/$outfile";
- if (not defined $fh) {
- localdie("FATAL: can't write to $outdir/$wdir/$outfile, $!\n");
- }
- # Create instance to prettify SQL query
- if (!$noprettify) {
- $sql_prettified = SQL::Beautify->new(keywords => \@beautify_pg_keywords);
- }
- &dump_as_html('../..');
- $fh->close;
-
- }
-
- &logmsg('LOG', "Ok, generating global index to access incremental reports...");
-
- $fh = new IO::File ">$outdir/index.html";
- if (not defined $fh) {
- localdie("FATAL: can't write to $outdir/index.html, $!\n");
- }
- my $date = localtime(time);
- my @tmpjscode = @jscode;
- map { s/EDIT_URI/\./; } @tmpjscode;
- my $local_title = 'Global Index on incremental reports';
- if ($report_title) {
- $local_title = 'Global Index - ' . $report_title;
- }
- print $fh qq{<!DOCTYPE html>
-<html lang="en">
-<head>
-<title>pgBadger :: $local_title</title>
-<meta http-equiv="Content-Type" content="text/html; charset=$charset" />
-<meta name="robots" content="noindex,nofollow">
-<meta http-equiv="Expires" content="$date">
-<meta http-equiv="Generator" content="pgBadger v$VERSION">
-<meta http-equiv="Date" content="$date">
-<link rel="shortcut icon" href="$pgbadger_ico" />
-<meta name="viewport" content="user-scalable=no, initial-scale = 1, minimum-scale = 1, maximum-scale = 1, width=device-width">
-@tmpjscode
-</head>
-
-<body>
-
-<nav class="navbar navbar-inverse navbar-fixed-top">
- <div class="navbar-inner">
- <div class="container-fluid">
- <a data-placement="bottom" rel="tooltip" data-original-title="PostgreSQL Log Analyzer" href="" id="pgbadger-brand" class="brand">$pgbadger_logo $local_title</a>
- </div>
- </div>
-</nav>
-
-<div id="top"><br /><br /></div>
-<div class="container" id="main-container">
-
-<script type="text/javascript">
-\$(function () {
- \$(".pgb-popover a").popover({
- placement : 'bottom',
- html : true,
- });
-});
-</script>
-<style type="text/css">
-.btn-primary {
- font-size: 2.0em;
- font-weight: bold;
- height: 60px;
- width: 184px;
-}
-</style>
-
-};
- # get year directories
- unless(opendir(DIR, "$outdir")) {
- localdie("Error: can't opendir $outdir: $!");
- }
- my @dyears = grep { !/^\./ && /^\d{4}$/ } readdir(DIR);
- closedir DIR;
- foreach my $y (sort { $b <=> $a } @dyears) {
- print $fh qq{
-<h1>Year $y</h1>
- <table class="table table-condensed"><tr>
-};
- # foreach year directory look for week directories
- unless(opendir(DIR, "$outdir/$y")) {
- localdie("Error: can't opendir $outdir/$y: $!");
- }
-
- my @ymonths = grep { /^\d{2}$/ } readdir(DIR);
- closedir DIR;
- my $i = 1;
- foreach my $m (sort {$a <=> $b } @ymonths) {
- print $fh "<td><span class=\"span3\">", &get_calendar($y, $m), "</span></td>\n";
- print $fh "</tr>\n<tr>\n" if ( ($i%4) == 0 );
- $i++;
- }
- print $fh qq{
- </tr></table>
-};
- }
- print $fh qq{
-</div>
-
-<footer>
-<div class="">
- <small class="pull-right">Report generated by <a href="$project_url" target="_new">pgBadger $VERSION.</a></small>
-</div>
-</footer>
-
-<div id="littleToc">
- <div id="littleTocTitle"><a href="#top"> ^ </a></div>
-</div>
-
-</body>
-</html>
-};
-
-
-
- $fh->close;
-
+ &build_incremental_reports(@build_directories);
}
my $t2 = Benchmark->new;
--pid-dir dirpath : set the path of the directory where the pid file
will be written to be able to run two pgbadger at
the same time.
+ --rebuild : used to rebuild all html reports in incremental
+ output directories where there is binary data files.
pgBadger is able to parse a remote log file using a passwordless ssh connection.
Use the -r or --remote-host to set the host ip address or hostname. There's also
You can also parse journalctl output just as if it was a log file:
- perl pgbadger --journalctl 'journalctl -u postgresql-9.5'
+ pgbadger --journalctl 'journalctl -u postgresql-9.5'
or worst, call it from a remote host:
- perl pgbadger -r 192.168.1.159 --journalctl 'journalctl -u postgresql-9.5'
+ pgbadger -r 192.168.1.159 --journalctl 'journalctl -u postgresql-9.5'
you don't need to specify any log file at command line, but if you have other
PostgreSQL log file to parse, you can add them as usual.
+To rebuild all incremantal html reports after, proceed as follow:
+
+ rm /path/to/reports/*.js
+ rm /path/to/reports/*.css
+ pgbadger -X -I -O /path/to/reports/ --rebuild
+
+it will also update all ressources file (JS and CSS).
+
};
exit 0;
}
+sub build_incremental_reports
+{
+ my @build_directories = @_;
+
+ my %weeks_directories = ();
+ foreach $incr_date (sort @build_directories) {
+
+ $last_incr_date = $incr_date;
+
+ # Set the path to binary files
+ my $bpath = $incr_date;
+ $bpath =~ s/\-/\//g;
+ $incr_date =~ /^(\d+)-(\d+)\-(\d+)$/;
+
+ # Get the week number following the date
+ my $wn = &get_week_number($1, $2, $3);
+ $weeks_directories{$wn} = "$1-$2" if ($rebuild || !exists $weeks_directories{$wn});
+
+ # First clear previous stored statistics
+ &init_stats_vars();
+
+ # Load all data gathered by all the different processes
+ unless(opendir(DIR, "$outdir/$bpath")) {
+ localdie("Error: can't opendir $outdir/$bpath: $!");
+ }
+ my @mfiles = grep { !/^\./ && ($_ =~ /\.bin$/) } readdir(DIR);
+ closedir DIR;
+ foreach my $f (@mfiles) {
+ my $fht = new IO::File;
+ $fht->open("< $outdir/$bpath/$f") or localdie("FATAL: can't open file $outdir/$bpath/$f, $!\n");
+ &load_stats($fht);
+ $fht->close();
+ }
+
+ &logmsg('LOG', "Ok, generating HTML daily report into $outdir/$bpath/...");
+
+ $fh = new IO::File ">$outdir/$bpath/$outfile";
+ if (not defined $fh) {
+ localdie("FATAL: can't write to $outdir/$bpath/$outfile, $!\n");
+ }
+ # Create instance to prettify SQL query
+ if (!$noprettify) {
+ $sql_prettified = SQL::Beautify->new(keywords => \@beautify_pg_keywords);
+ }
+ &dump_as_html('../../..');
+ $fh->close;
+ }
+
+ # Build a report per week
+ foreach my $wn (sort { $a <=> $b } keys %weeks_directories) {
+ &init_stats_vars();
+
+ # Get all days of the current week
+ my @wdays = &get_wdays_per_month($wn - 1, $weeks_directories{$wn});
+ my $wdir = '';
+
+ # Load data per day
+ foreach $incr_date (@wdays) {
+ my $bpath = $incr_date;
+ $bpath =~ s/\-/\//g;
+ $incr_date =~ /^(\d+)\-(\d+)\-(\d+)$/;
+ $wdir = "$1/week-$wn";
+
+ # Load all data gathered by all the differents processes
+ if (-e "$outdir/$bpath") {
+ unless(opendir(DIR, "$outdir/$bpath")) {
+ localdie("Error: can't opendir $outdir/$bpath: $!");
+ }
+ my @mfiles = grep { !/^\./ && ($_ =~ /\.bin$/) } readdir(DIR);
+ closedir DIR;
+ foreach my $f (@mfiles) {
+ my $fht = new IO::File;
+ $fht->open("< $outdir/$bpath/$f") or localdie("FATAL: can't open file $outdir/$bpath/$f, $!\n");
+ &load_stats($fht);
+ $fht->close();
+ }
+ }
+ }
+
+ &logmsg('LOG', "Ok, generating HTML weekly report into $outdir/$wdir/...");
+ if (!-d "$outdir/$wdir") {
+ mkdir("$outdir/$wdir");
+ }
+ $fh = new IO::File ">$outdir/$wdir/$outfile";
+ if (not defined $fh) {
+ localdie("FATAL: can't write to $outdir/$wdir/$outfile, $!\n");
+ }
+ # Create instance to prettify SQL query
+ if (!$noprettify) {
+ $sql_prettified = SQL::Beautify->new(keywords => \@beautify_pg_keywords);
+ }
+ &dump_as_html('../..');
+ $fh->close;
+
+ }
+
+ &logmsg('LOG', "Ok, generating global index to access incremental reports...");
+
+ $fh = new IO::File ">$outdir/index.html";
+ if (not defined $fh) {
+ localdie("FATAL: can't write to $outdir/index.html, $!\n");
+ }
+ my $date = localtime(time);
+ my @tmpjscode = @jscode;
+ map { s/EDIT_URI/\./; } @tmpjscode;
+ my $local_title = 'Global Index on incremental reports';
+ if ($report_title) {
+ $local_title = 'Global Index - ' . $report_title;
+ }
+ print $fh qq{<!DOCTYPE html>
+<html lang="en">
+<head>
+<title>pgBadger :: $local_title</title>
+<meta http-equiv="Content-Type" content="text/html; charset=$charset" />
+<meta name="robots" content="noindex,nofollow">
+<meta http-equiv="Expires" content="$date">
+<meta http-equiv="Generator" content="pgBadger v$VERSION">
+<meta http-equiv="Date" content="$date">
+<link rel="shortcut icon" href="$pgbadger_ico" />
+<meta name="viewport" content="user-scalable=no, initial-scale = 1, minimum-scale = 1, maximum-scale = 1, width=device-width">
+@tmpjscode
+</head>
+
+<body>
+
+<nav class="navbar navbar-inverse navbar-fixed-top">
+ <div class="navbar-inner">
+ <div class="container-fluid">
+ <a data-placement="bottom" rel="tooltip" data-original-title="PostgreSQL Log Analyzer" href="" id="pgbadger-brand" class="brand">$pgbadger_logo $local_title</a>
+ </div>
+ </div>
+</nav>
+
+<div id="top"><br /><br /></div>
+<div class="container" id="main-container">
+
+<script type="text/javascript">
+\$(function () {
+ \$(".pgb-popover a").popover({
+ placement : 'bottom',
+ html : true,
+ });
+});
+</script>
+<style type="text/css">
+.btn-primary {
+ font-size: 2.0em;
+ font-weight: bold;
+ height: 60px;
+ width: 184px;
+}
+</style>
+
+};
+ # get year directories
+ unless(opendir(DIR, "$outdir")) {
+ localdie("Error: can't opendir $outdir: $!");
+ }
+ my @dyears = grep { !/^\./ && /^\d{4}$/ } readdir(DIR);
+ closedir DIR;
+ foreach my $y (sort { $b <=> $a } @dyears) {
+ print $fh qq{
+<h1>Year $y</h1>
+ <table class="table table-condensed"><tr>
+};
+ # foreach year directory look for week directories
+ unless(opendir(DIR, "$outdir/$y")) {
+ localdie("Error: can't opendir $outdir/$y: $!");
+ }
+
+ my @ymonths = grep { /^\d{2}$/ } readdir(DIR);
+ closedir DIR;
+ my $i = 1;
+ foreach my $m (sort {$a <=> $b } @ymonths) {
+ print $fh "<td><span class=\"span3\">", &get_calendar($y, $m), "</span></td>\n";
+ print $fh "</tr>\n<tr>\n" if ( ($i%4) == 0 );
+ $i++;
+ }
+ print $fh qq{
+ </tr></table>
+};
+ }
+ print $fh qq{
+</div>
+
+<footer>
+<div class="">
+ <small class="pull-right">Report generated by <a href="$project_url" target="_new">pgBadger $VERSION.</a></small>
+</div>
+</footer>
+
+<div id="littleToc">
+ <div id="littleTocTitle"><a href="#top"> ^ </a></div>
+</div>
+
+</body>
+</html>
+};
+ $fh->close;
+}
+
sub cleanup_directory
{
my ($dir, $remove_dir) = @_;
my $curdate = localtime(time);
my $fmt_nlines = &comma_numbers($nlines);
+ my $t3 = Benchmark->new;
+ my $td = timediff($t3, $t0);
my $total_time = timestr($td);
$total_time =~ s/^([\.0-9]+) wallclock.*/$1/;
$total_time = &convert_time($total_time * 1000);
projects / pgbadger / commitdiff