From: Darold <gilles@darold.net>
Date: Sat, 28 Jul 2012 18:41:31 +0000 (+0200)
Subject: Add -l | --last-parsed options to allow incremental run of pgbadger
X-Git-Tag: v3.2~178
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=dc566c040fc54d384b287e0d76b5dd89f294f759;p=pgbadger

Add -l | --last-parsed options to allow incremental run of pgbadger
---

diff --git a/pgbadger b/pgbadger
index f19682e..31473c2 100755
--- a/pgbadger
+++ b/pgbadger
@@ -74,6 +74,7 @@ my $disable_lock        = 0;
 my $disable_temporary   = 0;
 my $disable_checkpoint  = 0;
 my $avg_minutes         = 5;
+my $last_parsed         = '';
 
 my $NUMPROGRESS = 10000;
 my @DIMENSIONS  = (800, 300);
@@ -127,6 +128,7 @@ my $result = GetOptions(
 	"disable-lock!"       => \$disable_lock,
 	"disable-temporary!"  => \$disable_temporary,
 	"disable-checkpoint!" => \$disable_checkpoint,
+	"-l|last-parsed=s"  => \$last_parsed,
 );
 
 if ($ver) {
@@ -356,9 +358,23 @@ my %checkpoint_info = ();
 my @graph_values    = ();
 my %cur_info        = ();
 my $nlines          = 0;
+my %last_line       = ();
+my %saved_last_line = ();
 
 my $t0 = Benchmark->new;
 
+# Reading last line parsed
+if ($last_parsed && -e $last_parsed) {
+	if (open(IN, "$last_parsed")) {
+		my $line = <IN>;
+		close(IN);
+		($saved_last_line{datetime}, $saved_last_line{orig}) = split(/\t/, $line, 2);
+	} else {
+		die "FATAL: can't read last parsed line from $last_parsed, $!\n";
+	}
+}
+
+# Main loop reading log files
 foreach my $logfile (@log_files) {
 	&logmsg('DEBUG', "Starting to parse log file: $logfile");
 
@@ -431,6 +447,9 @@ foreach my $logfile (@log_files) {
 				last if ($to   && ($to < $cur_date));
 				$cur_pid = $8;
 
+				# Jump to the last line parsed if required
+				next if (!&check_incremental_position($cur_date, $line));
+
 				# Process the log line
 				&parse_query($tmp_year, $month_abbr{$1}, $day, $3, $4, $5, $6, $8, $9, $10, $11, $12);
 
@@ -477,6 +496,9 @@ foreach my $logfile (@log_files) {
 				last if ($to   && ($to < $cur_date));
 				$cur_pid = $8;
 
+				# Jump to the last line parsed if required
+				next if (!&check_incremental_position($cur_date, $line));
+
 				# Process the log line
 				&parse_query($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12);
 
@@ -505,13 +527,16 @@ foreach my $logfile (@log_files) {
 
 				# Extract the date
 				$cols[0] =~ m/(\d+)-(\d+)-(\d+)\s+(\d+):(\d+):(\d+)/;
-				my @date = ($1, $2, $3, $4, $5, $6);
+				my @date = ($1,$2,$3,$4,$5,$6);
+				my $cur_date = join('', @date);
 
 				# Skip unwanted lines
-				my $cur_date = join('', @date);
 				next if ($from && ($from > $cur_date));
 				last if ($to   && ($to < $cur_date));
 
+				# Jump to the last line parsed if required
+				next if (!&check_incremental_position($cur_date, $line));
+
 				# Process the log line
 				&parse_query(
 					@date,
@@ -551,6 +576,16 @@ foreach my $logfile (@log_files) {
 
 }
 
+# Save last line parsed
+if ($last_parsed && scalar keys %last_line) {
+	if (open(OUT, ">$last_parsed")) {
+		print OUT "$last_line{datetime}\t$last_line{orig}\n";
+		close(OUT);
+	} else {
+		&logmsg('ERROR', "can't save last parsed line into $last_parsed, $!");
+	}
+}
+
 my $t1 = Benchmark->new;
 my $td = timediff($t1, $t0);
 &logmsg('DEBUG', "the log statistics gathering tooks:" . timestr($td));
@@ -642,6 +677,10 @@ Options:
     --disable-lock         : do not generate lock report.
     --disable-temporary    : do not generate temporary report.
     --disable-checkpoint   : do not generate checkpoint report.
+    -l | --last-parsed file: allow incremental log parsing by registering the
+			     last datetime and line parsed. Useful if you want
+			     to watch errors since last run or if you want one
+			     report per day with a log rotated each week.
 
 Examples:
 
@@ -663,6 +702,26 @@ This supposes that your log file and HTML report are also rotated every weeks.
 	exit 0;
 }
 
+sub check_incremental_position
+{
+	my ($cur_date, $line) = @_;
+
+	if ($last_parsed) {
+		if ($saved_last_line{datetime}) {
+			if ($cur_date < $saved_last_line{datetime}) {
+				return 0;
+			} elsif (!$last_line{datetime} && ($cur_date == $saved_last_line{datetime})) {
+				return 0 if ($line ne $saved_last_line{orig});
+			}
+		}
+		$last_line{datetime} = $cur_date;
+		$last_line{orig} = $line;
+	}
+
+	return 1;
+}
+
+
 # Display message following the log level
 sub logmsg
 {