Allow pgbadger to parse remote log file using a passwordless ssh connection.

diff --git a/pgbadger b/pgbadger
index 542d05012cc422cede34df83b137fa9c54edcd35..a4395ad0fa99141bca46d0ea8a429e77f070580d 100644 (file)
--- a/pgbadger
+++ b/pgbadger
@@ -168,6 +168,15 @@ my $img_format  = 'png';
 my @log_files   = ();
 my %prefix_vars = ();
 
+my $remote_host = '';
+my $ssh_command = '';
+my $ssh_bin = 'ssh';
+my $ssh_identity = '';
+my $ssh_user = '';
+my $ssh_timeout = 10;
+my $ssh_options = "-o ConnectTimeout=$ssh_timeout -o PreferredAuthentications=hostbased,publickey";
+
+
 # Load the DATA  part of the script
 my @jscode = <DATA>;
 
@@ -185,6 +194,7 @@ $num_sep = ' ' if ($n =~ /,/);
 # Inform the parent that it should stop iterate on parsing other files
 sub stop_parsing
 {
+       print STDERR "DEBUG: Received interrupt signal";
        $interrupt = 1;
 }
 
@@ -192,6 +202,9 @@ sub stop_parsing
 sub wait_child
 {
         my $sig = shift;
+
+       $interrupt = 1;
+
         print STDERR "Received terminating signal ($sig).\n";
        if ($^O !~ /MSWin32|dos/i) {
                1 while wait != -1;
@@ -240,6 +253,7 @@ my $result = GetOptions(
        "p|prefix=s"               => \$log_line_prefix,
        "P|no-prettify!"           => \$noprettify,
        "q|quiet!"                 => \$quiet,
+       "r|remote-host=s"          => \$remote_host,
        "s|sample=i"               => \$sample,
        "S|select-only!"           => \$select_only,
        "t|top=i"                  => \$top,
@@ -271,6 +285,12 @@ my $result = GetOptions(
        "charset=s"                => \$charset,
        "csv-separator=s"          => \$csv_sep_char,
        "exclude-time=s"           => \@exclude_time,
+        'ssh-command=s'            => \$ssh_command,
+        'ssh-program=s'            => \$ssh_bin,
+        'ssh-identity=s'           => \$ssh_identity,
+        'ssh-option=s'             => \$ssh_options,
+        'ssh-user=s'               => \$ssh_user,
+        'ssh-timeout=i'            => \$ssh_timeout,
 );
 die "FATAL: use pgbadger --help\n" if (not $result);
 
@@ -283,10 +303,25 @@ if ($ver) {
 # Rewrite some command line arguments as lists
 &compute_arg_list();
 
+# If pgBadger must parse remote files set the ssh command
+if ($remote_host) {
+       # If no user defined ssh command
+       if (!$ssh_command) {
+               $ssh_command = $ssh_bin || 'ssh';
+               $ssh_command .= " -i $ssh_identity" if ($ssh_identity);
+               $ssh_command .= " $ssh_options" if ($ssh_options);
+               if ($ssh_user) {
+                       $ssh_command .= " $ssh_user\@$remote_host";
+               } else {
+                       $ssh_command .= " $remote_host";
+               }
+       }
+}
+
 # Log files to be parsed are passed as command line arguments
 if ($#ARGV >= 0) {
        foreach my $file (@ARGV) {
-               if ($file ne '-') {
+               if (($file ne '-') && !$remote_host) {
                        die "FATAL: logfile $file must exist!\n" if not -f $file;
                        if (-z $file) {
                                print "WARNING: file $file is empty\n";
@@ -323,7 +358,7 @@ for (my $i = 0 ; $i < 60 ; $i += $histo_avg_minutes) {
 }
 
 # Set error like log level regex
-my $parse_regex = qr/^(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|HINT|STATEMENT|CONTEXT)/;
+my $parse_regex      = qr/^(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|HINT|STATEMENT|CONTEXT)/;
 my $full_error_regex = qr/^(WARNING|ERROR|FATAL|PANIC|DETAIL|HINT|STATEMENT|CONTEXT)/;
 my $main_error_regex = qr/^(WARNING|ERROR|FATAL|PANIC)/;
 
@@ -334,7 +369,12 @@ my $orphan_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)(?:\s[^\s]+)?\s([^\
 my $orphan_stderr_line = '';
 
 # Set default format
-my $frmt = &autodetect_format($log_files[0]);
+my $frmt = '';
+if (!$remote_host) {
+       $frmt = &autodetect_format($log_files[0]);
+} elsif (!$format) {
+       die "FATAL: you must give a log file format (-f or --format) when using remote connection.\n\n";
+}
 $format ||= $frmt;
 
 if ($format eq 'syslog2') {
@@ -791,9 +831,14 @@ my $global_totalsize = 0;
 my @given_log_files = ( @log_files );
 
 # Verify that the file has not changed for incremental move
-if ( ($saved_last_line{current_pos} > 0) && ($#given_log_files == 0)) {
-       $saved_last_line{current_pos} = 0 if (&check_file_changed($given_log_files[0], $saved_last_line{datetime}));
-       $saved_last_line{current_pos}++ if ($saved_last_line{current_pos} > 0);
+if (!$remote_host) {
+       if ( ($saved_last_line{current_pos} > 0) && ($#given_log_files == 0)) {
+               $saved_last_line{current_pos} = 0 if (&check_file_changed($given_log_files[0], $saved_last_line{datetime}));
+               $saved_last_line{current_pos}++ if ($saved_last_line{current_pos} > 0);
+       }
+} else {
+       # Disable multi process when using ssh to parse remote log
+       $queue_size = 1;
 }
 
 # log files must be erased when loading stats from binary format
@@ -847,6 +892,7 @@ if ( ($queue_size > 1) || ($job_per_file > 1) ) {
                        }
                        sleep(1);
                }
+
                # Do not use split method with compressed files
                if ( ($queue_size > 1) && ($logfile !~ /\.(gz|bz2|zip)/i) ) {
                        # Create multiple processes to parse one log file by chunks of data
@@ -860,6 +906,7 @@ if ( ($queue_size > 1) || ($job_per_file > 1) ) {
                                        }
                                        sleep(1);
                                }
+                               die "FATAL: Abort signal received when processing to next chunk\n" if ($interrupt);
                                push(@tempfiles, [ tempfile('tmp_pgbadgerXXXX', SUFFIX => '.bin', DIR => $TMP_DIR, UNLINK => 1 ) ]);
                                spawn sub {
                                        &process_file($logfile, $tempfiles[-1]->[0], $chunks[$i], $chunks[$i+1]);
@@ -877,7 +924,8 @@ if ( ($queue_size > 1) || ($job_per_file > 1) ) {
                        $child_count++;
 
                }
-               last if ($interrupt);
+
+               die "FATAL: Abort signal received when processing next file\n" if ($interrupt);
        }
 
        my $minproc = 1;
@@ -1205,9 +1253,9 @@ if (!$incremental) {
 
 my $t2 = Benchmark->new;
 $td = timediff($t2, $t1);
-&logmsg('DEBUG', "building reports took:" . timestr($td));
+&logmsg('DEBUG', "building reports took: " . timestr($td));
 $td = timediff($t2, $t0);
-&logmsg('DEBUG', "the total execution time took:" . timestr($td));
+&logmsg('DEBUG', "the total execution time took: " . timestr($td));
 
 exit 0;
 
@@ -1268,6 +1316,8 @@ Options:
                              application name. See examples below.
     -P | --no-prettify     : disable SQL queries prettify formatter.
     -q | --quiet           : don't print anything to stdout, not even a progress bar.
+    -r | --remote-host ip  : set the host where to execute the cat command on remote
+                             logfile to parse localy the file.
     -s | --sample number   : number of query samples to store/display. Default: 3
     -S | --select-only     : only report SELECT queries.
     -t | --top number      : number of queries to store/display. Default: 20
@@ -1310,6 +1360,19 @@ Options:
     --exclude-appname name : exclude entries for the specified application name
                             from report. Example: "pg_dump".
 
+pgBadger is able to parse a remote log file using a passwordless ssh connection.
+Use the -r or --remote-host to set the host ip address or hostname. There's also
+some additional options to fully control the ssh connection.
+
+    --ssh-program ssh        path to the ssh program to use. Default: ssh.
+    --ssh-user username      connection login name. Default to running user.
+    --ssh-identity file      path to the identity file to use.
+    --ssh-timeout second     timeout to ssh connection failure. Default 10 seconds.
+    --ssh-options  options   list of -o options to use for the ssh connection. Options
+                             always used:
+                                 -o ConnectTimeout=\$ssh_timeout
+                                 -o PreferredAuthentications=hostbased,publickey
+
 Examples:
 
        pgbadger /var/log/postgresql.log
@@ -1462,7 +1525,11 @@ sub process_file
 
        &init_stats_vars() if ($tmpoutfile);
 
-       &logmsg('DEBUG', "Starting to parse log file: $logfile");
+       if (!$remote_host) {
+               &logmsg('DEBUG', "Starting to parse log file: $logfile");
+       } else {
+               &logmsg('DEBUG', "Starting to parse remote log file: $remote_host:$logfile");
+       }
 
        my $terminate = 0;
        local $SIG{INT} = sub { $terminate = 1 };
@@ -1485,7 +1552,11 @@ sub process_file
                $totalsize = $stop_offset - $start_offset;
        }
 
-       &logmsg('DEBUG', "Starting reading file $logfile...");
+       if (!$remote_host) {
+               &logmsg('DEBUG', "Starting reading file $logfile...");
+       } else {
+               &logmsg('DEBUG', "Starting reading file $remote_host:$logfile...");
+       }
 
        if ($format eq 'csv') {
 
@@ -10016,7 +10087,18 @@ sub get_log_file
        my $lfile = undef;
 
        # get file size
-       my $totalsize = (stat("$logf"))[7] || 0;
+       my $totalsize = 0;
+       if (!$remote_host) {
+               $totalsize = (stat("$logf"))[7] || 0;
+       } elsif ($logf !~ /\.(gz|bz2|zip|xz)/i) {
+               &logmsg('DEBUG', "Looking for remote file size using command: $ssh_command \"ls -l $logf\" | awk '{print \$5}'");
+               $totalsize = `$ssh_command "ls -l $logf" | awk '{print \$5}'`;
+               chomp($totalsize);
+               if (!$totalsize) {
+                       die "FATAL: can't get size of remote file, please check what's going wrong with command: $ssh_command \"ls -l $logf\" | awk '{print \$5}'\n";
+               }
+               &logmsg('DEBUG', "Remote file size: $totalsize");
+       }
        my $iscompressed = 1;
 
        # Open a file handle
@@ -10034,10 +10116,16 @@ sub get_log_file
                elsif (($logf =~ /\.xz/i) && ($zcat =~ /^$zcat_cmd$/)) {
                        $uncompress = $xzcat;
                }
-               &logmsg('DEBUG', "Compressed log file, will use command: $uncompress \"$logf\"");
+               if (!$remote_host) {
+                       &logmsg('DEBUG', "Compressed log file, will use command: $uncompress \"$logf\"");
+                       # Open a pipe to zcat program for compressed log
+                       open($lfile,"$uncompress \"$logf\" |") || die "FATAL: cannot read from pipe to $uncompress \"$logf\". $!\n";
+               } else {
+                       &logmsg('DEBUG', "Compressed log file, will use command: $ssh_command \"$uncompress $logf\"");
+                       # Open a pipe to zcat program for compressed log
+                       open($lfile,"$ssh_command \"$uncompress $logf\" |") || die "FATAL: cannot read from pipe to $ssh_command \"$uncompress $logf\". $!\n";
+               }
 
-               # Open a pipe to zcat program for compressed log
-               open($lfile,"$uncompress \"$logf\" |") || die "FATAL: cannot read from pipe to $uncompress \"$logf\". $!\n";
 
                # Real size of the file is unknown, try to find it
                # bz2 does not report real size
@@ -10050,7 +10138,13 @@ sub get_log_file
                                $cmd_file_size = $xz_uncompress_size;
                        }                       
                        $cmd_file_size =~ s/\%f/$logf/g;
-                       $totalsize = `$cmd_file_size`;
+                       if (!$remote_host) {
+                               &logmsg('DEBUG', "Looking for remote file size using command: $cmd_file_size");
+                               $totalsize = `$cmd_file_size`;
+                       } else {
+                               &logmsg('DEBUG', "Looking for remote file size using command: $ssh_command $cmd_file_size");
+                               $totalsize = `$ssh_command $cmd_file_size`;
+                       }
                        chomp($totalsize);
                }
                $queue_size = 0;