From: Gilles Darold <gilles@darold.net>
Date: Fri, 5 Oct 2018 12:30:18 +0000 (+0200)
Subject: Fix missing replacement of bind parameters. Thanks to Bernhard J. M. Grun for the... 
X-Git-Tag: v10.2~27
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=14fef5ec9bb6be25a657b4199e0764374114a2b7;p=pgbadger

Fix missing replacement of bind parameters. Thanks to Bernhard J. M. Grun for the report.
---

diff --git a/pgbadger b/pgbadger
index b167d14..befc1e5 100755
--- a/pgbadger
+++ b/pgbadger
@@ -348,6 +348,8 @@ my $ssh_user = '';
 my $ssh_timeout = 10;
 my $ssh_options = "-o ConnectTimeout=$ssh_timeout -o PreferredAuthentications=hostbased,publickey";
 
+my $curl_command = 'curl -s ';
+
 # OBSOLETE, to be removed
 # List of regex that match fatal error message that do not
 # generate disconnection line in log. This is to prevent
@@ -576,7 +578,7 @@ if ($#ARGV >= 0) {
 			push(@log_files, $file);
 			$empty_files = 0;
 		} elsif ($file ne '-') {
-			if (!$remote_host) {
+			if (!$remote_host && $file !~ /^http[s]*:|ftp:/i) {
 				localdie("FATAL: logfile \"$file\" must exist!\n") if (not -f $file);
 				if (-z $file) {
 					print "WARNING: file $file is empty\n" if (!$quiet);
@@ -584,7 +586,7 @@ if ($#ARGV >= 0) {
 				}
 				push(@log_files, $file);
 				$empty_files = 0;
-			} else {
+			} elsif ($file !~ /^http[s]*:|ftp:/i) {
 				# Get files from remote host
 				&logmsg('DEBUG', "Looking for remote filename using command: $ssh_command \"ls $file\"");
 				my @rfiles = `$ssh_command "ls $file"`;
@@ -592,6 +594,9 @@ if ($#ARGV >= 0) {
 					push(@log_files, $f);
 				}
 				$empty_files = 0;
+			} else {
+				push(@log_files, $file);
+				$empty_files = 0;
 			}
 		} else {
 			if ($logfile_list) {
@@ -13192,23 +13197,29 @@ sub parse_query
 		delete $conn_received{$cur_info{$t_pid}{pid}};
 
 		# The query is complete but we are missing some debug/info/bind parameter logs
-		if ($is_log_level) {
+		#if ($is_log_level) {
+		if ($cur_info{$t_pid}{loglevel} eq 'LOG' && $prefix_vars{'t_loglevel'} eq 'DETAIL') {
 
 			# Apply bind parameters if any
-			if (($prefix_vars{'t_loglevel'} eq 'DETAIL') && ($prefix_vars{'t_query'} =~ /parameters: (.*)/)) {
+			if ($prefix_vars{'t_query'} =~ /parameters: (.*)/) {
 				$cur_info{$t_pid}{parameters} = "$1";
 				# go look at other params
 				return;
 			# replace the execute statements with the prepared query and set the parameters
-			} elsif (($prefix_vars{'t_loglevel'} eq 'DETAIL') && ($prefix_vars{'t_query'} =~ s/prepare: PREPARE\s+([^\s]+)\s+AS\s+(.*)//is)) {
+			} elsif ($prefix_vars{'t_query'} =~ s/prepare: PREPARE\s+([^\s]+)(\s*\([^\)]*\))?\s+AS\s*(.*)/[ EXECUTE $1 (PARAM) ]/is) {
 				my $q_name = $1;
-				my $real_query = $2;
+				my $real_query = $3;
 				if ($cur_info{$t_pid}{query} =~ /\b$q_name\b/) {
-					$cur_info{$t_pid}{query} =~ s/EXECUTE\s+$q_name(\s+)\(//is;
-					$cur_info{$t_pid}{parameters} = $cur_info{$t_pid}{query};
-					$cur_info{$t_pid}{parameters} =~ s/\)$//;
-					$cur_info{$t_pid}{query} = $real_query;
+					$cur_info{$t_pid}{query} =~ s/EXECUTE\s+$q_name\s*\((.*)\)[;]*$//is;
+					$cur_info{$t_pid}{parameters} = $1;
+					$prefix_vars{'t_query'} =~ s/\(PARAM\)/($cur_info{$t_pid}{parameters})/;
+					$cur_info{$t_pid}{query} = $prefix_vars{'t_query'} . ' - ' . $real_query;
 					$cur_info{$t_pid}{'bind'} = 1;
+					my @t_res = split(/[\s]*,[\s]*/, $cur_info{$t_pid}{parameters});
+					for (my $i = 0 ; $i <= $#t_res ; $i++) {
+						my $num = $i + 1;
+						$cur_info{$t_pid}{query} =~ s/\$$num\b/$t_res[$i]/s;
+					}
 				}
 				# go look at other params
 				return;
@@ -14116,10 +14127,11 @@ sub autodetect_format
 	my $nfound = 0;
 	my $nline  = 0;
 	my $fmt    = '';
+	my $http_download  = ($file =~ /^http[s]*:|ftp:/i) ? 1 : 0;
 	my %ident_name = ();
 	my $fltf;
 
-	if (!$remote_host) {
+	if (!$remote_host && !$http_download) {
 		if (open(my $in, '<', $file)) {
 			$fltf = <$in>;
 			close($in);
@@ -14133,7 +14145,7 @@ sub autodetect_format
 		$fmt = 'binary';
 	}
 	else { # try to detect syslogs, stderr, csv jsonlog or pgbouncer format
-		my ($tfile, $totalsize) = &get_log_file($file, $remote_host);
+		my ($tfile, $totalsize) = &get_log_file($file, $remote_host || $http_download);
 		if (defined $tfile) {
 			while (my $line = <$tfile>) {
 				chomp($line);
@@ -14514,6 +14526,7 @@ sub get_log_file
 
 	my $lfile = undef;
 	my $iscompressed = 1;
+	my $http_download  = ($logf =~ /^http[s]*:|ftp:/i) ? 1 : 0;
 
 	chomp($logf);
 
@@ -14521,11 +14534,16 @@ sub get_log_file
 	my $totalsize = 0;
 	if ( $journalctl_cmd && ($logf =~ m/\Q$journalctl_cmd\E/) ) {
 		$totalsize = 0;
-	} elsif (!$remote_host) {
+	} elsif (!$remote_host && !$http_download) {
 		$totalsize = (stat("$logf"))[7] || 0 if ($logf ne '-');
 	} elsif ($logf !~ /\.(gz|bz2|zip|xz)$/i) {
-		&logmsg('DEBUG', "Looking for file size using command: $ssh_command \"ls -l $logf\" | awk '{print \$5}'");
-		$totalsize = `$ssh_command "ls -l $logf" | awk '{print \$5}'`;
+		if ($http_download) {
+			&logmsg('DEBUG', "Looking for file size using command: $curl_command --head $logf | grep \"Content-Length:\" | awk '{print \$2}'");
+			$totalsize = `$curl_command --head $logf | grep "Content-Length:" | awk '{print \$2}'`;
+		} elsif ($remote_host) {
+			&logmsg('DEBUG', "Looking for file size using command: $ssh_command \"ls -l $logf\" | awk '{print \$5}'");
+			$totalsize = `$ssh_command "ls -l $logf" | awk '{print \$5}'`;
+		}
 		chomp($totalsize);
 		if ($totalsize eq '') {
 			localdie("FATAL: can't get size of remote file, please check what's going wrong with command: $ssh_command \"ls -l $logf\" | awk '{print \$5}'\n");
@@ -14546,16 +14564,14 @@ sub get_log_file
 		# For journalctl command we need to use a pipe as file handle
 		if (!$remote_host) {
 			open($lfile, '-|', $logf) || localdie("FATAL: cannot read output of command: $logf. $!\n");
+		} elsif (!$sample_only) {
+			&logmsg('DEBUG', "Retrieving log entries using command: $ssh_command \"$logf\" |");
+			# Open a pipe to remote journalctl program
+			open($lfile, '-|', "$ssh_command \"$logf\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"$logf\". $!\n");
 		} else {
-			if (!$sample_only) {
-				&logmsg('DEBUG', "Retrieving log entries using command: $ssh_command \"$logf\" |");
-				# Open a pipe to remote journalctl program
-				open($lfile, '-|', "$ssh_command \"$logf\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"$logf\". $!\n");
-			} else {
-				&logmsg('DEBUG', "Retrieving log entries using command: $ssh_command \"$logf -n 100\" |");
-				# Open a pipe to remote journalctl program
-				open($lfile, '-|', "$ssh_command \"$logf -n 100\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"$logf -n 100\". $!\n");
-			}
+			&logmsg('DEBUG', "Retrieving log entries using command: $ssh_command \"$logf -n 100\" |");
+			# Open a pipe to remote journalctl program
+			open($lfile, '-|', "$ssh_command \"$logf -n 100\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"$logf -n 100\". $!\n");
 		}
 		$iscompressed = 0;
 	} elsif ($logf !~ /\.(gz|bz2|zip|xz)$/i) {
@@ -14567,12 +14583,21 @@ sub get_log_file
 			}
 		} else {
 			if (!$sample_only) {
-				&logmsg('DEBUG', "Retrieving log entries using command: $ssh_command \" cat $logf\" |");
-				# Open a pipe to cat program
-				open($lfile, '-|', "$ssh_command \"cat $logf\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"cat $logf\". $!\n");
-			} else {
+				if (!$http_download) {
+					&logmsg('DEBUG', "Retrieving log entries using command: $ssh_command \" cat $logf\" |");
+					# Open a pipe to cat program
+					open($lfile, '-|', "$ssh_command \"cat $logf\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"cat $logf\". $!\n");
+				} else {
+					&logmsg('DEBUG', "Retrieving log entries using command: $curl_command \"$logf\" |");
+					# Open a pipe to GET program
+					open($lfile, '-|', "$curl_command \"$logf\"") || localdie("FATAL: cannot read from pipe to $curl_command \"$logf\". $!\n");
+				}
+			} elsif (!$http_download) {
 				# Open a pipe to cat program
 				open($lfile, '-|', "$ssh_command \"tail -n 100 $logf\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"tail -n 100 $logf\". $!\n");
+			} else {
+				# Open a pipe to GET program
+				open($lfile, '-|', "$curl_command --max-filesize 102400 \"$logf\"") || localdie("FATAL: cannot read from pipe to $curl_command --max-filesize 102400 \"$logf\". $!\n");
 			}
 		}
 		$totalsize = 0 if ($logf eq '-');
@@ -14590,19 +14615,28 @@ sub get_log_file
 			$uncompress = $xzcat;
 			$sample_cmd = 'xzgrep';
 		}
-		if (!$remote_host) {
+		if (!$remote_host && !$http_download) {
 			&logmsg('DEBUG', "Compressed log file, will use command: $uncompress \"$logf\"");
 			# Open a pipe to zcat program for compressed log
 			open($lfile, '-|', "$uncompress \"$logf\"") || localdie("FATAL: cannot read from pipe to $uncompress \"$logf\". $!\n");
 		} else {
 			if (!$sample_only) {
-				&logmsg('DEBUG', "Compressed log file, will use command: $ssh_command \"$uncompress $logf\"");
-				# Open a pipe to zcat program for compressed log
-				open($lfile, '-|', "$ssh_command \"$uncompress $logf\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"$uncompress $logf\". $!\n");
-			} else {
+				if (!$http_download) {
+					&logmsg('DEBUG', "Compressed log file, will use command: $ssh_command \"$uncompress $logf\"");
+					# Open a pipe to zcat program for compressed log
+					open($lfile, '-|', "$ssh_command \"$uncompress $logf\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"$uncompress $logf\". $!\n");
+				} else {
+					&logmsg('DEBUG', "Retrieving log entries using command: $curl_command \"$logf\" | $uncompress |");
+					# Open a pipe to GET program
+					open($lfile, '-|', "$curl_command \"$logf\" | $uncompress") || localdie("FATAL: cannot read from pipe to $curl_command \"$logf\". $!\n");
+				}
+			} elsif (!$http_download) {
 				&logmsg('DEBUG', "Compressed log file, will use command: $ssh_command \"$uncompress $logf\"");
 				# Open a pipe to zcat program for compressed log
 				open($lfile, '-|', "$ssh_command \"$sample_cmd -m 100 '[1234567890]' $logf\"") || localdie("FATAL: cannot read from pipe to $ssh_command \"$sample_cmd -m 100 '' $logf\". $!\n");
+			} else {
+				# Open a pipe to GET program
+				open($lfile, '-|', "$curl_command --max-filesize 102400 \"$logf\" | $uncompress") || localdie("FATAL: cannot read from pipe to $curl_command --max-filesize 102400 \"$logf\" | $uncompress . $!\n");
 			}
 		}
 
@@ -14621,7 +14655,7 @@ sub get_log_file
 			if (!$remote_host) {
 				&logmsg('DEBUG', "Looking for file size using command: $cmd_file_size");
 				$totalsize = `$cmd_file_size`;
-			} else {
+			} elsif (!$http_download) {
 				&logmsg('DEBUG', "Looking for remote file size using command: $ssh_command $cmd_file_size");
 				$totalsize = `$ssh_command $cmd_file_size`;
 			}