Fix unknown line format error for multiline log while incremental analysis over ssh...

diff --git a/pgbadger b/pgbadger
index 7e295d28faa7ab7ceda3c7e55c83f1fa2cd9abc1..30c8bf686c1b91303273d73d90ed227182ac65dc 100755 (executable)
--- a/pgbadger
+++ b/pgbadger
@@ -2528,7 +2528,7 @@ sub process_file
                                &logmsg('DEBUG', "Reverting start offset $start_offset to 0 for file $logfile, stoppping offset is " . ($stop_offset || $totalsize));
                                $start_offset = 0;
                        } else {
-                               &logmsg('DEBUG', "This might not be parsed: $logfile, $msg");
+                               &logmsg('DEBUG', "This log might not be parsed: $logfile, $msg");
                        }
                        $cursize = $start_offset;
                }
@@ -2565,7 +2565,7 @@ sub process_file
                if ($#exclude_line >= 0) {
                        $has_exclusion = 1;
                }
-               &logmsg('DEBUG', "Start parsing at offset $start_offset of file $logfile to " . ($stop_offset || $totalsize));
+               &logmsg('DEBUG', "Start parsing pgbouncer log at offset $start_offset of file $logfile to " . ($stop_offset || $totalsize));
                if ($start_offset) {
                        # Move to the starting offset position in file
                        $lfile->seek($start_offset, 0);
@@ -2797,7 +2797,8 @@ sub process_file
 
                &load_stats($lfile);
        }
-       else { # Format is not CSV.
+       # Format is not CSV and in incremental mode we are not at end of the file
+       else {
 
                my $time_pattern = qr/(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})/;
                my $cur_pid = '';
@@ -2807,7 +2808,7 @@ sub process_file
                if ($#exclude_line >= 0) {
                        $has_exclusion = 1;
                }
-               &logmsg('DEBUG', "Start parsing at offset $start_offset of file $logfile to " . ($stop_offset || $totalsize));
+               &logmsg('DEBUG', "Start parsing postgresql log at offset $start_offset of file $logfile to " . ($stop_offset || $totalsize));
                if ($start_offset) {
                        # Move to the starting offset position in file
                        $lfile->seek($start_offset, 0);
@@ -2851,6 +2852,9 @@ sub process_file
                                # Jump to the last line parsed if required
                                next if (!&check_incremental_position($fmt, $prefix_vars{'t_timestamp'}, $line));
 
+                               # We have reach previous incremental position (or we not in increment mode)
+                               $goon = 1;
+
                                # Store the current timestamp of the log line
                                &store_current_timestamp($prefix_vars{'t_timestamp'});
 
@@ -2930,7 +2934,9 @@ sub process_file
 
                                        # Jump to the last line parsed if required
                                        next if (!&check_incremental_position($fmt, $prefix_vars{'t_timestamp'}, $line));
+                                       # We have reach previous incremental position (or we not in increment mode)
                                        $goon = 1;
+
                                        $prefix_vars{'t_client'} = _gethostbyaddr($prefix_vars{'t_client'}) if ($dns_resolv);
 
                                        # Store the current timestamp of the log line
@@ -3048,6 +3054,9 @@ sub process_file
 
                                        # Jump to the last line parsed if required
                                        next if (!&check_incremental_position($fmt, $prefix_vars{'t_timestamp'}, $line));
+                                       # We have reach previous incremental position (or we not in increment mode)
+                                       $goon = 1;
+
                                        $prefix_vars{'t_client'} = _gethostbyaddr($prefix_vars{'t_client'}) if ($dns_resolv);
 
                                        # Store the current timestamp of the log line
@@ -3082,7 +3091,7 @@ sub process_file
                                        $cur_pid = $prefix_vars{'t_pid'};
 
                                # Collect additional query information
-                               } elsif ($cur_pid) {
+                               } elsif ($goon && $cur_pid) {
 
                                        # Some log line may be written by applications
                                        next if ($line =~ /\bLOG:  /);
@@ -3090,16 +3099,16 @@ sub process_file
                                        # Parse orphan lines to append information to the right place
                                        &parse_orphan_line($cur_pid, $line);
 
-                               } else {
+                               } elsif ($goon) {
 
                                        # unknown format
-                                       &logmsg('DEBUG', "Unknown line format: $line");
+                                       &logmsg('DEBUG', "Unknown $fmt line format: $line");
                                }
                        }
                        last if (($stop_offset > 0) && ($current_offset >= $stop_offset));
                }
 
-               if ($last_parsed) {
+               if ($goon && $last_parsed) {
                        $last_line{current_pos} = $current_offset;
                }
 
@@ -3416,14 +3425,21 @@ sub check_file_changed
                if ($saved_pos < $totalsize) {
                        $lfile->seek($saved_pos || 0, 0);
                        $startoffset = $saved_pos || 0;
+               # Case of file with same size
+               } elsif ($saved_pos == $totalsize) {
+                       if ($saved_pos > 8192) {
+                               $lfile->seek($saved_pos - 8192, 0);
+                               $startoffset = $saved_pos - 8192;
+                       } else {
+                               $lfile->seek(0, 0);
+                               $startoffset = 0;
+                       }
                }
        }
 
        my $more_lines = 0;
        while (my $line = <$lfile>) {
 
-               $more_lines++;
-
                $line =~ s/\r//;
 
                if ($fmt =~ /syslog/) {
@@ -3486,6 +3502,7 @@ sub check_file_changed
                        close($lfile);
                        return (0, "timestamp $prefix_vars{'t_timestamp'} read at offset $startoffset is lower than saved timestamp: $saved_date");
                } else {
+                       $more_lines++;
                        last;
                }