Fix parser on syslog with level info after the timestamp like <local0.info>. Thanks...

diff --git a/README b/README
index 6f483b15145b64fbb804ede0d62df141a917d44a..b8ab8d8535d5bedbf0678672968f021024c77ac2 100644 (file)
--- a/README
+++ b/README
@@ -23,6 +23,8 @@ SYNOPSIS
         -e | --end datetime    : end date/time for the data to be parsed in log.
         -f | --format logtype  : possible values: syslog,stderr,csv. Default: stderr.
                                  There's also syslog-ng for traces with [ID * local2.info]
+                                 and syslog-ll with syslog for line with <local0.info>.
+                                 Use them only if pgBager is unable to autodetect log format.
         -G | --nograph         : disable graphs on HTML output. Enable by default.
         -h | --help            : show this message and exit.
         -i | --ident name      : programname used as syslog ident. Default: postgres

diff --git a/doc/pgBadger.pod b/doc/pgBadger.pod
index 79708cd7b55858a8477553e71570afa78a474f53..484c693f1f19a404215505a1e29f52b024691964 100644 (file)
--- a/doc/pgBadger.pod
+++ b/doc/pgBadger.pod
@@ -25,6 +25,8 @@ Options:
     -e | --end datetime    : end date/time for the data to be parsed in log.
     -f | --format logtype  : possible values: syslog,stderr,csv. Default: stderr.
                              There's also syslog-ng for traces with [ID * local2.info]
+                             and syslog-ll with syslog for line with <local0.info>.
+                             Use them only if pgBager is unable to autodetect log format.
     -G | --nograph         : disable graphs on HTML output. Enable by default.
     -h | --help            : show this message and exit.
     -i | --ident name      : programname used as syslog ident. Default: postgres

diff --git a/pgbadger b/pgbadger
index bc643137d572dd814387b8970bdf646bbe7900df..8309af4bd290367a650286461a240240a8750ec6 100755 (executable)
--- a/pgbadger
+++ b/pgbadger
@@ -332,8 +332,8 @@ if ($#include_query >= 0) {
        }
 }
 
-my $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*)/;
-my $orphan_syslog_line = qr/^...\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\[]+\[\d+\]:/;
+my $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*)/;
+my $orphan_syslog_line = qr/^...\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\s\[]+\[\d+\]:/;
 my $orphan_stderr_line = qr/[^']*\d+-\d+-\d+\s\d+:\d+:\d+[\.\d]*\s[^\s]+[^']*/;
 
 # Compile custom log line prefie prefix
@@ -342,13 +342,20 @@ if ($log_line_prefix) {
        @prefix_params = &build_log_line_prefix_regex();
        &check_regex($log_line_prefix, '--prefix');
        if ($format eq 'syslog') {
-               $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)';
+               $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)';
+               $compiled_prefix = qr/$log_line_prefix/;
+               unshift(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_session_line');
+               push(@prefix_params, 't_loglevel', 't_query');
+       } elsif ($format eq 'syslog-ll') {
+               $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s[^\s]+\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)';
+               $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s[^\s]+\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*)/;
+               $orphan_syslog_line = qr/^...\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\s]+\s[^\s\[]+\[\d+\]:/;
                $compiled_prefix = qr/$log_line_prefix/;
                unshift(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_session_line');
                push(@prefix_params, 't_loglevel', 't_query');
        } elsif ($format eq 'syslog-ng') {
-               $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:(.*?)\[(\d+)\-\d+\]\s*(.*)/;
-               $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:(.*?)\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)';
+               $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:(.*?)\[(\d+)\-\d+\]\s*(.*)/;
+               $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:(.*?)\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)';
                $compiled_prefix = qr/$log_line_prefix/;
                unshift(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_id', 't_session_line');
        } elsif ($format eq 'stderr') {
@@ -359,6 +366,11 @@ if ($log_line_prefix) {
 } elsif ($format eq 'syslog') {
        $compiled_prefix = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)/;
        push(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_session_line', 't_logprefix', 't_loglevel', 't_query');
+} elsif ($format eq 'syslog-ll') {
+       $compiled_prefix = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s[^\s]+\s([^\s]+)\s([^\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)/;
+       push(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_session_line', 't_logprefix', 't_loglevel', 't_query');
+       $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s[^\s]+\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*)/;
+       $orphan_syslog_line = qr/^...\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\s]+\s[^\s\[]+\[\d+\]:/;
 } elsif ($format eq 'syslog-ng') {
        $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:(.*?)\[(\d+)\-\d+\]\s*(.*)/;
        $compiled_prefix = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:(.*?)\s\[(\d+)\-\d+\]\s*(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)/;
@@ -906,6 +918,8 @@ Options:
     -e | --end datetime    : end date/time for the data to be parsed in log.
     -f | --format logtype  : possible values: syslog,stderr,csv. Default: stderr.
                             There's also syslog-ng for traces with [ID * local2.info]
+                            and syslog-ll with syslog for line with <local0.info>.
+                            Use them only if pgBager is unable to autodetect log format.
     -G | --nograph         : disable graphs on HTML output. Enable by default.
     -h | --help            : show this message and exit.
     -i | --ident name      : programname used as syslog ident. Default: postgres
@@ -3990,13 +4004,20 @@ sub autodetect_format
                $nline++;
 
                # Is syslog lines ?
-               if ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s([^\[]+)\[\d+\]:\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) {
+               if ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s([^\s\[]+)\[\d+\]:\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) {
                        $fmt = 'syslog';
                        $nfound++;
                        $ident_name{$1}++;
 
+                       # Is syslog lines with log level information ?
+               } elsif ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\s]+\s([^\s\[]+)\[\d+\]:\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) {
+print STDERR "ZZZZZZZZZZZZZZZZZZZZZZZZ $1\n";
+                       $fmt = 'syslog-ll';
+                       $nfound++;
+                       $ident_name{$1}++;
+
                        # Is syslog-ng lines ?
-               } elsif ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s([^\[]+)\[\d+\]:(.*?)\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) {
+               } elsif ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s([^\s\[]+)\[\d+\]:(.*?)\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) {
                        $fmt = 'syslog-ng';
                        $nfound++;
                        $ident_name{$1}++;