From f539c4a7396c28c4872fb94839cac16925a4f1e1 Mon Sep 17 00:00:00 2001 From: Darold Gilles Date: Wed, 7 Nov 2012 11:53:38 +0100 Subject: [PATCH] Fix parser on syslog with level info after the timestamp like . Thanks to Jehan-Guillaume de Rorthais for the report. --- README | 2 ++ doc/pgBadger.pod | 2 ++ pgbadger | 35 ++++++++++++++++++++++++++++------- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/README b/README index 6f483b1..b8ab8d8 100644 --- a/README +++ b/README @@ -23,6 +23,8 @@ SYNOPSIS -e | --end datetime : end date/time for the data to be parsed in log. -f | --format logtype : possible values: syslog,stderr,csv. Default: stderr. There's also syslog-ng for traces with [ID * local2.info] + and syslog-ll with syslog for line with . + Use them only if pgBager is unable to autodetect log format. -G | --nograph : disable graphs on HTML output. Enable by default. -h | --help : show this message and exit. -i | --ident name : programname used as syslog ident. Default: postgres diff --git a/doc/pgBadger.pod b/doc/pgBadger.pod index 79708cd..484c693 100644 --- a/doc/pgBadger.pod +++ b/doc/pgBadger.pod @@ -25,6 +25,8 @@ Options: -e | --end datetime : end date/time for the data to be parsed in log. -f | --format logtype : possible values: syslog,stderr,csv. Default: stderr. There's also syslog-ng for traces with [ID * local2.info] + and syslog-ll with syslog for line with . + Use them only if pgBager is unable to autodetect log format. -G | --nograph : disable graphs on HTML output. Enable by default. -h | --help : show this message and exit. -i | --ident name : programname used as syslog ident. Default: postgres diff --git a/pgbadger b/pgbadger index bc64313..8309af4 100755 --- a/pgbadger +++ b/pgbadger @@ -332,8 +332,8 @@ if ($#include_query >= 0) { } } -my $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*)/; -my $orphan_syslog_line = qr/^...\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\[]+\[\d+\]:/; +my $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*)/; +my $orphan_syslog_line = qr/^...\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\s\[]+\[\d+\]:/; my $orphan_stderr_line = qr/[^']*\d+-\d+-\d+\s\d+:\d+:\d+[\.\d]*\s[^\s]+[^']*/; # Compile custom log line prefie prefix @@ -342,13 +342,20 @@ if ($log_line_prefix) { @prefix_params = &build_log_line_prefix_regex(); &check_regex($log_line_prefix, '--prefix'); if ($format eq 'syslog') { - $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)'; + $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)'; + $compiled_prefix = qr/$log_line_prefix/; + unshift(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_session_line'); + push(@prefix_params, 't_loglevel', 't_query'); + } elsif ($format eq 'syslog-ll') { + $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s[^\s]+\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)'; + $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s[^\s]+\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*)/; + $orphan_syslog_line = qr/^...\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\s]+\s[^\s\[]+\[\d+\]:/; $compiled_prefix = qr/$log_line_prefix/; unshift(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_session_line'); push(@prefix_params, 't_loglevel', 't_query'); } elsif ($format eq 'syslog-ng') { - $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:(.*?)\[(\d+)\-\d+\]\s*(.*)/; - $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:(.*?)\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)'; + $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:(.*?)\[(\d+)\-\d+\]\s*(.*)/; + $log_line_prefix = '^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:(.*?)\s\[(\d+)\-\d+\]\s*' . $log_line_prefix . '\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)'; $compiled_prefix = qr/$log_line_prefix/; unshift(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_id', 't_session_line'); } elsif ($format eq 'stderr') { @@ -359,6 +366,11 @@ if ($log_line_prefix) { } elsif ($format eq 'syslog') { $compiled_prefix = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)/; push(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_session_line', 't_logprefix', 't_loglevel', 't_query'); +} elsif ($format eq 'syslog-ll') { + $compiled_prefix = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s[^\s]+\s([^\s]+)\s([^\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)/; + push(@prefix_params, 't_month', 't_day', 't_hour', 't_min', 't_sec', 't_host', 't_ident', 't_pid', 't_session_line', 't_logprefix', 't_loglevel', 't_query'); + $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s[^\s]+\s([^\s]+)\s([^\s\[]+)\[(\d+)\]:\s\[(\d+)\-\d+\]\s*(.*)/; + $orphan_syslog_line = qr/^...\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\s]+\s[^\s\[]+\[\d+\]:/; } elsif ($format eq 'syslog-ng') { $other_syslog_line = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:(.*?)\[(\d+)\-\d+\]\s*(.*)/; $compiled_prefix = qr/^(...)\s+(\d+)\s(\d+):(\d+):(\d+)\s([^\s]+)\s([^\[]+)\[(\d+)\]:(.*?)\s\[(\d+)\-\d+\]\s*(.*?)\s*(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+(.*)/; @@ -906,6 +918,8 @@ Options: -e | --end datetime : end date/time for the data to be parsed in log. -f | --format logtype : possible values: syslog,stderr,csv. Default: stderr. There's also syslog-ng for traces with [ID * local2.info] + and syslog-ll with syslog for line with . + Use them only if pgBager is unable to autodetect log format. -G | --nograph : disable graphs on HTML output. Enable by default. -h | --help : show this message and exit. -i | --ident name : programname used as syslog ident. Default: postgres @@ -3990,13 +4004,20 @@ sub autodetect_format $nline++; # Is syslog lines ? - if ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s([^\[]+)\[\d+\]:\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) { + if ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s([^\s\[]+)\[\d+\]:\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) { $fmt = 'syslog'; $nfound++; $ident_name{$1}++; + # Is syslog lines with log level information ? + } elsif ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s[^\s]+\s([^\s\[]+)\[\d+\]:\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) { +print STDERR "ZZZZZZZZZZZZZZZZZZZZZZZZ $1\n"; + $fmt = 'syslog-ll'; + $nfound++; + $ident_name{$1}++; + # Is syslog-ng lines ? - } elsif ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s([^\[]+)\[\d+\]:(.*?)\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) { + } elsif ($line =~ /^[A-Z][a-z]{2}\s+\d+\s\d+:\d+:\d+\s[^\s]+\s([^\s\[]+)\[\d+\]:(.*?)\s\[[0-9\-]+\](.*?)(LOG|WARNING|ERROR|FATAL|PANIC|DETAIL|STATEMENT|HINT|CONTEXT):\s+$duration/) { $fmt = 'syslog-ng'; $nfound++; $ident_name{$1}++; -- 2.40.0