From 3e92142b5db3203ba43fbef3ed5c1e06ec1a3db0 Mon Sep 17 00:00:00 2001 From: Darold Gilles Date: Thu, 8 May 2014 16:32:23 +0200 Subject: [PATCH] Update documentation about remote mode feature. --- README | 51 +++++++++++++++++++++++++++++++------------ doc/pgBadger.pod | 56 +++++++++++++++++++++++++++++++++--------------- 2 files changed, 76 insertions(+), 31 deletions(-) diff --git a/README b/README index e26c326..34c49ec 100644 --- a/README +++ b/README @@ -15,7 +15,7 @@ SYNOPSIS Options: -a | --average minutes : number of minutes to build the average graphs of - queries and connections. + queries and connections. Default 5 minutes. -A | --histo-avg minutes: number of minutes to build the histogram graphs of queries. Default 60 minutes. -b | --begin datetime : start date/time for the data to be parsed in log. @@ -23,14 +23,14 @@ SYNOPSIS -C | --nocomment : remove comments like /* ... */ from queries. -d | --dbname database : only report on entries for the given database. -e | --end datetime : end date/time for the data to be parsed in log. - -f | --format logtype : possible values: syslog,stderr,csv. Default: stderr - -G | --nograph : disable graphs on HTML output. Enable by default. + -f | --format logtype : possible values: syslog,stderr,csv. Default: stderr. + -G | --nograph : disable graphs on HTML output. Enabled by default. -h | --help : show this message and exit. -i | --ident name : programname used as syslog ident. Default: postgres -I | --incremental : use incremental mode, reports will be generated by days in a separate directory, --outdir must be set. - -j | --jobs number : number of jobs to run on parallel on each log file. - Default is 1, run as single process. + -j | --jobs number : number of jobs to run at same time. Default is 1, + run as single process. -J | --Jobs number : number of log file to parse in parallel. Default is 1, run as single process. -l | --last-parsed file: allow incremental log parsing by registering the @@ -41,11 +41,11 @@ SYNOPSIS the given size. Default: no truncate -n | --nohighlight : disable SQL code highlighting. -N | --appname name : only report on entries for given application name - -o | --outfile filename: define the filename for output. Default depends on - the output format: out.html, out.txt or out.tsung. + -o | --outfile filename: define the filename for the output. Default depends + on the output format: out.html, out.txt or out.tsung. To dump output to stdout use - as filename. -O | --outdir path : directory where out file must be saved. - -p | --prefix string : give here the value of your custom log_line_prefix + -p | --prefix string : the value of your custom log_line_prefix as defined in your postgresql.conf. Only use it if you aren't using one of the standard prefixes specified in the pgBadger documentation, such as if your prefix @@ -53,8 +53,10 @@ SYNOPSIS application name. See examples below. -P | --no-prettify : disable SQL queries prettify formatter. -q | --quiet : don't print anything to stdout, not even a progress bar. + -r | --remote-host ip : set the host where to execute the cat command on remote + logfile to parse localy the file. -s | --sample number : number of query samples to store/display. Default: 3 - -S | --select-only : use it if you want to report select queries only. + -S | --select-only : only report SELECT queries. -t | --top number : number of queries to store/display. Default: 20 -T | --title string : change title of the HTML page report. -u | --dbuser username : only report on entries for the given user. @@ -64,7 +66,7 @@ SYNOPSIS -w | --watch-mode : only report errors just like logwatch could do. -x | --extension : output format. Values: text, html or tsung. Default: html -z | --zcat exec_path : set the full path to the zcat program. Use it if - zcat or bzcat or unzip is not on your path. + zcat or bzcat or unzip is not in your path. --pie-limit num : pie data lower than num% will show a sum instead. --exclude-query regex : any query matching the given regex will be excluded from the report. For example: "^(VACUUM|COMMIT)" @@ -78,14 +80,14 @@ SYNOPSIS queries to include from the report. One regex per line. --disable-error : do not generate error report. --disable-hourly : do not generate hourly report. - --disable-type : do not generate query type report. + --disable-type : do not generate report of queries by type, database... --disable-query : do not generate query reports (slowest, most - frequent, ...). + frequent, queries by users, by database, ...). --disable-session : do not generate session report. --disable-connection : do not generate connection report. --disable-lock : do not generate lock report. --disable-temporary : do not generate temporary report. - --disable-checkpoint : do not generate checkpoint report. + --disable-checkpoint : do not generate checkpoint/restartpoint report. --disable-autovacuum : do not generate autovacuum report. --charset : used to set the HTML charset to be used. Default: utf-8. --csv-separator : used to set the CSV field separator, default: , @@ -97,6 +99,20 @@ SYNOPSIS --exclude-line regex : pgbadger will start to exclude any log entry that will match the given regex. Can be used multiple time. + pgBadger is able to parse a remote log file using a passwordless ssh + connection. Use the -r or --remote-host to set the host ip address or + hostname. There's also some additional options to fully control the ssh + connection. + + --ssh-program ssh path to the ssh program to use. Default: ssh. + --ssh-user username connection login name. Default to running user. + --ssh-identity file path to the identity file to use. + --ssh-timeout second timeout to ssh connection failure. Default 10 seconds. + --ssh-options options list of -o options to use for the ssh connection. Options + always used: + -o ConnectTimeout=$ssh_timeout + -o PreferredAuthentications=hostbased,publickey + Examples: pgbadger /var/log/postgresql.log @@ -218,12 +234,19 @@ FEATURE reported are highlighted and beautified automatically. You can also have incremental reports with one report per day and a - cumulative report per week. + cumulative report per week. Two multiprocess modes are available to + speed up log parsing, one using one core per log file, and the second to + use multiple core to parse a single file. Both modes can be combined. Histogram granularity can be adjusted using the -A command line option. By default they will report the mean of each top queries/error occuring per hour, but you can specify the granularity down to the minute. + pgBadger can also be used in a central place to parse remote log files + using a password less SSH connection. This mode can be used with + compressed files and in mode multiprocess per file (-J) but can not be + used with CSV log format. + REQUIREMENT pgBadger comes as a single Perl script - you do not need anything other than a modern Perl distribution. Charts are rendered using a Javascript diff --git a/doc/pgBadger.pod b/doc/pgBadger.pod index 26ba34a..f5be59c 100644 --- a/doc/pgBadger.pod +++ b/doc/pgBadger.pod @@ -17,7 +17,7 @@ Arguments: Options: -a | --average minutes : number of minutes to build the average graphs of - queries and connections. + queries and connections. Default 5 minutes. -A | --histo-avg minutes: number of minutes to build the histogram graphs of queries. Default 60 minutes. -b | --begin datetime : start date/time for the data to be parsed in log. @@ -25,14 +25,14 @@ Options: -C | --nocomment : remove comments like /* ... */ from queries. -d | --dbname database : only report on entries for the given database. -e | --end datetime : end date/time for the data to be parsed in log. - -f | --format logtype : possible values: syslog,stderr,csv. Default: stderr - -G | --nograph : disable graphs on HTML output. Enable by default. + -f | --format logtype : possible values: syslog,stderr,csv. Default: stderr. + -G | --nograph : disable graphs on HTML output. Enabled by default. -h | --help : show this message and exit. -i | --ident name : programname used as syslog ident. Default: postgres -I | --incremental : use incremental mode, reports will be generated by days in a separate directory, --outdir must be set. - -j | --jobs number : number of jobs to run on parallel on each log file. - Default is 1, run as single process. + -j | --jobs number : number of jobs to run at same time. Default is 1, + run as single process. -J | --Jobs number : number of log file to parse in parallel. Default is 1, run as single process. -l | --last-parsed file: allow incremental log parsing by registering the @@ -43,11 +43,11 @@ Options: the given size. Default: no truncate -n | --nohighlight : disable SQL code highlighting. -N | --appname name : only report on entries for given application name - -o | --outfile filename: define the filename for output. Default depends on - the output format: out.html, out.txt or out.tsung. + -o | --outfile filename: define the filename for the output. Default depends + on the output format: out.html, out.txt or out.tsung. To dump output to stdout use - as filename. -O | --outdir path : directory where out file must be saved. - -p | --prefix string : give here the value of your custom log_line_prefix + -p | --prefix string : the value of your custom log_line_prefix as defined in your postgresql.conf. Only use it if you aren't using one of the standard prefixes specified in the pgBadger documentation, such as if your prefix @@ -55,8 +55,10 @@ Options: application name. See examples below. -P | --no-prettify : disable SQL queries prettify formatter. -q | --quiet : don't print anything to stdout, not even a progress bar. + -r | --remote-host ip : set the host where to execute the cat command on remote + logfile to parse localy the file. -s | --sample number : number of query samples to store/display. Default: 3 - -S | --select-only : use it if you want to report select queries only. + -S | --select-only : only report SELECT queries. -t | --top number : number of queries to store/display. Default: 20 -T | --title string : change title of the HTML page report. -u | --dbuser username : only report on entries for the given user. @@ -66,7 +68,7 @@ Options: -w | --watch-mode : only report errors just like logwatch could do. -x | --extension : output format. Values: text, html or tsung. Default: html -z | --zcat exec_path : set the full path to the zcat program. Use it if - zcat or bzcat or unzip is not on your path. + zcat or bzcat or unzip is not in your path. --pie-limit num : pie data lower than num% will show a sum instead. --exclude-query regex : any query matching the given regex will be excluded from the report. For example: "^(VACUUM|COMMIT)" @@ -80,24 +82,37 @@ Options: queries to include from the report. One regex per line. --disable-error : do not generate error report. --disable-hourly : do not generate hourly report. - --disable-type : do not generate query type report. + --disable-type : do not generate report of queries by type, database... --disable-query : do not generate query reports (slowest, most - frequent, ...). + frequent, queries by users, by database, ...). --disable-session : do not generate session report. --disable-connection : do not generate connection report. --disable-lock : do not generate lock report. --disable-temporary : do not generate temporary report. - --disable-checkpoint : do not generate checkpoint report. + --disable-checkpoint : do not generate checkpoint/restartpoint report. --disable-autovacuum : do not generate autovacuum report. --charset : used to set the HTML charset to be used. Default: utf-8. --csv-separator : used to set the CSV field separator, default: , --exclude-time regex : any timestamp matching the given regex will be - excluded from the report. Example: "2013-04-12 .*" + excluded from the report. Example: "2013-04-12 .*" You can use this option multiple times. --exclude-appname name : exclude entries for the specified application name - from report. Example: "pg_dump". + from report. Example: "pg_dump". --exclude-line regex : pgbadger will start to exclude any log entry that - will match the given regex. Can be used multiple time. + will match the given regex. Can be used multiple time. + +pgBadger is able to parse a remote log file using a passwordless ssh connection. +Use the -r or --remote-host to set the host ip address or hostname. There's also +some additional options to fully control the ssh connection. + + --ssh-program ssh path to the ssh program to use. Default: ssh. + --ssh-user username connection login name. Default to running user. + --ssh-identity file path to the identity file to use. + --ssh-timeout second timeout to ssh connection failure. Default 10 seconds. + --ssh-options options list of -o options to use for the ssh connection. Options + always used: + -o ConnectTimeout=$ssh_timeout + -o PreferredAuthentications=hostbased,publickey Examples: @@ -203,12 +218,19 @@ All charts are zoomable and can be saved as PNG images. SQL queries reported are highlighted and beautified automatically. You can also have incremental reports with one report per day and a cumulative -report per week. +report per week. Two multiprocess modes are available to speed up log parsing, +one using one core per log file, and the second to use multiple core to parse +a single file. Both modes can be combined. Histogram granularity can be adjusted using the -A command line option. By default they will report the mean of each top queries/error occuring per hour, but you can specify the granularity down to the minute. +pgBadger can also be used in a central place to parse remote log files using a +password less SSH connection. This mode can be used with compressed files and +in mode multiprocess per file (-J) but can not be used with CSV log format. + + =head1 REQUIREMENT pgBadger comes as a single Perl script - you do not need anything other than a modern -- 2.50.1