From: Tom Lane <tgl@sss.pgh.pa.us>
Date: Wed, 21 Mar 2012 17:04:12 +0000 (-0400)
Subject: Back-patch contrib/vacuumlo's new -l (limit) option into 9.0 and 9.1.
X-Git-Tag: REL9_0_8~46
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=3bf25a2a16ca6efefa97f058da062b6c5933ebe1;p=postgresql
Back-patch contrib/vacuumlo's new -l (limit) option into 9.0 and 9.1.
Since 9.0, removing lots of large objects in a single transaction risks
exceeding max_locks_per_transaction, because we merged large object removal
into the generic object-drop mechanism, which takes out an exclusive lock
on each object to be dropped. This creates a hazard for contrib/vacuumlo,
which has historically tried to drop all unreferenced large objects in one
transaction. There doesn't seem to be any correctness requirement to do it
that way, though; we only need to drop enough large objects per transaction
to amortize the commit costs.
To prevent a regression from pre-9.0 releases wherein vacuumlo worked just
fine, back-patch commits b69f2e36402aaa222ed03c1769b3de6d5be5f302 and
64c604898e812aa93c124c666e8709fff1b8dd26, which break vacuumlo's deletions
into multiple transactions with a user-controllable upper limit on the
number of objects dropped per transaction.
Tim Lewis, Robert Haas, Tom Lane
---
diff --git a/contrib/vacuumlo/vacuumlo.c b/contrib/vacuumlo/vacuumlo.c
index 2ed1cbeb93..641a8c3425 100644
--- a/contrib/vacuumlo/vacuumlo.c
+++ b/contrib/vacuumlo/vacuumlo.c
@@ -3,12 +3,12 @@
* vacuumlo.c
* This removes orphaned large objects from a database.
*
- * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1996-2012, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/contrib/vacuumlo/vacuumlo.c,v 1.45 2010/02/17 04:19:37 tgl Exp $
+ * contrib/vacuumlo/vacuumlo.c
*
*-------------------------------------------------------------------------
*/
@@ -22,7 +22,6 @@
#endif
#include "libpq-fe.h"
-#include "libpq/libpq-fs.h"
#define atooid(x) ((Oid) strtoul((x), NULL, 10))
@@ -30,8 +29,7 @@
extern char *optarg;
extern int optind,
- opterr,
- optopt;
+ opterr;
enum trivalue
{
@@ -48,29 +46,32 @@ struct _param
char *pg_host;
int verbose;
int dry_run;
+ long transaction_limit;
};
-int vacuumlo(char *, struct _param *);
-void usage(const char *progname);
+static int vacuumlo(const char *database, const struct _param * param);
+static void usage(const char *progname);
/*
* This vacuums LOs of one database. It returns 0 on success, -1 on failure.
*/
-int
-vacuumlo(char *database, struct _param * param)
+static int
+vacuumlo(const char *database, const struct _param * param)
{
PGconn *conn;
PGresult *res,
*res2;
char buf[BUFSIZE];
- int matched;
- int deleted;
+ long matched;
+ long deleted;
int i;
static char *password = NULL;
bool new_pass;
+ bool success = true;
+ /* Note: password can be carried over from a previous call */
if (param->pg_prompt == TRI_YES && password == NULL)
password = simple_prompt("Password: ", 100, false);
@@ -118,7 +119,7 @@ vacuumlo(char *database, struct _param * param)
if (param->verbose)
{
- fprintf(stdout, "Connected to %s\n", database);
+ fprintf(stdout, "Connected to database \"%s\"\n", database);
if (param->dry_run)
fprintf(stdout, "Test run: no large objects will be removed!\n");
}
@@ -219,9 +220,21 @@ vacuumlo(char *database, struct _param * param)
if (param->verbose)
fprintf(stdout, "Checking %s in %s.%s\n", field, schema, table);
+ schema = PQescapeIdentifier(conn, schema, strlen(schema));
+ table = PQescapeIdentifier(conn, table, strlen(table));
+ field = PQescapeIdentifier(conn, field, strlen(field));
+
+ if (!schema || !table || !field)
+ {
+ fprintf(stderr, "Out of memory\n");
+ PQclear(res);
+ PQfinish(conn);
+ return -1;
+ }
+
snprintf(buf, BUFSIZE,
"DELETE FROM vacuum_l "
- "WHERE lo IN (SELECT \"%s\" FROM \"%s\".\"%s\")",
+ "WHERE lo IN (SELECT %s FROM %s.%s)",
field, schema, table);
res2 = PQexec(conn, buf);
if (PQresultStatus(res2) != PGRES_COMMAND_OK)
@@ -235,23 +248,35 @@ vacuumlo(char *database, struct _param * param)
return -1;
}
PQclear(res2);
+
+ PQfreemem(schema);
+ PQfreemem(table);
+ PQfreemem(field);
}
PQclear(res);
/*
- * Run the actual deletes in a single transaction. Note that this would
- * be a bad idea in pre-7.1 Postgres releases (since rolling back a table
- * delete used to cause problems), but it should be safe now.
+ * Now, those entries remaining in vacuum_l are orphans. Delete 'em.
+ *
+ * We don't want to run each delete as an individual transaction, because
+ * the commit overhead would be high. However, since 9.0 the backend will
+ * acquire a lock per deleted LO, so deleting too many LOs per transaction
+ * risks running out of room in the shared-memory lock table.
+ * Accordingly, we delete up to transaction_limit LOs per transaction.
*/
res = PQexec(conn, "begin");
+ if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ {
+ fprintf(stderr, "Failed to start transaction:\n");
+ fprintf(stderr, "%s", PQerrorMessage(conn));
+ PQclear(res);
+ PQfinish(conn);
+ return -1;
+ }
PQclear(res);
- /*
- * Finally, those entries remaining in vacuum_l are orphans.
- */
buf[0] = '\0';
- strcat(buf, "SELECT lo ");
- strcat(buf, "FROM vacuum_l");
+ strcat(buf, "SELECT lo FROM vacuum_l");
res = PQexec(conn, buf);
if (PQresultStatus(res) != PGRES_TUPLES_OK)
{
@@ -280,37 +305,87 @@ vacuumlo(char *database, struct _param * param)
{
fprintf(stderr, "\nFailed to remove lo %u: ", lo);
fprintf(stderr, "%s", PQerrorMessage(conn));
+ if (PQtransactionStatus(conn) == PQTRANS_INERROR)
+ {
+ success = false;
+ break;
+ }
}
else
deleted++;
}
else
deleted++;
+ if (param->transaction_limit > 0 &&
+ (deleted % param->transaction_limit) == 0)
+ {
+ res2 = PQexec(conn, "commit");
+ if (PQresultStatus(res2) != PGRES_COMMAND_OK)
+ {
+ fprintf(stderr, "Failed to commit transaction:\n");
+ fprintf(stderr, "%s", PQerrorMessage(conn));
+ PQclear(res2);
+ PQclear(res);
+ PQfinish(conn);
+ return -1;
+ }
+ PQclear(res2);
+ res2 = PQexec(conn, "begin");
+ if (PQresultStatus(res2) != PGRES_COMMAND_OK)
+ {
+ fprintf(stderr, "Failed to start transaction:\n");
+ fprintf(stderr, "%s", PQerrorMessage(conn));
+ PQclear(res2);
+ PQclear(res);
+ PQfinish(conn);
+ return -1;
+ }
+ PQclear(res2);
+ }
}
PQclear(res);
/*
* That's all folks!
*/
- res = PQexec(conn, "end");
+ res = PQexec(conn, "commit");
+ if (PQresultStatus(res) != PGRES_COMMAND_OK)
+ {
+ fprintf(stderr, "Failed to commit transaction:\n");
+ fprintf(stderr, "%s", PQerrorMessage(conn));
+ PQclear(res);
+ PQfinish(conn);
+ return -1;
+ }
PQclear(res);
PQfinish(conn);
if (param->verbose)
- fprintf(stdout, "\r%s %d large objects from %s.\n",
- (param->dry_run ? "Would remove" : "Removed"), deleted, database);
+ {
+ if (param->dry_run)
+ fprintf(stdout, "\rWould remove %ld large objects from database \"%s\".\n",
+ deleted, database);
+ else if (success)
+ fprintf(stdout,
+ "\rSuccessfully removed %ld large objects from database \"%s\".\n",
+ deleted, database);
+ else
+ fprintf(stdout, "\rRemoval from database \"%s\" failed at object %ld of %ld.\n",
+ database, deleted, matched);
+ }
- return 0;
+ return ((param->dry_run || success) ? 0 : -1);
}
-void
+static void
usage(const char *progname)
{
printf("%s removes unreferenced large objects from databases.\n\n", progname);
printf("Usage:\n %s [OPTION]... DBNAME...\n\n", progname);
printf("Options:\n");
printf(" -h HOSTNAME database server host or socket directory\n");
+ printf(" -l LIMIT commit after removing each LIMIT large objects\n");
printf(" -n don't remove large objects, just show what would be done\n");
printf(" -p PORT database server port\n");
printf(" -U USERNAME user name to connect as\n");
@@ -335,14 +410,16 @@ main(int argc, char **argv)
progname = get_progname(argv[0]);
- /* Parameter handling */
+ /* Set default parameter values */
param.pg_user = NULL;
param.pg_prompt = TRI_DEFAULT;
param.pg_host = NULL;
param.pg_port = NULL;
param.verbose = 0;
param.dry_run = 0;
+ param.transaction_limit = 1000;
+ /* Process command-line arguments */
if (argc > 1)
{
if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
@@ -359,7 +436,7 @@ main(int argc, char **argv)
while (1)
{
- c = getopt(argc, argv, "h:U:p:vnwW");
+ c = getopt(argc, argv, "h:l:U:p:vnwW");
if (c == -1)
break;
@@ -377,6 +454,16 @@ main(int argc, char **argv)
param.dry_run = 1;
param.verbose = 1;
break;
+ case 'l':
+ param.transaction_limit = strtol(optarg, NULL, 10);
+ if (param.transaction_limit < 0)
+ {
+ fprintf(stderr,
+ "%s: transaction limit must not be negative (0 disables)\n",
+ progname);
+ exit(1);
+ }
+ break;
case 'U':
param.pg_user = strdup(optarg);
break;
@@ -405,7 +492,7 @@ main(int argc, char **argv)
if (optind >= argc)
{
fprintf(stderr, "vacuumlo: missing required argument: database name\n");
- fprintf(stderr, "Try 'vacuumlo -?' for help.\n");
+ fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
}
diff --git a/doc/src/sgml/vacuumlo.sgml b/doc/src/sgml/vacuumlo.sgml
index 04b59c70d7..f9431fe993 100644
--- a/doc/src/sgml/vacuumlo.sgml
+++ b/doc/src/sgml/vacuumlo.sgml
@@ -49,6 +49,19 @@ vacuumlo [options] database [database2 ... databaseN]
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-l</option> <replaceable>limit</></term>
+ <listitem>
+ <para>
+ Remove no more than <replaceable>limit</> large objects per
+ transaction (default 1000). Since the server acquires a lock per LO
+ removed, removing too many LOs in one transaction risks exceeding
+ <xref linkend="guc-max-locks-per-transaction">. Set the limit to
+ zero if you want all removals done in a single transaction.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>-U</option> <replaceable>username</></term>
<listitem>
@@ -75,7 +88,7 @@ vacuumlo [options] database [database2 ... databaseN]
<listitem>
<para>
Force <application>vacuumlo</application> to prompt for a
- password before connecting to a database.
+ password before connecting to a database.
</para>
<para>
@@ -110,18 +123,19 @@ vacuumlo [options] database [database2 ... databaseN]
<title>Method</title>
<para>
- First, it builds a temporary table which contains all of the OIDs of the
- large objects in that database.
+ First, <application>vacuumlo</> builds a temporary table which contains all
+ of the OIDs of the large objects in the selected database.
</para>
<para>
It then scans through all columns in the database that are of type
<type>oid</> or <type>lo</>, and removes matching entries from the
- temporary table.
+ temporary table. (Note: only types with these names are considered;
+ in particular, domains over them are not considered.)
</para>
<para>
- The remaining entries in the temp table identify orphaned LOs.
+ The remaining entries in the temporary table identify orphaned LOs.
These are removed.
</para>
</sect2>