From f98fbc78c317754c7b356a988470c0c55e9f02b5 Mon Sep 17 00:00:00 2001 From: Bruce Momjian Date: Wed, 6 Jan 2010 03:04:03 +0000 Subject: [PATCH] Preserve relfilenodes: Add support to pg_dump --binary-upgrade to preserve all relfilenodes, for use by pg_migrator. --- src/backend/catalog/heap.c | 33 +++++++--- src/backend/catalog/index.c | 26 +++++--- src/backend/catalog/toasting.c | 35 ++++------- src/backend/commands/cluster.c | 4 +- src/backend/commands/tablecmds.c | 5 +- src/backend/executor/execMain.c | 4 +- src/backend/tcop/utility.c | 10 +-- src/bin/pg_dump/pg_dump.c | 103 +++++++++++++++++++++++++------ src/include/catalog/toasting.h | 5 +- 9 files changed, 152 insertions(+), 73 deletions(-) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index d3530f07e1..7b1eba463f 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/catalog/heap.c,v 1.364 2010/01/02 16:57:36 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/catalog/heap.c,v 1.365 2010/01/06 03:03:58 momjian Exp $ * * * INTERFACE ROUTINES @@ -96,6 +96,9 @@ static Node *cookConstraint(ParseState *pstate, char *relname); static List *insert_ordered_unique_oid(List *list, Oid datum); +Oid binary_upgrade_next_heap_relfilenode = InvalidOid; +Oid binary_upgrade_next_toast_relfilenode = InvalidOid; + /* ---------------------------------------------------------------- * XXX UGLY HARD CODED BADNESS FOLLOWS XXX @@ -942,15 +945,29 @@ heap_create_with_catalog(const char *relname, errmsg("only shared relations can be placed in pg_global tablespace"))); } - /* - * Allocate an OID for the relation, unless we were told what to use. - * - * The OID will be the relfilenode as well, so make sure it doesn't - * collide with either pg_class OIDs or existing physical files. - */ - if (!OidIsValid(relid)) + if ((relkind == RELKIND_RELATION || relkind == RELKIND_SEQUENCE) && + OidIsValid(binary_upgrade_next_heap_relfilenode)) + { + relid = binary_upgrade_next_heap_relfilenode; + binary_upgrade_next_heap_relfilenode = InvalidOid; + } + else if (relkind == RELKIND_TOASTVALUE && + OidIsValid(binary_upgrade_next_toast_relfilenode)) + { + relid = binary_upgrade_next_toast_relfilenode; + binary_upgrade_next_toast_relfilenode = InvalidOid; + } + else if (!OidIsValid(relid)) + { + /* + * Allocate an OID for the relation, unless we were told what to use. + * + * The OID will be the relfilenode as well, so make sure it doesn't + * collide with either pg_class OIDs or existing physical files. + */ relid = GetNewRelFileNode(reltablespace, shared_relation, pg_class_desc); + } /* * Determine the relation's initial permissions. diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 0409876b80..7a6d914567 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.328 2010/01/02 16:57:36 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.329 2010/01/06 03:03:58 momjian Exp $ * * * INTERFACE ROUTINES @@ -79,6 +79,9 @@ typedef struct tups_inserted; } v_i_state; +/* For simple relation creation, this is the toast index relfilenode */ +Oid binary_upgrade_next_index_relfilenode = InvalidOid; + /* non-export function prototypes */ static TupleDesc ConstructTupleDescriptor(Relation heapRelation, IndexInfo *indexInfo, @@ -640,15 +643,22 @@ index_create(Oid heapRelationId, accessMethodObjectId, classObjectId); - /* - * Allocate an OID for the index, unless we were told what to use. - * - * The OID will be the relfilenode as well, so make sure it doesn't - * collide with either pg_class OIDs or existing physical files. - */ - if (!OidIsValid(indexRelationId)) + if (OidIsValid(binary_upgrade_next_index_relfilenode)) + { + indexRelationId = binary_upgrade_next_index_relfilenode; + binary_upgrade_next_index_relfilenode = InvalidOid; + } + else if (!OidIsValid(indexRelationId)) + { + /* + * Allocate an OID for the index, unless we were told what to use. + * + * The OID will be the relfilenode as well, so make sure it doesn't + * collide with either pg_class OIDs or existing physical files. + */ indexRelationId = GetNewRelFileNode(tableSpaceId, shared_relation, pg_class); + } /* * create the index relation's relcache entry and physical disk file. (If diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index 8e938cca4f..b1faccfbf9 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -8,7 +8,7 @@ * Portions Copyright (c) 1994, Regents of the University of California * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/catalog/toasting.c,v 1.26 2010/01/02 16:57:36 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/catalog/toasting.c,v 1.27 2010/01/06 03:03:58 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -32,22 +32,17 @@ #include "utils/syscache.h" Oid binary_upgrade_next_pg_type_toast_oid = InvalidOid; +extern Oid binary_upgrade_next_toast_relfilenode; static bool create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, - Datum reloptions, bool force); + Datum reloptions); static bool needs_toast_table(Relation rel); /* * AlterTableCreateToastTable * If the table needs a toast table, and doesn't already have one, - * then create a toast table for it. (With the force option, make - * a toast table even if it appears unnecessary.) - * - * The caller can also specify the OID to be used for the toast table. - * Usually, toastOid should be InvalidOid to allow a free OID to be assigned. - * (This option, as well as the force option, is not used by core Postgres, - * but is provided to support pg_migrator.) + * then create a toast table for it. * * reloptions for the toast table can be passed, too. Pass (Datum) 0 * for default reloptions. @@ -57,8 +52,7 @@ static bool needs_toast_table(Relation rel); * to end with CommandCounterIncrement if it makes any changes. */ void -AlterTableCreateToastTable(Oid relOid, Oid toastOid, - Datum reloptions, bool force) +AlterTableCreateToastTable(Oid relOid, Datum reloptions) { Relation rel; @@ -70,7 +64,7 @@ AlterTableCreateToastTable(Oid relOid, Oid toastOid, rel = heap_open(relOid, AccessExclusiveLock); /* create_toast_table does all the work */ - (void) create_toast_table(rel, toastOid, InvalidOid, reloptions, force); + (void) create_toast_table(rel, InvalidOid, InvalidOid, reloptions); heap_close(rel, NoLock); } @@ -96,7 +90,7 @@ BootstrapToastTable(char *relName, Oid toastOid, Oid toastIndexOid) relName))); /* create_toast_table does all the work */ - if (!create_toast_table(rel, toastOid, toastIndexOid, (Datum) 0, false)) + if (!create_toast_table(rel, toastOid, toastIndexOid, (Datum) 0)) elog(ERROR, "\"%s\" does not require a toast table", relName); @@ -108,12 +102,11 @@ BootstrapToastTable(char *relName, Oid toastOid, Oid toastIndexOid) * create_toast_table --- internal workhorse * * rel is already opened and exclusive-locked - * toastOid and toastIndexOid are normally InvalidOid, but - * either or both can be nonzero to specify caller-assigned OIDs + * toastOid and toastIndexOid are normally InvalidOid, but during + * bootstrap they can be nonzero to specify hand-assigned OIDs */ static bool -create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, - Datum reloptions, bool force) +create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, Datum reloptions) { Oid relOid = RelationGetRelid(rel); HeapTuple reltup; @@ -152,12 +145,10 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, /* * Check to see whether the table actually needs a TOAST table. - * - * Caller can optionally override this check. (Note: at present no - * callers in core Postgres do so, but this option is needed by - * pg_migrator.) + * If the relfilenode is specified, force toast file creation. */ - if (!force && !needs_toast_table(rel)) + if (!needs_toast_table(rel) && + !OidIsValid(binary_upgrade_next_toast_relfilenode)) return false; /* diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 7fdfebdc8b..48ba57518f 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -11,7 +11,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/cluster.c,v 1.189 2010/01/02 16:57:37 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/commands/cluster.c,v 1.190 2010/01/06 03:04:00 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -743,7 +743,7 @@ make_new_heap(Oid OIDOldHeap, const char *NewName, Oid NewTableSpace) if (isNull) reloptions = (Datum) 0; } - AlterTableCreateToastTable(OIDNewHeap, InvalidOid, reloptions, false); + AlterTableCreateToastTable(OIDNewHeap, reloptions); if (OidIsValid(toastid)) ReleaseSysCache(tuple); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 282cd45f08..dd7da742d2 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -8,7 +8,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/commands/tablecmds.c,v 1.313 2010/01/02 16:57:37 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/commands/tablecmds.c,v 1.314 2010/01/06 03:04:00 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -2614,8 +2614,7 @@ ATRewriteCatalogs(List **wqueue) (tab->subcmds[AT_PASS_ADD_COL] || tab->subcmds[AT_PASS_ALTER_TYPE] || tab->subcmds[AT_PASS_COL_ATTRS])) - AlterTableCreateToastTable(tab->relid, InvalidOid, - (Datum) 0, false); + AlterTableCreateToastTable(tab->relid, (Datum) 0); } } diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index cb4058ac66..203ed8d928 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -26,7 +26,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.339 2010/01/02 16:57:40 momjian Exp $ + * $PostgreSQL: pgsql/src/backend/executor/execMain.c,v 1.340 2010/01/06 03:04:01 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -2194,7 +2194,7 @@ OpenIntoRel(QueryDesc *queryDesc) (void) heap_reloptions(RELKIND_TOASTVALUE, reloptions, true); - AlterTableCreateToastTable(intoRelationId, InvalidOid, reloptions, false); + AlterTableCreateToastTable(intoRelationId, reloptions); /* * And open the constructed table for writing. diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 3da89ba08a..acacbec094 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -10,7 +10,7 @@ * * * IDENTIFICATION - * $PostgreSQL: pgsql/src/backend/tcop/utility.c,v 1.327 2010/01/05 21:53:58 rhaas Exp $ + * $PostgreSQL: pgsql/src/backend/tcop/utility.c,v 1.328 2010/01/06 03:04:01 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -492,14 +492,10 @@ standard_ProcessUtility(Node *parsetree, "toast", validnsps, true, false); - (void) heap_reloptions(RELKIND_TOASTVALUE, - toast_options, + (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); - AlterTableCreateToastTable(relOid, - InvalidOid, - toast_options, - false); + AlterTableCreateToastTable(relOid, toast_options); } else { diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index faee5c5b20..cff6832cd5 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -12,7 +12,7 @@ * by PostgreSQL * * IDENTIFICATION - * $PostgreSQL: pgsql/src/bin/pg_dump/pg_dump.c,v 1.564 2010/01/02 16:57:59 momjian Exp $ + * $PostgreSQL: pgsql/src/bin/pg_dump/pg_dump.c,v 1.565 2010/01/06 03:04:02 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -200,7 +200,8 @@ static void binary_upgrade_set_type_oids_by_type_oid( PQExpBuffer upgrade_buffer, Oid pg_type_oid); static bool binary_upgrade_set_type_oids_by_rel_oid( PQExpBuffer upgrade_buffer, Oid pg_rel_oid); -static void binary_upgrade_clear_pg_type_toast_oid(PQExpBuffer upgrade_buffer); +static void binary_upgrade_set_relfilenodes(PQExpBuffer upgrade_buffer, + Oid pg_class_oid, bool is_index); static const char *getAttrName(int attrnum, TableInfo *tblInfo); static const char *fmtCopyColumnList(const TableInfo *ti); static void do_sql_command(PGconn *conn, const char *query); @@ -2289,21 +2290,78 @@ binary_upgrade_set_type_oids_by_rel_oid(PQExpBuffer upgrade_buffer, } static void -binary_upgrade_clear_pg_type_toast_oid(PQExpBuffer upgrade_buffer) +binary_upgrade_set_relfilenodes(PQExpBuffer upgrade_buffer, Oid pg_class_oid, + bool is_index) { - /* - * One complexity is that while the heap might now have a TOAST table, - * the TOAST table might have been created long after creation when - * the table was loaded with wide data. For that reason, we clear - * binary_upgrade_set_next_pg_type_toast_oid so it is not reused - * by a later table. Logically any later creation that needs a TOAST - * table should have its own TOAST pg_type oid, but we are cautious. - */ - appendPQExpBuffer(upgrade_buffer, - "\n-- For binary upgrade, clear toast oid because it might not have been needed\n"); + PQExpBuffer upgrade_query = createPQExpBuffer(); + int ntups; + PGresult *upgrade_res; + Oid pg_class_relfilenode; + Oid pg_class_reltoastrelid; + Oid pg_class_reltoastidxid; + + appendPQExpBuffer(upgrade_query, + "SELECT c.relfilenode, c.reltoastrelid, t.reltoastidxid " + "FROM pg_catalog.pg_class c LEFT JOIN " + "pg_catalog.pg_class t ON (c.reltoastrelid = t.oid) " + "WHERE c.oid = '%u'::pg_catalog.oid;", + pg_class_oid); + + upgrade_res = PQexec(g_conn, upgrade_query->data); + check_sql_result(upgrade_res, g_conn, upgrade_query->data, PGRES_TUPLES_OK); + + /* Expecting a single result only */ + ntups = PQntuples(upgrade_res); + if (ntups != 1) + { + write_msg(NULL, ngettext("query returned %d row instead of one: %s\n", + "query returned %d rows instead of one: %s\n", + ntups), + ntups, upgrade_query->data); + exit_nicely(); + } + + pg_class_relfilenode = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "relfilenode"))); + pg_class_reltoastrelid = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "reltoastrelid"))); + pg_class_reltoastidxid = atooid(PQgetvalue(upgrade_res, 0, PQfnumber(upgrade_res, "reltoastidxid"))); + appendPQExpBuffer(upgrade_buffer, - "SELECT binary_upgrade.set_next_pg_type_oid('%u'::pg_catalog.oid);\n\n", - InvalidOid); + "\n-- For binary upgrade, must preserve relfilenodes\n"); + + if (!is_index) + appendPQExpBuffer(upgrade_buffer, + "SELECT binary_upgrade.set_next_heap_relfilenode('%u'::pg_catalog.oid);\n", + pg_class_relfilenode); + else + appendPQExpBuffer(upgrade_buffer, + "SELECT binary_upgrade.set_next_index_relfilenode('%u'::pg_catalog.oid);\n", + pg_class_relfilenode); + + if (OidIsValid(pg_class_reltoastrelid)) + { + /* + * One complexity is that the table definition might not require + * the creation of a TOAST table, and the TOAST table might have + * been created long after table creation, when the table was + * loaded with wide data. By setting the TOAST relfilenode we + * force creation of the TOAST heap and TOAST index by the + * backend so we can cleanly migrate the files during binary + * migration. + */ + + appendPQExpBuffer(upgrade_buffer, + "SELECT binary_upgrade.set_next_toast_relfilenode('%u'::pg_catalog.oid);\n", + pg_class_reltoastrelid); + + /* every toast table has an index */ + appendPQExpBuffer(upgrade_buffer, + "SELECT binary_upgrade.set_next_index_relfilenode('%u'::pg_catalog.oid);\n", + pg_class_reltoastidxid); + } + appendPQExpBuffer(upgrade_buffer, "\n"); + + PQclear(upgrade_res); + destroyPQExpBuffer(upgrade_query); } /* @@ -10480,6 +10538,9 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) appendPQExpBuffer(delq, "%s;\n", fmtId(tbinfo->dobj.name)); + if (binary_upgrade) + binary_upgrade_set_relfilenodes(q, tbinfo->dobj.catId.oid, false); + appendPQExpBuffer(q, "CREATE TABLE %s (", fmtId(tbinfo->dobj.name)); actual_atts = 0; @@ -10781,9 +10842,6 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) } } - if (binary_upgrade && toast_set) - binary_upgrade_clear_pg_type_toast_oid(q); - ArchiveEntry(fout, tbinfo->dobj.catId, tbinfo->dobj.dumpId, tbinfo->dobj.name, tbinfo->dobj.namespace->dobj.name, @@ -10926,6 +10984,9 @@ dumpIndex(Archive *fout, IndxInfo *indxinfo) */ if (indxinfo->indexconstraint == 0) { + if (binary_upgrade) + binary_upgrade_set_relfilenodes(q, indxinfo->dobj.catId.oid, true); + /* Plain secondary index */ appendPQExpBuffer(q, "%s;\n", indxinfo->indexdef); @@ -11006,6 +11067,9 @@ dumpConstraint(Archive *fout, ConstraintInfo *coninfo) exit_nicely(); } + if (binary_upgrade && !coninfo->condef) + binary_upgrade_set_relfilenodes(q, indxinfo->dobj.catId.oid, true); + appendPQExpBuffer(q, "ALTER TABLE ONLY %s\n", fmtId(tbinfo->dobj.name)); appendPQExpBuffer(q, " ADD CONSTRAINT %s ", @@ -11416,7 +11480,10 @@ dumpSequence(Archive *fout, TableInfo *tbinfo) resetPQExpBuffer(query); if (binary_upgrade) + { + binary_upgrade_set_relfilenodes(query, tbinfo->dobj.catId.oid, false); binary_upgrade_set_type_oids_by_rel_oid(query, tbinfo->dobj.catId.oid); + } appendPQExpBuffer(query, "CREATE SEQUENCE %s\n", diff --git a/src/include/catalog/toasting.h b/src/include/catalog/toasting.h index 156b2b6374..4f4f95c384 100644 --- a/src/include/catalog/toasting.h +++ b/src/include/catalog/toasting.h @@ -7,7 +7,7 @@ * Portions Copyright (c) 1996-2010, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * - * $PostgreSQL: pgsql/src/include/catalog/toasting.h,v 1.12 2010/01/05 01:06:57 tgl Exp $ + * $PostgreSQL: pgsql/src/include/catalog/toasting.h,v 1.13 2010/01/06 03:04:03 momjian Exp $ * *------------------------------------------------------------------------- */ @@ -17,8 +17,7 @@ /* * toasting.c prototypes */ -extern void AlterTableCreateToastTable(Oid relOid, Oid toastOid, - Datum reloptions, bool force); +extern void AlterTableCreateToastTable(Oid relOid, Datum reloptions); extern void BootstrapToastTable(char *relName, Oid toastOid, Oid toastIndexOid); -- 2.40.0