-<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.15 2007/03/02 00:48:44 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.16 2007/04/03 04:14:26 tgl Exp $ -->
<chapter id="storage">
<para>
Out-of-line values are divided (after compression if used) into chunks of at
-most <literal>TOAST_MAX_CHUNK_SIZE</> bytes (this value is a little less than
-<literal>BLCKSZ/4</>, or about 2000 bytes by default). Each chunk is stored
+most <symbol>TOAST_MAX_CHUNK_SIZE</> bytes (by default this value is chosen
+so that four chunk rows will fit on a page, making it about 2000 bytes).
+Each chunk is stored
as a separate row in the <acronym>TOAST</> table for the owning table. Every
<acronym>TOAST</> table has the columns <structfield>chunk_id</> (an OID
identifying the particular <acronym>TOAST</>ed value),
<para>
The <acronym>TOAST</> code is triggered only
-when a row value to be stored in a table is wider than <literal>BLCKSZ/4</>
-bytes (normally 2 kB). The <acronym>TOAST</> code will compress and/or move
+when a row value to be stored in a table is wider than
+<symbol>TOAST_TUPLE_THRESHOLD</> bytes (normally 2 kB).
+The <acronym>TOAST</> code will compress and/or move
field values out-of-line until the row value is shorter than
-<literal>BLCKSZ/4</> bytes or no more gains can be had. During an UPDATE
+<symbol>TOAST_TUPLE_TARGET</> bytes (also normally 2 kB)
+or no more gains can be had. During an UPDATE
operation, values of unchanged fields are normally preserved as-is; so an
UPDATE of a row with out-of-line values incurs no <acronym>TOAST</> costs if
none of the out-of-line values change.
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.230 2007/03/29 00:15:37 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/access/heap/heapam.c,v 1.231 2007/04/03 04:14:26 tgl Exp $
*
*
* INTERFACE ROUTINES
* Note: below this point, heaptup is the data we actually intend to store
* into the relation; tup is the caller's original untoasted data.
*/
- if (HeapTupleHasExternal(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD)
+ if (relation->rd_rel->relkind == RELKIND_TOASTVALUE)
+ {
+ /* toast table entries should never be recursively toasted */
+ Assert(!HeapTupleHasExternal(tup));
+ heaptup = tup;
+ }
+ else if (HeapTupleHasExternal(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD)
heaptup = toast_insert_or_update(relation, tup, NULL,
use_wal, use_fsm);
else
* because we need to look at the contents of the tuple, but it's OK to
* release the content lock on the buffer first.
*/
- if (HeapTupleHasExternal(&tp))
+ if (relation->rd_rel->relkind == RELKIND_TOASTVALUE)
+ {
+ /* toast table entries should never be recursively toasted */
+ Assert(!HeapTupleHasExternal(&tp));
+ }
+ else if (HeapTupleHasExternal(&tp))
toast_delete(relation, &tp);
/*
* We need to invoke the toaster if there are already any out-of-line
* toasted values present, or if the new tuple is over-threshold.
*/
- need_toast = (HeapTupleHasExternal(&oldtup) ||
- HeapTupleHasExternal(newtup) ||
- newtup->t_len > TOAST_TUPLE_THRESHOLD);
+ if (relation->rd_rel->relkind == RELKIND_TOASTVALUE)
+ {
+ /* toast table entries should never be recursively toasted */
+ Assert(!HeapTupleHasExternal(&oldtup));
+ Assert(!HeapTupleHasExternal(newtup));
+ need_toast = false;
+ }
+ else
+ need_toast = (HeapTupleHasExternal(&oldtup) ||
+ HeapTupleHasExternal(newtup) ||
+ newtup->t_len > TOAST_TUPLE_THRESHOLD);
pagefree = PageGetFreeSpace((Page) dp);
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.72 2007/03/29 00:15:37 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.73 2007/04/03 04:14:26 tgl Exp $
*
*
* INTERFACE ROUTINES
Datum toast_values[MaxHeapAttributeNumber];
bool toast_isnull[MaxHeapAttributeNumber];
+ /*
+ * We should only ever be called for tuples of plain relations ---
+ * recursing on a toast rel is bad news.
+ */
+ Assert(rel->rd_rel->relkind == RELKIND_RELATION);
+
/*
* Get the tuple descriptor and break down the tuple into fields.
*
bool has_nulls = false;
Size maxDataLen;
+ Size hoff;
char toast_action[MaxHeapAttributeNumber];
bool toast_isnull[MaxHeapAttributeNumber];
bool toast_free[MaxHeapAttributeNumber];
bool toast_delold[MaxHeapAttributeNumber];
+ /*
+ * We should only ever be called for tuples of plain relations ---
+ * recursing on a toast rel is bad news.
+ */
+ Assert(rel->rd_rel->relkind == RELKIND_RELATION);
+
/*
* Get the tuple descriptor and break down the tuple(s) into fields.
*/
*/
/* compute header overhead --- this should match heap_form_tuple() */
- maxDataLen = offsetof(HeapTupleHeaderData, t_bits);
+ hoff = offsetof(HeapTupleHeaderData, t_bits);
if (has_nulls)
- maxDataLen += BITMAPLEN(numAttrs);
+ hoff += BITMAPLEN(numAttrs);
if (newtup->t_data->t_infomask & HEAP_HASOID)
- maxDataLen += sizeof(Oid);
- maxDataLen = MAXALIGN(maxDataLen);
- Assert(maxDataLen == newtup->t_data->t_hoff);
+ hoff += sizeof(Oid);
+ hoff = MAXALIGN(hoff);
+ Assert(hoff == newtup->t_data->t_hoff);
/* now convert to a limit on the tuple data size */
- maxDataLen = TOAST_TUPLE_TARGET - maxDataLen;
+ maxDataLen = TOAST_TUPLE_TARGET - hoff;
/*
* Look for attributes with attstorage 'x' to compress
/*
* Second we look for attributes of attstorage 'x' or 'e' that are still
- * inline.
+ * inline. But skip this if there's no toast table to push them to.
*/
while (heap_compute_data_size(tupleDesc,
toast_values, toast_isnull) > maxDataLen &&
}
/*
- * Finally we store attributes of type 'm' external
+ * Finally we store attributes of type 'm' external, if possible.
*/
while (heap_compute_data_size(tupleDesc,
toast_values, toast_isnull) > maxDataLen &&
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
- * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.265 2007/03/03 20:02:26 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.266 2007/04/03 04:14:26 tgl Exp $
*
*-------------------------------------------------------------------------
*/
#include "access/multixact.h"
#include "access/subtrans.h"
#include "access/transam.h"
+#include "access/tuptoaster.h"
#include "access/twophase.h"
#include "access/xact.h"
#include "access/xlog_internal.h"
ControlFile->nameDataLen = NAMEDATALEN;
ControlFile->indexMaxKeys = INDEX_MAX_KEYS;
+ ControlFile->toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE;
+
#ifdef HAVE_INT64_TIMESTAMP
ControlFile->enableIntTimes = TRUE;
#else
" but the server was compiled with INDEX_MAX_KEYS %d.",
ControlFile->indexMaxKeys, INDEX_MAX_KEYS),
errhint("It looks like you need to recompile or initdb.")));
+ if (ControlFile->toast_max_chunk_size != TOAST_MAX_CHUNK_SIZE)
+ ereport(FATAL,
+ (errmsg("database files are incompatible with server"),
+ errdetail("The database cluster was initialized with TOAST_MAX_CHUNK_SIZE %d,"
+ " but the server was compiled with TOAST_MAX_CHUNK_SIZE %d.",
+ ControlFile->toast_max_chunk_size, (int) TOAST_MAX_CHUNK_SIZE),
+ errhint("It looks like you need to recompile or initdb.")));
#ifdef HAVE_INT64_TIMESTAMP
if (ControlFile->enableIntTimes != TRUE)
* copyright (c) Oliver Elphick <olly@lfix.co.uk>, 2001;
* licence: BSD
*
- * $PostgreSQL: pgsql/src/bin/pg_controldata/pg_controldata.c,v 1.34 2007/03/18 16:50:43 neilc Exp $
+ * $PostgreSQL: pgsql/src/bin/pg_controldata/pg_controldata.c,v 1.35 2007/04/03 04:14:26 tgl Exp $
*/
#include "postgres.h"
ControlFile.nameDataLen);
printf(_("Maximum columns in an index: %u\n"),
ControlFile.indexMaxKeys);
+ printf(_("Maximum size of a TOAST chunk: %u\n"),
+ ControlFile.toast_max_chunk_size);
printf(_("Date/time type storage: %s\n"),
(ControlFile.enableIntTimes ? _("64-bit integers") : _("floating-point numbers")));
printf(_("Maximum length of locale name: %u\n"),
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
- * $PostgreSQL: pgsql/src/bin/pg_resetxlog/pg_resetxlog.c,v 1.58 2007/03/03 20:02:27 momjian Exp $
+ * $PostgreSQL: pgsql/src/bin/pg_resetxlog/pg_resetxlog.c,v 1.59 2007/04/03 04:14:26 tgl Exp $
*
*-------------------------------------------------------------------------
*/
#endif
#include "access/transam.h"
+#include "access/tuptoaster.h"
#include "access/multixact.h"
#include "access/xlog_internal.h"
#include "catalog/catversion.h"
ControlFile.xlog_seg_size = XLOG_SEG_SIZE;
ControlFile.nameDataLen = NAMEDATALEN;
ControlFile.indexMaxKeys = INDEX_MAX_KEYS;
+ ControlFile.toast_max_chunk_size = TOAST_MAX_CHUNK_SIZE;
#ifdef HAVE_INT64_TIMESTAMP
ControlFile.enableIntTimes = TRUE;
#else
ControlFile.nameDataLen);
printf(_("Maximum columns in an index: %u\n"),
ControlFile.indexMaxKeys);
+ printf(_("Maximum size of a TOAST chunk: %u\n"),
+ ControlFile.toast_max_chunk_size);
printf(_("Date/time type storage: %s\n"),
(ControlFile.enableIntTimes ? _("64-bit integers") : _("floating-point numbers")));
printf(_("Maximum length of locale name: %u\n"),
*
* Copyright (c) 2000-2007, PostgreSQL Global Development Group
*
- * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.33 2007/03/29 00:15:39 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.34 2007/04/03 04:14:26 tgl Exp $
*
*-------------------------------------------------------------------------
*/
* TOAST_TUPLE_TARGET bytes. Both numbers include all tuple header overhead
* and between-fields alignment padding, but we do *not* consider any
* end-of-tuple alignment padding; hence the values can be compared directly
- * to a tuple's t_len field. We choose TOAST_TUPLE_THRESHOLD with the
- * knowledge that toast-table tuples will be exactly that size, and we'd
- * like to fit four of them per page with minimal space wastage.
+ * to a tuple's t_len field.
*
- * The numbers need not be the same, though they currently are.
+ * The numbers need not be the same, though they currently are. It doesn't
+ * make sense for TARGET to exceed THRESHOLD, but it could be useful to make
+ * it be smaller.
*
- * Note: sizeof(PageHeaderData) includes the first ItemId, but we have
- * to allow for 3 more, if we want to fit 4 tuples on a page.
+ * Currently we choose both values to match the largest tuple size for which
+ * TOAST_TUPLES_PER_PAGE tuples can fit on a disk page.
+ *
+ * XXX while these can be modified without initdb, some thought needs to be
+ * given to needs_toast_table() in toasting.c before unleashing random
+ * changes.
*/
+#define TOAST_TUPLES_PER_PAGE 4
+
+/* Note: sizeof(PageHeaderData) includes the first ItemId on the page */
#define TOAST_TUPLE_THRESHOLD \
MAXALIGN_DOWN((BLCKSZ - \
- MAXALIGN(sizeof(PageHeaderData) + 3 * sizeof(ItemIdData))) \
- / 4)
+ MAXALIGN(sizeof(PageHeaderData) + (TOAST_TUPLES_PER_PAGE-1) * sizeof(ItemIdData))) \
+ / TOAST_TUPLES_PER_PAGE)
#define TOAST_TUPLE_TARGET TOAST_TUPLE_THRESHOLD
* When we store an oversize datum externally, we divide it into chunks
* containing at most TOAST_MAX_CHUNK_SIZE data bytes. This number *must*
* be small enough that the completed toast-table tuple (including the
- * ID and sequence fields and all overhead) is no more than MaxHeapTupleSize
- * bytes. It *should* be small enough to make toast-table tuples no more
- * than TOAST_TUPLE_THRESHOLD bytes, else heapam.c will uselessly invoke
- * the toaster on toast-table tuples. The current coding ensures that the
- * maximum tuple length is exactly TOAST_TUPLE_THRESHOLD bytes.
- *
- * NB: you cannot change this value without forcing initdb, at least not
- * if your DB contains any multi-chunk toasted values.
+ * ID and sequence fields and all overhead) will fit on a page.
+ * The coding here sets the size on the theory that we want to fit
+ * EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page.
+ *
+ * NB: Changing TOAST_MAX_CHUNK_SIZE requires an initdb.
*/
-#define TOAST_MAX_CHUNK_SIZE (TOAST_TUPLE_THRESHOLD - \
- MAXALIGN(offsetof(HeapTupleHeaderData, t_bits)) - \
- sizeof(Oid) - \
- sizeof(int32) - \
- VARHDRSZ)
+#define EXTERN_TUPLES_PER_PAGE 4 /* tweak only this */
+
+/* Note: sizeof(PageHeaderData) includes the first ItemId on the page */
+#define EXTERN_TUPLE_MAX_SIZE \
+ MAXALIGN_DOWN((BLCKSZ - \
+ MAXALIGN(sizeof(PageHeaderData) + (EXTERN_TUPLES_PER_PAGE-1) * sizeof(ItemIdData))) \
+ / EXTERN_TUPLES_PER_PAGE)
+
+#define TOAST_MAX_CHUNK_SIZE \
+ (EXTERN_TUPLE_MAX_SIZE - \
+ MAXALIGN(offsetof(HeapTupleHeaderData, t_bits)) - \
+ sizeof(Oid) - \
+ sizeof(int32) - \
+ VARHDRSZ)
/* ----------
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
- * $PostgreSQL: pgsql/src/include/catalog/pg_control.h,v 1.36 2007/03/03 20:02:27 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/pg_control.h,v 1.37 2007/04/03 04:14:26 tgl Exp $
*
*-------------------------------------------------------------------------
*/
/* Version identifier for this pg_control format */
-#define PG_CONTROL_VERSION 832
+#define PG_CONTROL_VERSION 833
/*
* Body of CheckPoint XLOG records. This is declared here because we keep
uint32 nameDataLen; /* catalog name field width */
uint32 indexMaxKeys; /* max number of columns in an index */
+ uint32 toast_max_chunk_size; /* chunk size in TOAST tables */
+
/* flag indicating internal format of timestamp, interval, time */
uint32 enableIntTimes; /* int64 storage enabled? */
projects / postgresql / commitdiff