Although we can't support out-of-line TOAST storage in indexes (yet),

compressed storage works perfectly well.  Might as well have a coherent
strategy for applying it, rather than the haphazard store-what-you-get
approach that was in the code before.  The strategy I've set up here is
to attempt compression of any compressible index value exceeding
BLCKSZ/16, or about 500 bytes by default.

diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c
index 16b5233eb3907b2b1ead4dde2175a00a4bf1d6b3..4b46c202dcd47064038267fcbb5f99fb47df8787 100644 (file)
--- a/src/backend/access/common/indextuple.c
+++ b/src/backend/access/common/indextuple.c
@@ -2,14 +2,14 @@
  *
  * indextuple.c
  *        This file contains index tuple accessor and mutator routines,
- *        as well as a few various tuple utilities.
+ *        as well as various tuple utilities.
  *
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  *
  * IDENTIFICATION
- *       $Header: /cvsroot/pgsql/src/backend/access/common/indextuple.c,v 1.50 2001/01/24 19:42:47 momjian Exp $
+ *       $Header: /cvsroot/pgsql/src/backend/access/common/indextuple.c,v 1.51 2001/02/15 20:57:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -57,25 +57,45 @@ index_formtuple(TupleDesc tupleDescriptor,
 #ifdef TOAST_INDEX_HACK
        for (i = 0; i < numberOfAttributes; i++)
        {
-               if (null[i] != ' ' || tupleDescriptor->attrs[i]->attlen >= 0)
+               Form_pg_attribute  att = tupleDescriptor->attrs[i];
+
+               untoasted_value[i] = value[i];
+               untoasted_free[i] = false;
+
+               /* Do nothing if value is NULL or not of varlena type */
+               if (null[i] != ' ' || att->attlen >= 0)
+                       continue;
+
+               /*
+                * If value is stored EXTERNAL, must fetch it so we are not
+                * depending on outside storage.  This should be improved someday.
+                */
+               if (VARATT_IS_EXTERNAL(value[i]))
                {
-                       untoasted_value[i] = value[i];
-                       untoasted_free[i] = false;
+                       untoasted_value[i] = PointerGetDatum(
+                               heap_tuple_fetch_attr(
+                                       (varattrib *) DatumGetPointer(value[i])));
+                       untoasted_free[i] = true;
                }
-               else
+
+               /*
+                * If value is above size target, and is of a compressible datatype,
+                * try to compress it in-line.
+                */
+               if (VARATT_SIZE(untoasted_value[i]) > TOAST_INDEX_TARGET &&
+                       !VARATT_IS_EXTENDED(untoasted_value[i]) &&
+                       (att->attstorage == 'x' || att->attstorage == 'm'))
                {
-                       if (VARATT_IS_EXTERNAL(value[i]))
+                       Datum   cvalue = toast_compress_datum(untoasted_value[i]);
+
+                       if (DatumGetPointer(cvalue) != NULL)
                        {
-                               untoasted_value[i] = PointerGetDatum(
-                                               heap_tuple_fetch_attr(
-                                               (varattrib *)DatumGetPointer(value[i])));
+                               /* successful compression */
+                               if (untoasted_free[i])
+                                       pfree(DatumGetPointer(untoasted_value[i]));
+                               untoasted_value[i] = cvalue;
                                untoasted_free[i] = true;
                        }
-                       else
-                       {
-                               untoasted_value[i] = value[i];
-                               untoasted_free[i] = false;
-                       }
                }
        }
 #endif
@@ -137,10 +157,9 @@ index_formtuple(TupleDesc tupleDescriptor,
         * Here we make sure that the size will fit in the field reserved for
         * it in t_info.
         */
-
        if ((size & INDEX_SIZE_MASK) != size)
-               elog(ERROR, "index_formtuple: data takes %lu bytes: too big",
-                       (unsigned long)size);
+               elog(ERROR, "index_formtuple: data takes %lu bytes, max is %d",
+                        (unsigned long) size, INDEX_SIZE_MASK);
 
        infomask |= size;
 

diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c
index a1497e40f1f0b16bc42fe1c846ff749ece09083a..a3cf6ae7116a7e4dc11eb40752e1dfe74efc9c9f 100644 (file)
--- a/src/backend/access/heap/tuptoaster.c
+++ b/src/backend/access/heap/tuptoaster.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *       $Header: /cvsroot/pgsql/src/backend/access/heap/tuptoaster.c,v 1.16 2001/02/09 17:30:03 tgl Exp $
+ *       $Header: /cvsroot/pgsql/src/backend/access/heap/tuptoaster.c,v 1.17 2001/02/15 20:57:01 tgl Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -45,7 +45,6 @@ static void                   toast_delete(Relation rel, HeapTuple oldtup);
 static void                    toast_delete_datum(Relation rel, Datum value);
 static void                    toast_insert_or_update(Relation rel, HeapTuple newtup,
                                                                HeapTuple oldtup);
-static Datum           toast_compress_datum(Datum value);
 static Datum           toast_save_datum(Relation rel, Oid mainoid, int16 attno, Datum value);
 static varattrib   *toast_fetch_datum(varattrib *attr);
 
@@ -721,7 +720,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
  *     the tuple!
  * ----------
  */
-static Datum
+Datum
 toast_compress_datum(Datum value)
 {
        varattrib          *tmp;

diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h
index 23f31aec522572def9504b8c91623880d3d088bd..05c4b2c1e408eb353ec4b997a5d921dca5f9f34f 100644 (file)
--- a/src/include/access/tuptoaster.h
+++ b/src/include/access/tuptoaster.h
@@ -6,7 +6,7 @@
  *
  * Copyright (c) 2000, PostgreSQL Development Team
  *
- * $Id: tuptoaster.h,v 1.8 2000/08/04 04:16:10 tgl Exp $
+ * $Id: tuptoaster.h,v 1.9 2001/02/15 20:57:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -40,6 +40,12 @@
 
 #define TOAST_TUPLE_TARGET             (MaxTupleSize / 4)
 
+/*
+ * If an index value is larger than TOAST_INDEX_TARGET, we will try to
+ * compress it (we can't move it out-of-line, however).  Note that this
+ * number is per-datum, not per-tuple, for simplicity in index_formtuple().
+ */
+#define TOAST_INDEX_TARGET             (MaxTupleSize / 16)
 
 /*
  * When we store an oversize datum externally, we divide it into chunks
@@ -95,6 +101,14 @@ extern varattrib *heap_tuple_fetch_attr(varattrib * attr);
  */
 extern varattrib *heap_tuple_untoast_attr(varattrib * attr);
 
+/* ----------
+ * toast_compress_datum -
+ *
+ *     Create a compressed version of a varlena datum, if possible
+ * ----------
+ */
+extern Datum toast_compress_datum(Datum value);
+
 #endif  /* TUPLE_TOASTER_ACTIVE */