Improve GIN index build's tracking of memory usage by using

GetMemoryChunkSpace, not just the palloc request size.  This brings the
allocatedMemory counter close enough to reality (as measured by
MemoryContextStats printouts) that I think we can get rid of the arbitrary
factor-of-2 adjustment that was put into the code initially.  Given the
sensitivity of GIN build to work memory size, not using as much of work
memory as we're allowed to seems a pretty bad idea.

diff --git a/src/backend/access/gin/ginbulk.c b/src/backend/access/gin/ginbulk.c
index d0ac0b621213e72c0c3d3f22a80e5c1af76dd39f..09dd0094907e00f9c923708c8a9f82ae63dc29fb 100644 (file)
--- a/src/backend/access/gin/ginbulk.c
+++ b/src/backend/access/gin/ginbulk.c
@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *                     $PostgreSQL: pgsql/src/backend/access/gin/ginbulk.c,v 1.9 2007/02/01 04:16:08 neilc Exp $
+ *                     $PostgreSQL: pgsql/src/backend/access/gin/ginbulk.c,v 1.10 2007/11/16 21:55:59 tgl Exp $
  *-------------------------------------------------------------------------
  */
 
@@ -16,6 +16,7 @@
 
 #include "access/gin.h"
 #include "utils/datum.h"
+#include "utils/memutils.h"
 
 
 #define DEF_NENTRY     2048
@@ -38,7 +39,7 @@ EAAllocate(BuildAccumulator *accum)
        if (accum->entryallocator == NULL || accum->length >= DEF_NENTRY)
        {
                accum->entryallocator = palloc(sizeof(EntryAccumulator) * DEF_NENTRY);
-               accum->allocatedMemory += sizeof(EntryAccumulator) * DEF_NENTRY;
+               accum->allocatedMemory += GetMemoryChunkSpace(accum->entryallocator);
                accum->length = 0;
        }
 
@@ -55,10 +56,11 @@ ginInsertData(BuildAccumulator *accum, EntryAccumulator *entry, ItemPointer heap
 {
        if (entry->number >= entry->length)
        {
-               accum->allocatedMemory += sizeof(ItemPointerData) * entry->length;
+               accum->allocatedMemory -= GetMemoryChunkSpace(entry->list);
                entry->length *= 2;
                entry->list = (ItemPointerData *) repalloc(entry->list,
                                                                        sizeof(ItemPointerData) * entry->length);
+               accum->allocatedMemory += GetMemoryChunkSpace(entry->list);
        }
 
        if (entry->shouldSort == FALSE)
@@ -95,10 +97,10 @@ getDatumCopy(BuildAccumulator *accum, Datum value)
                realSize = datumGetSize(value, false, att[0]->attlen);
 
                s = (char *) palloc(realSize);
+               accum->allocatedMemory += GetMemoryChunkSpace(s);
+
                memcpy(s, DatumGetPointer(value), realSize);
                res = PointerGetDatum(s);
-
-               accum->allocatedMemory += realSize;
        }
        return res;
 }
@@ -143,8 +145,8 @@ ginInsertEntry(BuildAccumulator *accum, ItemPointer heapptr, Datum entry)
                ea->number = 1;
                ea->shouldSort = FALSE;
                ea->list = (ItemPointerData *) palloc(sizeof(ItemPointerData) * DEF_NPTR);
+               accum->allocatedMemory += GetMemoryChunkSpace(ea->list);
                ea->list[0] = *heapptr;
-               accum->allocatedMemory += sizeof(ItemPointerData) * DEF_NPTR;
 
                if (pea == NULL)
                        accum->entries = ea;
@@ -270,11 +272,11 @@ ginGetEntry(BuildAccumulator *accum, Datum *value, uint32 *n)
        EntryAccumulator *entry;
        ItemPointerData *list;
 
-
        if (accum->stack == NULL)
        {
                /* first call */
                accum->stack = palloc0(sizeof(EntryAccumulator *) * (accum->maxdepth + 1));
+               accum->allocatedMemory += GetMemoryChunkSpace(accum->stack);
                entry = accum->entries;
 
                if (entry == NULL)
@@ -295,6 +297,7 @@ ginGetEntry(BuildAccumulator *accum, Datum *value, uint32 *n)
        }
        else
        {
+               accum->allocatedMemory -= GetMemoryChunkSpace(accum->stack[accum->stackpos]->list);
                pfree(accum->stack[accum->stackpos]->list);
                accum->stack[accum->stackpos]->list = NULL;
                entry = walkTree(accum);

diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c
index db5fa68338ab414c1deabc79ed5e9fd2dc29d082..8afd8afd313f05c7614ee5bb94bc61cb5e7a4d7f 100644 (file)
--- a/src/backend/access/gin/gininsert.c
+++ b/src/backend/access/gin/gininsert.c
@@ -8,17 +8,19 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *                     $PostgreSQL: pgsql/src/backend/access/gin/gininsert.c,v 1.9 2007/06/05 12:47:49 teodor Exp $
+ *                     $PostgreSQL: pgsql/src/backend/access/gin/gininsert.c,v 1.10 2007/11/16 21:55:59 tgl Exp $
  *-------------------------------------------------------------------------
  */
 
 #include "postgres.h"
+
 #include "access/genam.h"
 #include "access/gin.h"
 #include "catalog/index.h"
 #include "miscadmin.h"
 #include "utils/memutils.h"
 
+
 typedef struct
 {
        GinState        ginstate;
@@ -132,8 +134,7 @@ addItemPointersToTuple(Relation index, GinState *ginstate, GinBtreeStack *stack,
 }
 
 /*
- * Inserts only one entry to the index, but it can adds more that 1
- * ItemPointer.
+ * Inserts only one entry to the index, but it can add more than 1 ItemPointer.
  */
 static void
 ginEntryInsert(Relation index, GinState *ginstate, Datum value, ItemPointerData *items, uint32 nitem, bool isBuild)
@@ -198,7 +199,7 @@ ginEntryInsert(Relation index, GinState *ginstate, Datum value, ItemPointerData
 
 /*
  * Saves indexed value in memory accumulator during index creation
- * Function isn't use during normal insert
+ * Function isn't used during normal insert
  */
 static uint32
 ginHeapTupleBulkInsert(GinBuildState *buildstate, Datum value, ItemPointer heapptr)
@@ -226,7 +227,6 @@ static void
 ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
                                 bool *isnull, bool tupleIsAlive, void *state)
 {
-
        GinBuildState *buildstate = (GinBuildState *) state;
        MemoryContext oldCtx;
 
@@ -237,11 +237,8 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
 
        buildstate->indtuples += ginHeapTupleBulkInsert(buildstate, *values, &htup->t_self);
 
-       /*
-        * we use only half maintenance_work_mem, because there is some leaks
-        * during insertion and extract values
-        */
-       if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L / 2L)
+       /* If we've maxed out our available memory, dump everything to the index */
+       if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)
        {
                ItemPointerData *list;
                Datum           entry;