Allow usage of huge maintenance_work_mem for GIN build.

Currently, in-memory posting list during GIN build process is limited 1GB
because of using repalloc. The patch replaces call of repalloc to repalloc_huge.
It increases limit of posting list from 180 millions
(1GB / sizeof(ItemPointerData)) to 4 billions limited by maxcount/count fields
in GinEntryAccumulator and subsequent calls. Check added.

Also, fix accounting of allocatedMemory during build to prevent integer
overflow with maintenance_work_mem > 4GB.

Robert Abraham <robert.abraham86@googlemail.com> with additions by me

diff --git a/src/backend/access/gin/ginbulk.c b/src/backend/access/gin/ginbulk.c
index e40e5be36bc77f2c6b7cc0863a0aaa93f08fd529..5a8546998d53c8335a9119d34cea6141c6a2d8c1 100644 (file)
--- a/src/backend/access/gin/ginbulk.c
+++ b/src/backend/access/gin/ginbulk.c
@@ -14,6 +14,8 @@
 
 #include "postgres.h"
 
+#include <limits.h>
+
 #include "access/gin_private.h"
 #include "utils/datum.h"
 #include "utils/memutils.h"
@@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg)
         */
        if (eo->count >= eo->maxcount)
        {
+               if (eo->maxcount > INT_MAX)
+                       ereport(ERROR,
+                                       (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+                                        errmsg("posting list is too long"),
+                                        errhint("Reduce maintenance_work_mem")));
+
                accum->allocatedMemory -= GetMemoryChunkSpace(eo->list);
                eo->maxcount *= 2;
                eo->list = (ItemPointerData *)
-                       repalloc(eo->list, sizeof(ItemPointerData) * eo->maxcount);
+                       repalloc_huge(eo->list, sizeof(ItemPointerData) * eo->maxcount);
                accum->allocatedMemory += GetMemoryChunkSpace(eo->list);
        }
 

diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c
index c5732c3dc030ad6be789d424a09a09de9b216c48..54e593d8a974d5de9b73d00ba97a57f549bd1c86 100644 (file)
--- a/src/backend/access/gin/ginfast.c
+++ b/src/backend/access/gin/ginfast.c
@@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate,
                 */
                if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber ||
                        (GinPageHasFullRow(page) &&
-                        (accum.allocatedMemory >= maintenance_work_mem * 1024L)))
+                        (accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)))
                {
                        ItemPointerData *list;
                        uint32          nlist;

diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c
index fc44f0205d07e3611c7a41437c2ef5d3c1958fb7..49e91853c3cec33159f8a3c64be79ed8ee60a6c5 100644 (file)
--- a/src/backend/access/gin/gininsert.c
+++ b/src/backend/access/gin/gininsert.c
@@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values,
                                                           &htup->t_self);
 
        /* If we've maxed out our available memory, dump everything to the index */
-       if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L)
+       if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L)
        {
                ItemPointerData *list;
                Datum           key;

diff --git a/src/include/access/gin_private.h b/src/include/access/gin_private.h
index 5095fc1dba85061970bf6cfdc8a2522ed5d6c3e7..acbe36a74147e64a5c888de409b8197a24baae79 100644 (file)
--- a/src/include/access/gin_private.h
+++ b/src/include/access/gin_private.h
@@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator
 typedef struct
 {
        GinState   *ginstate;
-       long            allocatedMemory;
+       Size            allocatedMemory;
        GinEntryAccumulator *entryallocator;
        uint32          eas_used;
        RBTree     *tree;