Fix bogus "out of memory" reports in tuplestore.c.

The tuplesort/tuplestore memory management logic assumed that the chunk
allocation overhead for its memtuples array could not increase when
increasing the array size.  This is and always was true for tuplesort,
but we (I, I think) blindly copied that logic into tuplestore.c without
noticing that the assumption failed to hold for the much smaller array
elements used by tuplestore.  Given rather small work_mem, this could
result in an improper complaint about "unexpected out-of-memory situation",
as reported by Brent DeSpain in bug #13530.

The easiest way to fix this is just to increase tuplestore's initial
array size so that the assumption holds.  Rather than relying on magic
constants, though, let's export a #define from aset.c that represents
the safe allocation threshold, and make tuplestore's calculation depend
on that.

Do the same in tuplesort.c to keep the logic looking parallel, even though
tuplesort.c isn't actually at risk at present.  This will keep us from
breaking it if we ever muck with the allocation parameters in aset.c.

Back-patch to all supported versions.  The error message doesn't occur
pre-9.3, not so much because the problem can't happen as because the
pre-9.3 tuplestore code neglected to check for it.  (The chance of
trouble is a great deal larger as of 9.3, though, due to changes in the
array-size-increasing strategy.)  However, allowing LACKMEM() to become
true unexpectedly could still result in less-than-desirable behavior,
so let's patch it all the way back.

diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c
index 81b734a6340be24ed456610f6dc06a210c95892e..e3571a28a21d743fe9bea34676cc2c45a48b7946 100644 (file)
--- a/src/backend/utils/mmgr/aset.c
+++ b/src/backend/utils/mmgr/aset.c
@@ -89,9 +89,9 @@
  *
  * With the current parameters, request sizes up to 8K are treated as chunks,
  * larger requests go into dedicated blocks.  Change ALLOCSET_NUM_FREELISTS
- * to adjust the boundary point.  (But in contexts with small maxBlockSize,
- * we may set the allocChunkLimit to less than 8K, so as to avoid space
- * wastage.)
+ * to adjust the boundary point; and adjust ALLOCSET_SEPARATE_THRESHOLD in
+ * memutils.h to agree.  (Note: in contexts with small maxBlockSize, we may
+ * set the allocChunkLimit to less than 8K, so as to avoid space wastage.)
  *--------------------
  */
 
@@ -393,7 +393,11 @@ AllocSetContextCreate(MemoryContext parent,
         * We have to have allocChunkLimit a power of two, because the requested
         * and actually-allocated sizes of any chunk must be on the same side of
         * the limit, else we get confused about whether the chunk is "big".
+        *
+        * Also, allocChunkLimit must not exceed ALLOCSET_SEPARATE_THRESHOLD.
         */
+       Assert(ALLOC_CHUNK_LIMIT == ALLOCSET_SEPARATE_THRESHOLD);
+
        context->allocChunkLimit = ALLOC_CHUNK_LIMIT;
        while ((Size) (context->allocChunkLimit + ALLOC_CHUNKHDRSZ) >
                   (Size) ((maxBlockSize - ALLOC_BLOCKHDRSZ) / ALLOC_CHUNK_FRACTION))

diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 8d9f1e79416a1a583d7c421b6747191683c4e4c9..69042911ceac0e7da9e1cff9a9696f9accceb421 100644 (file)
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -568,7 +568,14 @@ tuplesort_begin_common(int workMem, bool randomAccess)
        state->tapeset = NULL;
 
        state->memtupcount = 0;
-       state->memtupsize = 1024;       /* initial guess */
+
+       /*
+        * Initial size of array must be more than ALLOCSET_SEPARATE_THRESHOLD;
+        * see comments in grow_memtuples().
+        */
+       state->memtupsize = Max(1024,
+                                               ALLOCSET_SEPARATE_THRESHOLD / sizeof(SortTuple) + 1);
+
        state->memtuples = (SortTuple *) palloc(state->memtupsize * sizeof(SortTuple));
 
        USEMEM(state, GetMemoryChunkSpace(state->memtuples));
@@ -990,7 +997,7 @@ grow_memtuples(Tuplesortstate *state)
                                 state->memtupsize * sizeof(SortTuple));
        USEMEM(state, GetMemoryChunkSpace(state->memtuples));
        if (LACKMEM(state))
-               elog(ERROR, "unexpected out-of-memory situation during sort");
+               elog(ERROR, "unexpected out-of-memory situation in tuplesort");
        return true;
 }
 

diff --git a/src/backend/utils/sort/tuplestore.c b/src/backend/utils/sort/tuplestore.c
index a216992a7a3f6f51933f4f25c2485c6bc6d171df..1db2fa4e526be18b286dbd499f7303e97045a81c 100644 (file)
--- a/src/backend/utils/sort/tuplestore.c
+++ b/src/backend/utils/sort/tuplestore.c
@@ -260,7 +260,14 @@ tuplestore_begin_common(int eflags, bool interXact, int maxKBytes)
 
        state->memtupdeleted = 0;
        state->memtupcount = 0;
-       state->memtupsize = 1024;       /* initial guess */
+
+       /*
+        * Initial size of array must be more than ALLOCSET_SEPARATE_THRESHOLD;
+        * see comments in grow_memtuples().
+        */
+       state->memtupsize = Max(16384 / sizeof(void *),
+                                                       ALLOCSET_SEPARATE_THRESHOLD / sizeof(void *) + 1);
+
        state->memtuples = (void **) palloc(state->memtupsize * sizeof(void *));
 
        USEMEM(state, GetMemoryChunkSpace(state->memtuples));
@@ -643,6 +650,8 @@ tuplestore_puttuple_common(Tuplestorestate *state, void *tuple)
                                                repalloc(state->memtuples,
                                                                 state->memtupsize * sizeof(void *));
                                        USEMEM(state, GetMemoryChunkSpace(state->memtuples));
+                                       if (LACKMEM(state))
+                                               elog(ERROR, "unexpected out-of-memory situation in tuplestore");
                                }
                        }
 

diff --git a/src/include/utils/memutils.h b/src/include/utils/memutils.h
index aee23d3d2ecf3234501f0c722c6edc986773d242..84bb062f1d67a34be4e6895a9e6057ecf1a19d06 100644 (file)
--- a/src/include/utils/memutils.h
+++ b/src/include/utils/memutils.h
@@ -141,4 +141,12 @@ extern MemoryContext AllocSetContextCreate(MemoryContext parent,
 #define ALLOCSET_SMALL_INITSIZE  (1 * 1024)
 #define ALLOCSET_SMALL_MAXSIZE  (8 * 1024)
 
+/*
+ * Threshold above which a request in an AllocSet context is certain to be
+ * allocated separately (and thereby have constant allocation overhead).
+ * Few callers should be interested in this, but tuplesort/tuplestore need
+ * to know it.
+ */
+#define ALLOCSET_SEPARATE_THRESHOLD  8192
+
 #endif   /* MEMUTILS_H */