Fix decoding of MULTI_INSERTs when rows other than the last are toasted.

When decoding the results of a HEAP2_MULTI_INSERT (currently only
generated by COPY FROM) toast columns for all but the last tuple
weren't replaced by their actual contents before being handed to the
output plugin. The reassembled toast datums where disregarded after
every REORDER_BUFFER_CHANGE_(INSERT|UPDATE|DELETE) which is correct
for plain inserts, updates, deletes, but not multi inserts - there we
generate several REORDER_BUFFER_CHANGE_INSERTs for a single
xl_heap_multi_insert record.

To solve the problem add a clear_toast_afterwards boolean to
ReorderBufferChange's union member that's used by modifications. All
row changes but multi_inserts always set that to true, but
multi_insert sets it only for the last change generated.

Add a regression test covering decoding of multi_inserts - there was
none at all before.

Backpatch to 9.4 where logical decoding was introduced.

Bug found by Petr Jelinek.

diff --git a/contrib/test_decoding/expected/toast.out b/contrib/test_decoding/expected/toast.out
index 6adef83f02908b9c886071bbf9d7e024e8130203..322afdb4539b19c08bbe02156c35f4277c34ffc5 100644 (file)
--- a/contrib/test_decoding/expected/toast.out
+++ b/contrib/test_decoding/expected/toast.out
@@ -40,6 +40,14 @@ UPDATE toasted_key SET toasted_col2 = toasted_col1;
 -- test update of a toasted key, changing it
 UPDATE toasted_key SET toasted_key = toasted_key || '1';
 DELETE FROM toasted_key;
+-- Test that HEAP2_MULTI_INSERT insertions with and without toasted
+-- columns are handled correctly
+CREATE TABLE toasted_copy (
+    id int primary key, -- no default, copy didn't use to handle that with multi inserts
+    data text
+);
+ALTER TABLE toasted_copy ALTER COLUMN data SET STORAGE EXTERNAL;
+\copy toasted_copy FROM STDIN
 SELECT substr(data, 1, 200) FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0');
                                                                                                   substr                                                                                                  
 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -80,7 +88,17 @@ SELECT substr(data, 1, 200) FROM pg_logical_slot_get_changes('regression_slot',
  BEGIN
  table public.toasted_key: DELETE: toasted_key[text]:'123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567
  COMMIT
-(37 rows)
+ BEGIN
+ COMMIT
+ BEGIN
+ COMMIT
+ BEGIN
+ table public.toasted_copy: INSERT: id[integer]:1 data[text]:'untoasted1'
+ table public.toasted_copy: INSERT: id[integer]:2 data[text]:'toasted1-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ table public.toasted_copy: INSERT: id[integer]:3 data[text]:'untoasted2'
+ table public.toasted_copy: INSERT: id[integer]:4 data[text]:'toasted2-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ COMMIT
+(47 rows)
 
 SELECT pg_drop_replication_slot('regression_slot');
  pg_drop_replication_slot 

diff --git a/contrib/test_decoding/sql/toast.sql b/contrib/test_decoding/sql/toast.sql
index 943db9d2eedccf2d3b9af34382466b7eb4b6a151..a5f9a5f2597031ae6f9e37e5f9fe5aa212e3d22c 100644 (file)
--- a/contrib/test_decoding/sql/toast.sql
+++ b/contrib/test_decoding/sql/toast.sql
@@ -47,5 +47,18 @@ UPDATE toasted_key SET toasted_key = toasted_key || '1';
 
 DELETE FROM toasted_key;
 
+-- Test that HEAP2_MULTI_INSERT insertions with and without toasted
+-- columns are handled correctly
+CREATE TABLE toasted_copy (
+    id int primary key, -- no default, copy didn't use to handle that with multi inserts
+    data text
+);
+ALTER TABLE toasted_copy ALTER COLUMN data SET STORAGE EXTERNAL;
+\copy toasted_copy FROM STDIN
+1      untoasted1
+2      toasted1-12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+3      untoasted2
+4      toasted2-12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+\.
 SELECT substr(data, 1, 200) FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0');
 SELECT pg_drop_replication_slot('regression_slot');

diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 00b5b838d7cafc81c296773c716bd7b59601abb1..1734ec96599ad85770c01fc5476662893cc45f6d 100644 (file)
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -608,6 +608,8 @@ DecodeInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
                                                change->data.tp.newtuple);
        }
 
+       change->data.tp.clear_toast_afterwards = true;
+
        ReorderBufferQueueChange(ctx->reorder, r->xl_xid, buf->origptr, change);
 }
 
@@ -673,6 +675,8 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
 #endif
        }
 
+       change->data.tp.clear_toast_afterwards = true;
+
        ReorderBufferQueueChange(ctx->reorder, r->xl_xid, buf->origptr, change);
 }
 
@@ -710,6 +714,9 @@ DecodeDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
                                                r->xl_len - SizeOfHeapDelete,
                                                change->data.tp.oldtuple);
        }
+
+       change->data.tp.clear_toast_afterwards = true;
+
        ReorderBufferQueueChange(ctx->reorder, r->xl_xid, buf->origptr, change);
 }
 
@@ -795,6 +802,9 @@ DecodeMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
                        tuple->header.t_hoff = xlhdr->t_hoff;
                }
 
+               /* reset toast reassembly only after the last chunk */
+               change->data.tp.clear_toast_afterwards = (i + 1) == xlrec->ntuples;
+
                ReorderBufferQueueChange(ctx->reorder, r->xl_xid,
                                                                 buf->origptr, change);
        }

diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index 3f5c241d95a6ba7fcd3b47b5edd60861dcef6856..2b0929cb78bd8c19a6632fd3742f507814e94a67 100644 (file)
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -1383,7 +1383,14 @@ ReorderBufferCommit(ReorderBuffer *rb, TransactionId xid,
                                                {
                                                        ReorderBufferToastReplace(rb, txn, relation, change);
                                                        rb->apply_change(rb, txn, relation, change);
-                                                       ReorderBufferToastReset(rb, txn);
+
+                                                       /*
+                                                        * Only clear reassembled toast chunks if we're
+                                                        * sure they're not required anymore. The creator
+                                                        * of the tuple tells us.
+                                                        */
+                                                       if (change->data.tp.clear_toast_afterwards)
+                                                               ReorderBufferToastReset(rb, txn);
                                                }
                                                /* we're not interested in toast deletions */
                                                else if (change->action == REORDER_BUFFER_CHANGE_INSERT)

diff --git a/src/include/replication/reorderbuffer.h b/src/include/replication/reorderbuffer.h
index eaea5884efa32ee4c96357039f8e51d2336fc0c1..7ce0a4221f6d10e1f07f34a690bfe9a7678ca8df 100644 (file)
--- a/src/include/replication/reorderbuffer.h
+++ b/src/include/replication/reorderbuffer.h
@@ -75,6 +75,10 @@ typedef struct ReorderBufferChange
                {
                        /* relation that has been changed */
                        RelFileNode relnode;
+
+                       /* no previously reassembled toast chunks are necessary anymore */
+                       bool clear_toast_afterwards;
+
                        /* valid for DELETE || UPDATE */
                        ReorderBufferTupleBuf *oldtuple;
                        /* valid for INSERT || UPDATE */