Fix handling of files that source server removes during pg_rewind is running.

After processing the filemap to build the list of chunks that will be
fetched from the source to rewing the target server, it is possible that
a file which was previously processed is removed from the source.  A
simple example of such an occurence is a WAL segment which gets recycled
on the target in-between.  When the filemap is processed, files not
categorized as relation files are first truncated to prepare for its
full copy of which is going to be taken from the source, divided into a
set of junks.  However, for a recycled WAL segment, this would result in
a segment which has a zero-byte size.  With such an empty file,
post-rewind recovery thinks that records are saved but they are actually
not because of the truncation which happened when processing the
filemap, resulting in data loss.

In order to fix the problem, make sure that files which are found as
removed on the source when receiving chunks of them are as well deleted
on the target server for consistency.

Back-patch to 9.5 where pg_rewind was added.

Author: Tsunakawa Takayuki
Reviewed-by: Michael Paquier
Reported-by: Tsunakawa Takayuki
Discussion: https://postgr.es/m/0A3221C70F24FB45833433255569204D1F8DAAA2%40G01JPEXMBYT05

diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c
index 4229f0e056cfa2b894d8abace295a58e76c582c9..65ee36717951c76d5f8db49602d28a1eef88c587 100644 (file)
--- a/src/bin/pg_rewind/file_ops.c
+++ b/src/bin/pg_rewind/file_ops.c
@@ -29,7 +29,6 @@
 static int     dstfd = -1;
 static char dstpath[MAXPGPATH] = "";
 
-static void remove_target_file(const char *path);
 static void create_target_dir(const char *path);
 static void remove_target_dir(const char *path);
 static void create_target_symlink(const char *path, const char *link);
@@ -134,7 +133,7 @@ remove_target(file_entry_t *entry)
                        break;
 
                case FILE_TYPE_REGULAR:
-                       remove_target_file(entry->path);
+                       remove_target_file(entry->path, false);
                        break;
 
                case FILE_TYPE_SYMLINK:
@@ -165,8 +164,12 @@ create_target(file_entry_t *entry)
        }
 }
 
-static void
-remove_target_file(const char *path)
+/*
+ * Remove a file from target data directory.  If missing_ok is true, it
+ * is fine for the target file to not exist.
+ */
+void
+remove_target_file(const char *path, bool missing_ok)
 {
        char            dstpath[MAXPGPATH];
 
@@ -175,8 +178,13 @@ remove_target_file(const char *path)
 
        snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path);
        if (unlink(dstpath) != 0)
+       {
+               if (errno == ENOENT && missing_ok)
+                       return;
+
                pg_fatal("could not remove file \"%s\": %s\n",
                                 dstpath, strerror(errno));
+       }
 }
 
 void

diff --git a/src/bin/pg_rewind/file_ops.h b/src/bin/pg_rewind/file_ops.h
index f9aea52d6d9eee78475d05e917bb204478174c25..17b67eab676c8eff78bdeb4cc4c55ab1be94bb51 100644 (file)
--- a/src/bin/pg_rewind/file_ops.h
+++ b/src/bin/pg_rewind/file_ops.h
@@ -15,6 +15,7 @@
 extern void open_target_file(const char *path, bool trunc);
 extern void write_target_range(char *buf, off_t begin, size_t size);
 extern void close_target_file(void);
+extern void remove_target_file(const char *path, bool missing_ok);
 extern void truncate_target_file(const char *path, off_t newsize);
 extern void create_target(file_entry_t *t);
 extern void remove_target(file_entry_t *t);

diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c
index b590371e799a01d37542f2dd8b4c57670bc3dac5..89d7886fd1733a75cdf0e0cd358457fd060afd94 100644 (file)
--- a/src/bin/pg_rewind/libpq_fetch.c
+++ b/src/bin/pg_rewind/libpq_fetch.c
@@ -336,15 +336,19 @@ receiveFileChunks(const char *sql)
                chunk = PQgetvalue(res, 0, 2);
 
                /*
-                * It's possible that the file was deleted on remote side after we
-                * created the file map. In this case simply ignore it, as if it was
-                * not there in the first place, and move on.
+                * If a file has been deleted on the source, remove it on the target
+                * as well.  Note that multiple unlink() calls may happen on the same
+                * file if multiple data chunks are associated with it, hence ignore
+                * unconditionally anything missing.  If this file is not a relation
+                * data file, then it has been already truncated when creating the
+                * file chunk list at the previous execution of the filemap.
                 */
                if (PQgetisnull(res, 0, 2))
                {
                        pg_log(PG_DEBUG,
                                   "received null value for chunk for file \"%s\", file has been deleted\n",
                                   filename);
+                       remove_target_file(filename, true);
                        pg_free(filename);
                        PQclear(res);
                        continue;