Don't choke on files that are removed while pg_rewind runs.

If a file is removed from the source server, while pg_rewind is running, the
invocation of pg_read_binary_file() will fail. Use the just-added missing_ok
option to that function, to have it return NULL instead, and handle that
gracefully. And similarly for pg_ls_dir and pg_stat_file.

Reported by Fujii Masao, fix by Michael Paquier.

diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c
index 37e515bc98476a0b21b5550725b636fb5bca733b..05aa133cf36eaa527f0a331ba9e2ec0d145530c8 100644 (file)
--- a/src/bin/pg_rewind/libpq_fetch.c
+++ b/src/bin/pg_rewind/libpq_fetch.c
@@ -149,14 +149,14 @@ libpqProcessFileList(void)
        sql =
                "WITH RECURSIVE files (path, filename, size, isdir) AS (\n"
                "  SELECT '' AS path, filename, size, isdir FROM\n"
-               "  (SELECT pg_ls_dir('.') AS filename) AS fn,\n"
-               "        pg_stat_file(fn.filename) AS this\n"
+               "  (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n"
+               "        pg_stat_file(fn.filename, true) AS this\n"
                "  UNION ALL\n"
                "  SELECT parent.path || parent.filename || '/' AS path,\n"
                "         fn, this.size, this.isdir\n"
                "  FROM files AS parent,\n"
-               "       pg_ls_dir(parent.path || parent.filename) AS fn,\n"
-               "       pg_stat_file(parent.path || parent.filename || '/' || fn) AS this\n"
+               "       pg_ls_dir(parent.path || parent.filename, true, false) AS fn,\n"
+               "       pg_stat_file(parent.path || parent.filename || '/' || fn, true) AS this\n"
                "       WHERE parent.isdir = 't'\n"
                ")\n"
                "SELECT path || filename, size, isdir,\n"
@@ -183,6 +183,15 @@ libpqProcessFileList(void)
                char       *link_target = PQgetvalue(res, i, 3);
                file_type_t type;
 
+               if (PQgetisnull(res, 0, 1))
+               {
+                       /*
+                        * The file was removed from the server while the query was
+                        * running. Ignore it.
+                        */
+                       continue;
+               }
+
                if (link_target[0])
                        type = FILE_TYPE_SYMLINK;
                else if (isdir)
@@ -259,8 +268,7 @@ receiveFileChunks(const char *sql)
                }
 
                if (PQgetisnull(res, 0, 0) ||
-                       PQgetisnull(res, 0, 1) ||
-                       PQgetisnull(res, 0, 2))
+                       PQgetisnull(res, 0, 1))
                {
                        pg_fatal("unexpected null values in result while fetching remote files\n");
                }
@@ -280,6 +288,21 @@ receiveFileChunks(const char *sql)
 
                chunk = PQgetvalue(res, 0, 2);
 
+               /*
+                * It's possible that the file was deleted on remote side after we
+                * created the file map. In this case simply ignore it, as if it was
+                * not there in the first place, and move on.
+                */
+               if (PQgetisnull(res, 0, 2))
+               {
+                       pg_log(PG_DEBUG,
+                                  "received NULL chunk for file \"%s\", file has been deleted\n",
+                                  filename);
+                       pg_free(filename);
+                       PQclear(res);
+                       continue;
+               }
+
                pg_log(PG_DEBUG, "received chunk for file \"%s\", offset %d, size %d\n",
                           filename, chunkoff, chunksize);
 
@@ -445,7 +468,7 @@ libpq_executeFileMap(filemap_t *map)
         */
        sql =
                "SELECT path, begin, \n"
-               "  pg_read_binary_file(path, begin, len) AS chunk\n"
+               "  pg_read_binary_file(path, begin, len, true) AS chunk\n"
                "FROM fetchchunks\n";
 
        receiveFileChunks(sql);