-/*-------------------------------------------------------------------------\r
- *\r
- * pg_backup_custom.c\r
- *\r
- * Implements the custom output format.\r
- *\r
- * See the headers to pg_restore for more details.\r
- *\r
- * Copyright (c) 2000, Philip Warner\r
- * Rights are granted to use this software in any way so long\r
- * as this notice is not removed.\r
- *\r
- * The author is not responsible for loss or damages that may\r
- * result from it's use.\r
- *\r
- *\r
- * IDENTIFICATION\r
- *\r
- * Modifications - 28-Jun-2000 - pjw@rhyme.com.au\r
- *\r
- * Initial version. \r
- *\r
- *-------------------------------------------------------------------------\r
- */\r
-\r
-#include <stdlib.h>\r
-#include "pg_backup.h"\r
-#include "pg_backup_archiver.h"\r
-\r
-extern int errno;\r
-\r
-static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te);\r
-static void _StartData(ArchiveHandle* AH, TocEntry* te);\r
-static int _WriteData(ArchiveHandle* AH, const void* data, int dLen);\r
-static void _EndData(ArchiveHandle* AH, TocEntry* te);\r
-static int _WriteByte(ArchiveHandle* AH, const int i);\r
-static int _ReadByte(ArchiveHandle* );\r
-static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len);\r
-static int _ReadBuf(ArchiveHandle* AH, void* buf, int len);\r
-static void _CloseArchive(ArchiveHandle* AH);\r
-static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt);\r
-static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te);\r
-static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te);\r
-static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te);\r
-\r
-static void _PrintData(ArchiveHandle* AH);\r
-static void _skipData(ArchiveHandle* AH);\r
-\r
-#define zlibOutSize 4096\r
-#define zlibInSize 4096\r
-\r
-typedef struct {\r
- z_streamp zp;\r
- char* zlibOut;\r
- char* zlibIn;\r
- int inSize;\r
- int hasSeek;\r
- int filePos;\r
- int dataStart;\r
-} lclContext;\r
-\r
-typedef struct {\r
- int dataPos;\r
- int dataLen;\r
-} lclTocEntry;\r
-\r
-static int _getFilePos(ArchiveHandle* AH, lclContext* ctx);\r
-\r
-static char* progname = "Archiver(custom)";\r
-\r
-/*\r
- * Handler functions. \r
- */\r
-void InitArchiveFmt_Custom(ArchiveHandle* AH) \r
-{\r
- lclContext* ctx;\r
-\r
- /* Assuming static functions, this can be copied for each format. */\r
- AH->ArchiveEntryPtr = _ArchiveEntry;\r
- AH->StartDataPtr = _StartData;\r
- AH->WriteDataPtr = _WriteData;\r
- AH->EndDataPtr = _EndData;\r
- AH->WriteBytePtr = _WriteByte;\r
- AH->ReadBytePtr = _ReadByte;\r
- AH->WriteBufPtr = _WriteBuf;\r
- AH->ReadBufPtr = _ReadBuf;\r
- AH->ClosePtr = _CloseArchive;\r
- AH->PrintTocDataPtr = _PrintTocData;\r
- AH->ReadExtraTocPtr = _ReadExtraToc;\r
- AH->WriteExtraTocPtr = _WriteExtraToc;\r
- AH->PrintExtraTocPtr = _PrintExtraToc;\r
-\r
- /*\r
- * Set up some special context used in compressing data.\r
- */\r
- ctx = (lclContext*)malloc(sizeof(lclContext));\r
- if (ctx == NULL)\r
- die_horribly("%s: Unable to allocate archive context",progname);\r
- AH->formatData = (void*)ctx;\r
-\r
- ctx->zp = (z_streamp)malloc(sizeof(z_stream));\r
- if (ctx->zp == NULL)\r
- die_horribly("%s: unable to allocate zlib stream archive context",progname);\r
-\r
- ctx->zlibOut = (char*)malloc(zlibOutSize);\r
- ctx->zlibIn = (char*)malloc(zlibInSize);\r
- ctx->inSize = zlibInSize;\r
- ctx->filePos = 0;\r
-\r
- if (ctx->zlibOut == NULL || ctx->zlibIn == NULL)\r
- die_horribly("%s: unable to allocate buffers in archive context",progname);\r
-\r
- /*\r
- * Now open the file\r
- */\r
- if (AH->mode == archModeWrite) {\r
- if (AH->fSpec && strcmp(AH->fSpec,"") != 0) {\r
- AH->FH = fopen(AH->fSpec, PG_BINARY_W);\r
- } else {\r
- AH->FH = stdout;\r
- }\r
-\r
- if (!AH)\r
- die_horribly("%s: unable to open archive file %s",progname, AH->fSpec);\r
-\r
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);\r
-\r
- } else {\r
- if (AH->fSpec && strcmp(AH->fSpec,"") != 0) {\r
- AH->FH = fopen(AH->fSpec, PG_BINARY_R);\r
- } else {\r
- AH->FH = stdin;\r
- }\r
- if (!AH)\r
- die_horribly("%s: unable to open archive file %s",progname, AH->fSpec);\r
-\r
- ctx->hasSeek = (fseek(AH->FH, 0, SEEK_CUR) == 0);\r
-\r
- ReadHead(AH);\r
- ReadToc(AH);\r
- ctx->dataStart = _getFilePos(AH, ctx);\r
- }\r
-\r
-}\r
-\r
-/*\r
- * - Start a new TOC entry\r
-*/\r
-static void _ArchiveEntry(ArchiveHandle* AH, TocEntry* te) \r
-{\r
- lclTocEntry* ctx;\r
-\r
- ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry));\r
- if (te->dataDumper) {\r
- ctx->dataPos = -1;\r
- } else {\r
- ctx->dataPos = 0;\r
- }\r
- ctx->dataLen = 0;\r
- te->formatData = (void*)ctx;\r
-\r
-}\r
-\r
-static void _WriteExtraToc(ArchiveHandle* AH, TocEntry* te)\r
-{\r
- lclTocEntry* ctx = (lclTocEntry*)te->formatData;\r
-\r
- WriteInt(AH, ctx->dataPos);\r
- WriteInt(AH, ctx->dataLen);\r
-}\r
-\r
-static void _ReadExtraToc(ArchiveHandle* AH, TocEntry* te)\r
-{\r
- lclTocEntry* ctx = (lclTocEntry*)te->formatData;\r
-\r
- if (ctx == NULL) {\r
- ctx = (lclTocEntry*)malloc(sizeof(lclTocEntry));\r
- te->formatData = (void*)ctx;\r
- }\r
-\r
- ctx->dataPos = ReadInt( AH );\r
- ctx->dataLen = ReadInt( AH );\r
- \r
-}\r
-\r
-static void _PrintExtraToc(ArchiveHandle* AH, TocEntry* te)\r
-{\r
- lclTocEntry* ctx = (lclTocEntry*)te->formatData;\r
-\r
- ahprintf(AH, "-- Data Pos: %d (Length %d)\n", ctx->dataPos, ctx->dataLen);\r
-}\r
-\r
-static void _StartData(ArchiveHandle* AH, TocEntry* te)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- z_streamp zp = ctx->zp;\r
- lclTocEntry* tctx = (lclTocEntry*)te->formatData;\r
-\r
- tctx->dataPos = _getFilePos(AH, ctx);\r
-\r
- WriteInt(AH, te->id); /* For sanity check */\r
-\r
-#ifdef HAVE_LIBZ\r
-\r
- if (AH->compression < 0 || AH->compression > 9) {\r
- AH->compression = Z_DEFAULT_COMPRESSION;\r
- }\r
-\r
- if (AH->compression != 0) {\r
- zp->zalloc = Z_NULL;\r
- zp->zfree = Z_NULL;\r
- zp->opaque = Z_NULL;\r
-\r
- if (deflateInit(zp, AH->compression) != Z_OK)\r
- die_horribly("%s: could not initialize compression library - %s\n",progname, zp->msg);\r
- }\r
-\r
-#else\r
-\r
- AH->compression = 0;\r
-\r
-#endif\r
-\r
- /* Just be paranoid - maye End is called after Start, with no Write */\r
- zp->next_out = ctx->zlibOut;\r
- zp->avail_out = zlibOutSize;\r
-}\r
-\r
-static int _DoDeflate(ArchiveHandle* AH, lclContext* ctx, int flush) \r
-{\r
- z_streamp zp = ctx->zp;\r
-\r
-#ifdef HAVE_LIBZ\r
- char* out = ctx->zlibOut;\r
- int res = Z_OK;\r
-\r
- if (AH->compression != 0) \r
- {\r
- res = deflate(zp, flush);\r
- if (res == Z_STREAM_ERROR)\r
- die_horribly("%s: could not compress data - %s\n",progname, zp->msg);\r
-\r
- if ( ( (flush == Z_FINISH) && (zp->avail_out < zlibOutSize) )\r
- || (zp->avail_out == 0) \r
- || (zp->avail_in != 0)\r
- ) \r
- {\r
- /*\r
- * Extra paranoia: avoid zero-length chunks since a zero \r
- * length chunk is the EOF marker. This should never happen\r
- * but...\r
- */\r
- if (zp->avail_out < zlibOutSize) {\r
- /* printf("Wrote %d byte deflated chunk\n", zlibOutSize - zp->avail_out); */\r
- WriteInt(AH, zlibOutSize - zp->avail_out);\r
- fwrite(out, 1, zlibOutSize - zp->avail_out, AH->FH);\r
- ctx->filePos += zlibOutSize - zp->avail_out;\r
- }\r
- zp->next_out = out;\r
- zp->avail_out = zlibOutSize;\r
- }\r
- } else {\r
-#endif\r
- if (zp->avail_in > 0)\r
- {\r
- WriteInt(AH, zp->avail_in);\r
- fwrite(zp->next_in, 1, zp->avail_in, AH->FH);\r
- ctx->filePos += zp->avail_in;\r
- zp->avail_in = 0;\r
- } else {\r
-#ifdef HAVE_LIBZ\r
- if (flush == Z_FINISH)\r
- res = Z_STREAM_END;\r
-#endif\r
- }\r
-\r
-\r
-#ifdef HAVE_LIBZ\r
- }\r
-\r
- return res;\r
-#else\r
- return 1;\r
-#endif\r
-\r
-}\r
-\r
-static int _WriteData(ArchiveHandle* AH, const void* data, int dLen)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- z_streamp zp = ctx->zp;\r
-\r
- zp->next_in = (void*)data;\r
- zp->avail_in = dLen;\r
-\r
- while (zp->avail_in != 0) {\r
- /* printf("Deflating %d bytes\n", dLen); */\r
- _DoDeflate(AH, ctx, 0);\r
- }\r
- return dLen;\r
-}\r
-\r
-static void _EndData(ArchiveHandle* AH, TocEntry* te)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- lclTocEntry* tctx = (lclTocEntry*) te->formatData;\r
-\r
-#ifdef HAVE_LIBZ\r
- z_streamp zp = ctx->zp;\r
- int res;\r
-\r
- if (AH->compression != 0)\r
- {\r
- zp->next_in = NULL;\r
- zp->avail_in = 0;\r
-\r
- do { \r
- /* printf("Ending data output\n"); */\r
- res = _DoDeflate(AH, ctx, Z_FINISH);\r
- } while (res != Z_STREAM_END);\r
-\r
- if (deflateEnd(zp) != Z_OK)\r
- die_horribly("%s: error closing compression stream - %s\n", progname, zp->msg);\r
- }\r
-#endif\r
-\r
- /* Send the end marker */\r
- WriteInt(AH, 0);\r
-\r
- tctx->dataLen = _getFilePos(AH, ctx) - tctx->dataPos;\r
-\r
-}\r
-\r
-/*\r
- * Print data for a gievn TOC entry\r
-*/\r
-static void _PrintTocData(ArchiveHandle* AH, TocEntry* te, RestoreOptions *ropt)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- int id;\r
- lclTocEntry* tctx = (lclTocEntry*) te->formatData;\r
-\r
- if (tctx->dataPos == 0) \r
- return;\r
-\r
- if (!ctx->hasSeek || tctx->dataPos < 0) {\r
- id = ReadInt(AH);\r
-\r
- while (id != te->id) {\r
- if (TocIDRequired(AH, id, ropt) & 2)\r
- die_horribly("%s: Dumping a specific TOC data block out of order is not supported"\r
- " without on this input stream (fseek required)\n", progname);\r
- _skipData(AH);\r
- id = ReadInt(AH);\r
- }\r
- } else {\r
-\r
- if (fseek(AH->FH, tctx->dataPos, SEEK_SET) != 0)\r
- die_horribly("%s: error %d in file seek\n",progname, errno);\r
-\r
- id = ReadInt(AH);\r
-\r
- }\r
-\r
- if (id != te->id)\r
- die_horribly("%s: Found unexpected block ID (%d) when reading data - expected %d\n",\r
- progname, id, te->id);\r
-\r
- ahprintf(AH, "--\n-- Data for TOC Entry ID %d (OID %s) %s %s\n--\n\n",\r
- te->id, te->oid, te->desc, te->name);\r
-\r
- _PrintData(AH);\r
-\r
- ahprintf(AH, "\n\n");\r
-}\r
-\r
-/*\r
- * Print data from current file position.\r
-*/\r
-static void _PrintData(ArchiveHandle* AH)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- z_streamp zp = ctx->zp;\r
- int blkLen;\r
- char* in = ctx->zlibIn;\r
- int cnt;\r
-\r
-#ifdef HAVE_LIBZ\r
-\r
- int res;\r
- char* out = ctx->zlibOut;\r
-\r
- res = Z_OK;\r
-\r
- if (AH->compression != 0) {\r
- zp->zalloc = Z_NULL;\r
- zp->zfree = Z_NULL;\r
- zp->opaque = Z_NULL;\r
-\r
- if (inflateInit(zp) != Z_OK)\r
- die_horribly("%s: could not initialize compression library - %s\n", progname, zp->msg);\r
- }\r
-\r
-#endif\r
-\r
- blkLen = ReadInt(AH);\r
- while (blkLen != 0) {\r
- if (blkLen > ctx->inSize) {\r
- free(ctx->zlibIn);\r
- ctx->zlibIn = NULL;\r
- ctx->zlibIn = (char*)malloc(blkLen);\r
- if (!ctx->zlibIn)\r
- die_horribly("%s: failed to allocate decompression buffer\n", progname);\r
-\r
- ctx->inSize = blkLen;\r
- in = ctx->zlibIn;\r
- }\r
- cnt = fread(in, 1, blkLen, AH->FH);\r
- if (cnt != blkLen) \r
- die_horribly("%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt);\r
-\r
- ctx->filePos += blkLen;\r
-\r
- zp->next_in = in;\r
- zp->avail_in = blkLen;\r
-\r
-#ifdef HAVE_LIBZ\r
-\r
- if (AH->compression != 0) {\r
-\r
- while (zp->avail_in != 0) {\r
- zp->next_out = out;\r
- zp->avail_out = zlibOutSize;\r
- res = inflate(zp, 0);\r
- if (res != Z_OK && res != Z_STREAM_END)\r
- die_horribly("%s: unable to uncompress data - %s\n", progname, zp->msg);\r
-\r
- out[zlibOutSize - zp->avail_out] = '\0';\r
- ahwrite(out, 1, zlibOutSize - zp->avail_out, AH);\r
- }\r
- } else {\r
-#endif\r
- ahwrite(in, 1, zp->avail_in, AH);\r
- zp->avail_in = 0;\r
-\r
-#ifdef HAVE_LIBZ\r
- }\r
-#endif\r
-\r
- blkLen = ReadInt(AH);\r
- }\r
-\r
-#ifdef HAVE_LIBZ\r
- if (AH->compression != 0) \r
- {\r
- zp->next_in = NULL;\r
- zp->avail_in = 0;\r
- while (res != Z_STREAM_END) {\r
- zp->next_out = out;\r
- zp->avail_out = zlibOutSize;\r
- res = inflate(zp, 0);\r
- if (res != Z_OK && res != Z_STREAM_END)\r
- die_horribly("%s: unable to uncompress data - %s\n", progname, zp->msg);\r
-\r
- out[zlibOutSize - zp->avail_out] = '\0';\r
- ahwrite(out, 1, zlibOutSize - zp->avail_out, AH);\r
- }\r
- }\r
-#endif\r
-\r
-}\r
-\r
-/*\r
- * Skip data from current file position.\r
-*/\r
-static void _skipData(ArchiveHandle* AH)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- int blkLen;\r
- char* in = ctx->zlibIn;\r
- int cnt;\r
-\r
- blkLen = ReadInt(AH);\r
- while (blkLen != 0) {\r
- if (blkLen > ctx->inSize) {\r
- free(ctx->zlibIn);\r
- ctx->zlibIn = (char*)malloc(blkLen);\r
- ctx->inSize = blkLen;\r
- in = ctx->zlibIn;\r
- }\r
- cnt = fread(in, 1, blkLen, AH->FH);\r
- if (cnt != blkLen) \r
- die_horribly("%s: could not read data block - expected %d, got %d\n", progname, blkLen, cnt);\r
-\r
- ctx->filePos += blkLen;\r
-\r
- blkLen = ReadInt(AH);\r
- }\r
-\r
-}\r
-\r
-static int _WriteByte(ArchiveHandle* AH, const int i)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- int res;\r
-\r
- res = fputc(i, AH->FH);\r
- if (res != EOF) {\r
- ctx->filePos += 1;\r
- }\r
- return res;\r
-}\r
-\r
-static int _ReadByte(ArchiveHandle* AH)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- int res;\r
-\r
- res = fgetc(AH->FH);\r
- if (res != EOF) {\r
- ctx->filePos += 1;\r
- }\r
- return res;\r
-}\r
-\r
-static int _WriteBuf(ArchiveHandle* AH, const void* buf, int len)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- int res;\r
- res = fwrite(buf, 1, len, AH->FH);\r
- ctx->filePos += res;\r
- return res;\r
-}\r
-\r
-static int _ReadBuf(ArchiveHandle* AH, void* buf, int len)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- int res;\r
- res = fread(buf, 1, len, AH->FH);\r
- ctx->filePos += res;\r
- return res;\r
-}\r
-\r
-static void _CloseArchive(ArchiveHandle* AH)\r
-{\r
- lclContext* ctx = (lclContext*)AH->formatData;\r
- int tpos;\r
-\r
- if (AH->mode == archModeWrite) {\r
- WriteHead(AH);\r
- tpos = ftell(AH->FH);\r
- WriteToc(AH);\r
- ctx->dataStart = _getFilePos(AH, ctx);\r
- WriteDataChunks(AH);\r
- /* This is not an essential operation - it is really only\r
- * needed if we expect to be doing seeks to read the data back\r
- * - it may be ok to just use the existing self-consistent block\r
- * formatting.\r
- */\r
- if (ctx->hasSeek) {\r
- fseek(AH->FH, tpos, SEEK_SET);\r
- WriteToc(AH);\r
- }\r
- }\r
-\r
- fclose(AH->FH);\r
- AH->FH = NULL; \r
-}\r
-\r
-static int _getFilePos(ArchiveHandle* AH, lclContext* ctx) \r
-{\r
- int pos;\r
- if (ctx->hasSeek) {\r
- pos = ftell(AH->FH);\r
- if (pos != ctx->filePos) {\r
- fprintf(stderr, "Warning: ftell mismatch with filePos\n");\r
- }\r
- } else {\r
- pos = ctx->filePos;\r
- }\r
- return pos;\r
-}\r
-\r
-\r
+/*-------------------------------------------------------------------------
+ *
+ * pg_backup_custom.c
+ *
+ * Implements the custom output format.
+ *
+ * The comments with the routined in this code are a good place to
+ * understand how to write a new format.
+ *
+ * See the headers to pg_restore for more details.
+ *
+ * Copyright (c) 2000, Philip Warner
+ * Rights are granted to use this software in any way so long
+ * as this notice is not removed.
+ *
+ * The author is not responsible for loss or damages that may
+ * and any liability will be limited to the time taken to fix any
+ * related bug.
+ *
+ *
+ * IDENTIFICATION
+ * src/bin/pg_dump/pg_backup_custom.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "compress_io.h"
+#include "common.h"
+
+/*--------
+ * Routines in the format interface
+ *--------
+ */
+
+static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
+static void _StartData(ArchiveHandle *AH, TocEntry *te);
+static size_t _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
+static void _EndData(ArchiveHandle *AH, TocEntry *te);
+static int _WriteByte(ArchiveHandle *AH, const int i);
+static int _ReadByte(ArchiveHandle *);
+static size_t _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
+static size_t _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
+static void _CloseArchive(ArchiveHandle *AH);
+static void _ReopenArchive(ArchiveHandle *AH);
+static void _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt);
+static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
+static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
+static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
+
+static void _PrintData(ArchiveHandle *AH);
+static void _skipData(ArchiveHandle *AH);
+static void _skipBlobs(ArchiveHandle *AH);
+
+static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
+static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
+static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
+static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
+static void _LoadBlobs(ArchiveHandle *AH, bool drop);
+static void _Clone(ArchiveHandle *AH);
+static void _DeClone(ArchiveHandle *AH);
+
+typedef struct
+{
+ CompressorState *cs;
+ int hasSeek;
+ pgoff_t filePos;
+ pgoff_t dataStart;
+} lclContext;
+
+typedef struct
+{
+ int dataState;
+ pgoff_t dataPos;
+} lclTocEntry;
+
+
+/*------
+ * Static declarations
+ *------
+ */
+static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id);
+static pgoff_t _getFilePos(ArchiveHandle *AH, lclContext *ctx);
+
+static size_t _CustomWriteFunc(ArchiveHandle *AH, const char *buf, size_t len);
+static size_t _CustomReadFunc(ArchiveHandle *AH, char **buf, size_t *buflen);
+
+static const char *modulename = gettext_noop("custom archiver");
+
+
+
+/*
+ * Init routine required by ALL formats. This is a global routine
+ * and should be declared in pg_backup_archiver.h
+ *
+ * It's task is to create any extra archive context (using AH->formatData),
+ * and to initialize the supported function pointers.
+ *
+ * It should also prepare whatever it's input source is for reading/writing,
+ * and in the case of a read mode connection, it should load the Header & TOC.
+ */
+void
+InitArchiveFmt_Custom(ArchiveHandle *AH)
+{
+ lclContext *ctx;
+
+ /* Assuming static functions, this can be copied for each format. */
+ AH->ArchiveEntryPtr = _ArchiveEntry;
+ AH->StartDataPtr = _StartData;
+ AH->WriteDataPtr = _WriteData;
+ AH->EndDataPtr = _EndData;
+ AH->WriteBytePtr = _WriteByte;
+ AH->ReadBytePtr = _ReadByte;
+ AH->WriteBufPtr = _WriteBuf;
+ AH->ReadBufPtr = _ReadBuf;
+ AH->ClosePtr = _CloseArchive;
+ AH->ReopenPtr = _ReopenArchive;
+ AH->PrintTocDataPtr = _PrintTocData;
+ AH->ReadExtraTocPtr = _ReadExtraToc;
+ AH->WriteExtraTocPtr = _WriteExtraToc;
+ AH->PrintExtraTocPtr = _PrintExtraToc;
+
+ AH->StartBlobsPtr = _StartBlobs;
+ AH->StartBlobPtr = _StartBlob;
+ AH->EndBlobPtr = _EndBlob;
+ AH->EndBlobsPtr = _EndBlobs;
+ AH->ClonePtr = _Clone;
+ AH->DeClonePtr = _DeClone;
+
+ /* Set up a private area. */
+ ctx = (lclContext *) pg_calloc(1, sizeof(lclContext));
+ AH->formatData = (void *) ctx;
+
+ /* Initialize LO buffering */
+ AH->lo_buf_size = LOBBUFSIZE;
+ AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
+
+ ctx->filePos = 0;
+
+ /*
+ * Now open the file
+ */
+ if (AH->mode == archModeWrite)
+ {
+ if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
+ {
+ AH->FH = fopen(AH->fSpec, PG_BINARY_W);
+ if (!AH->FH)
+ die_horribly(AH, modulename, "could not open output file \"%s\": %s\n",
+ AH->fSpec, strerror(errno));
+ }
+ else
+ {
+ AH->FH = stdout;
+ if (!AH->FH)
+ die_horribly(AH, modulename, "could not open output file: %s\n",
+ strerror(errno));
+ }
+
+ ctx->hasSeek = checkSeek(AH->FH);
+ }
+ else
+ {
+ if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
+ {
+ AH->FH = fopen(AH->fSpec, PG_BINARY_R);
+ if (!AH->FH)
+ die_horribly(AH, modulename, "could not open input file \"%s\": %s\n",
+ AH->fSpec, strerror(errno));
+ }
+ else
+ {
+ AH->FH = stdin;
+ if (!AH->FH)
+ die_horribly(AH, modulename, "could not open input file: %s\n",
+ strerror(errno));
+ }
+
+ ctx->hasSeek = checkSeek(AH->FH);
+
+ ReadHead(AH);
+ ReadToc(AH);
+ ctx->dataStart = _getFilePos(AH, ctx);
+ }
+
+}
+
+/*
+ * Called by the Archiver when the dumper creates a new TOC entry.
+ *
+ * Optional.
+ *
+ * Set up extrac format-related TOC data.
+*/
+static void
+_ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
+{
+ lclTocEntry *ctx;
+
+ ctx = (lclTocEntry *) pg_calloc(1, sizeof(lclTocEntry));
+ if (te->dataDumper)
+ ctx->dataState = K_OFFSET_POS_NOT_SET;
+ else
+ ctx->dataState = K_OFFSET_NO_DATA;
+
+ te->formatData = (void *) ctx;
+}
+
+/*
+ * Called by the Archiver to save any extra format-related TOC entry
+ * data.
+ *
+ * Optional.
+ *
+ * Use the Archiver routines to write data - they are non-endian, and
+ * maintain other important file information.
+ */
+static void
+_WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
+{
+ lclTocEntry *ctx = (lclTocEntry *) te->formatData;
+
+ WriteOffset(AH, ctx->dataPos, ctx->dataState);
+}
+
+/*
+ * Called by the Archiver to read any extra format-related TOC data.
+ *
+ * Optional.
+ *
+ * Needs to match the order defined in _WriteExtraToc, and sould also
+ * use the Archiver input routines.
+ */
+static void
+_ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
+{
+ lclTocEntry *ctx = (lclTocEntry *) te->formatData;
+
+ if (ctx == NULL)
+ {
+ ctx = (lclTocEntry *) pg_calloc(1, sizeof(lclTocEntry));
+ te->formatData = (void *) ctx;
+ }
+
+ ctx->dataState = ReadOffset(AH, &(ctx->dataPos));
+
+ /*
+ * Prior to V1.7 (pg7.3), we dumped the data size as an int now we don't
+ * dump it at all.
+ */
+ if (AH->version < K_VERS_1_7)
+ ReadInt(AH);
+}
+
+/*
+ * Called by the Archiver when restoring an archive to output a comment
+ * that includes useful information about the TOC entry.
+ *
+ * Optional.
+ *
+ */
+static void
+_PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
+{
+ lclTocEntry *ctx = (lclTocEntry *) te->formatData;
+
+ if (AH->public.verbose)
+ ahprintf(AH, "-- Data Pos: " INT64_FORMAT "\n",
+ (int64) ctx->dataPos);
+}
+
+/*
+ * Called by the archiver when saving TABLE DATA (not schema). This routine
+ * should save whatever format-specific information is needed to read
+ * the archive back.
+ *
+ * It is called just prior to the dumper's 'DataDumper' routine being called.
+ *
+ * Optional, but strongly recommended.
+ *
+ */
+static void
+_StartData(ArchiveHandle *AH, TocEntry *te)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ lclTocEntry *tctx = (lclTocEntry *) te->formatData;
+
+ tctx->dataPos = _getFilePos(AH, ctx);
+ tctx->dataState = K_OFFSET_POS_SET;
+
+ _WriteByte(AH, BLK_DATA); /* Block type */
+ WriteInt(AH, te->dumpId); /* For sanity check */
+
+ ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+}
+
+/*
+ * Called by archiver when dumper calls WriteData. This routine is
+ * called for both BLOB and TABLE data; it is the responsibility of
+ * the format to manage each kind of data using StartBlob/StartData.
+ *
+ * It should only be called from within a DataDumper routine.
+ *
+ * Mandatory.
+ */
+static size_t
+_WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ CompressorState *cs = ctx->cs;
+
+ if (dLen == 0)
+ return 0;
+
+ return WriteDataToArchive(AH, cs, data, dLen);
+}
+
+/*
+ * Called by the archiver when a dumper's 'DataDumper' routine has
+ * finished.
+ *
+ * Optional.
+ *
+ */
+static void
+_EndData(ArchiveHandle *AH, TocEntry *te)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+
+ EndCompressor(AH, ctx->cs);
+ /* Send the end marker */
+ WriteInt(AH, 0);
+}
+
+/*
+ * Called by the archiver when starting to save all BLOB DATA (not schema).
+ * This routine should save whatever format-specific information is needed
+ * to read the BLOBs back into memory.
+ *
+ * It is called just prior to the dumper's DataDumper routine.
+ *
+ * Optional, but strongly recommended.
+ */
+static void
+_StartBlobs(ArchiveHandle *AH, TocEntry *te)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ lclTocEntry *tctx = (lclTocEntry *) te->formatData;
+
+ tctx->dataPos = _getFilePos(AH, ctx);
+ tctx->dataState = K_OFFSET_POS_SET;
+
+ _WriteByte(AH, BLK_BLOBS); /* Block type */
+ WriteInt(AH, te->dumpId); /* For sanity check */
+}
+
+/*
+ * Called by the archiver when the dumper calls StartBlob.
+ *
+ * Mandatory.
+ *
+ * Must save the passed OID for retrieval at restore-time.
+ */
+static void
+_StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+
+ if (oid == 0)
+ die_horribly(AH, modulename, "invalid OID for large object\n");
+
+ WriteInt(AH, oid);
+
+ ctx->cs = AllocateCompressor(AH->compression, _CustomWriteFunc);
+}
+
+/*
+ * Called by the archiver when the dumper calls EndBlob.
+ *
+ * Optional.
+ */
+static void
+_EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+
+ EndCompressor(AH, ctx->cs);
+ /* Send the end marker */
+ WriteInt(AH, 0);
+}
+
+/*
+ * Called by the archiver when finishing saving all BLOB DATA.
+ *
+ * Optional.
+ */
+static void
+_EndBlobs(ArchiveHandle *AH, TocEntry *te)
+{
+ /* Write out a fake zero OID to mark end-of-blobs. */
+ WriteInt(AH, 0);
+}
+
+/*
+ * Print data for a given TOC entry
+ */
+static void
+_PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ lclTocEntry *tctx = (lclTocEntry *) te->formatData;
+ int blkType;
+ int id;
+
+ if (tctx->dataState == K_OFFSET_NO_DATA)
+ return;
+
+ if (!ctx->hasSeek || tctx->dataState == K_OFFSET_POS_NOT_SET)
+ {
+ /*
+ * We cannot seek directly to the desired block. Instead, skip over
+ * block headers until we find the one we want. This could fail if we
+ * are asked to restore items out-of-order.
+ */
+ _readBlockHeader(AH, &blkType, &id);
+
+ while (blkType != EOF && id != te->dumpId)
+ {
+ switch (blkType)
+ {
+ case BLK_DATA:
+ _skipData(AH);
+ break;
+
+ case BLK_BLOBS:
+ _skipBlobs(AH);
+ break;
+
+ default: /* Always have a default */
+ die_horribly(AH, modulename,
+ "unrecognized data block type (%d) while searching archive\n",
+ blkType);
+ break;
+ }
+ _readBlockHeader(AH, &blkType, &id);
+ }
+ }
+ else
+ {
+ /* We can just seek to the place we need to be. */
+ if (fseeko(AH->FH, tctx->dataPos, SEEK_SET) != 0)
+ die_horribly(AH, modulename, "error during file seek: %s\n",
+ strerror(errno));
+
+ _readBlockHeader(AH, &blkType, &id);
+ }
+
+ /* Produce suitable failure message if we fell off end of file */
+ if (blkType == EOF)
+ {
+ if (tctx->dataState == K_OFFSET_POS_NOT_SET)
+ die_horribly(AH, modulename, "could not find block ID %d in archive -- "
+ "possibly due to out-of-order restore request, "
+ "which cannot be handled due to lack of data offsets in archive\n",
+ te->dumpId);
+ else if (!ctx->hasSeek)
+ die_horribly(AH, modulename, "could not find block ID %d in archive -- "
+ "possibly due to out-of-order restore request, "
+ "which cannot be handled due to non-seekable input file\n",
+ te->dumpId);
+ else /* huh, the dataPos led us to EOF? */
+ die_horribly(AH, modulename, "could not find block ID %d in archive -- "
+ "possibly corrupt archive\n",
+ te->dumpId);
+ }
+
+ /* Are we sane? */
+ if (id != te->dumpId)
+ die_horribly(AH, modulename, "found unexpected block ID (%d) when reading data -- expected %d\n",
+ id, te->dumpId);
+
+ switch (blkType)
+ {
+ case BLK_DATA:
+ _PrintData(AH);
+ break;
+
+ case BLK_BLOBS:
+ _LoadBlobs(AH, ropt->dropSchema);
+ break;
+
+ default: /* Always have a default */
+ die_horribly(AH, modulename, "unrecognized data block type %d while restoring archive\n",
+ blkType);
+ break;
+ }
+}
+
+/*
+ * Print data from current file position.
+*/
+static void
+_PrintData(ArchiveHandle *AH)
+{
+ ReadDataFromArchive(AH, AH->compression, _CustomReadFunc);
+}
+
+static void
+_LoadBlobs(ArchiveHandle *AH, bool drop)
+{
+ Oid oid;
+
+ StartRestoreBlobs(AH);
+
+ oid = ReadInt(AH);
+ while (oid != 0)
+ {
+ StartRestoreBlob(AH, oid, drop);
+ _PrintData(AH);
+ EndRestoreBlob(AH, oid);
+ oid = ReadInt(AH);
+ }
+
+ EndRestoreBlobs(AH);
+}
+
+/*
+ * Skip the BLOBs from the current file position.
+ * BLOBS are written sequentially as data blocks (see below).
+ * Each BLOB is preceded by it's original OID.
+ * A zero OID indicated the end of the BLOBS
+ */
+static void
+_skipBlobs(ArchiveHandle *AH)
+{
+ Oid oid;
+
+ oid = ReadInt(AH);
+ while (oid != 0)
+ {
+ _skipData(AH);
+ oid = ReadInt(AH);
+ }
+}
+
+/*
+ * Skip data from current file position.
+ * Data blocks are formatted as an integer length, followed by data.
+ * A zero length denoted the end of the block.
+*/
+static void
+_skipData(ArchiveHandle *AH)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ size_t blkLen;
+ char *buf = NULL;
+ int buflen = 0;
+ size_t cnt;
+
+ blkLen = ReadInt(AH);
+ while (blkLen != 0)
+ {
+ if (blkLen > buflen)
+ {
+ if (buf)
+ free(buf);
+ buf = (char *) pg_malloc(blkLen);
+ buflen = blkLen;
+ }
+ cnt = fread(buf, 1, blkLen, AH->FH);
+ if (cnt != blkLen)
+ {
+ if (feof(AH->FH))
+ die_horribly(AH, modulename,
+ "could not read from input file: end of file\n");
+ else
+ die_horribly(AH, modulename,
+ "could not read from input file: %s\n", strerror(errno));
+ }
+
+ ctx->filePos += blkLen;
+
+ blkLen = ReadInt(AH);
+ }
+
+ if (buf)
+ free(buf);
+}
+
+/*
+ * Write a byte of data to the archive.
+ *
+ * Mandatory.
+ *
+ * Called by the archiver to do integer & byte output to the archive.
+ */
+static int
+_WriteByte(ArchiveHandle *AH, const int i)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ int res;
+
+ res = fputc(i, AH->FH);
+ if (res != EOF)
+ ctx->filePos += 1;
+ else
+ die_horribly(AH, modulename, "could not write byte: %s\n", strerror(errno));
+ return res;
+}
+
+/*
+ * Read a byte of data from the archive.
+ *
+ * Mandatory
+ *
+ * Called by the archiver to read bytes & integers from the archive.
+ * EOF should be treated as a fatal error.
+ */
+static int
+_ReadByte(ArchiveHandle *AH)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ int res;
+
+ res = getc(AH->FH);
+ if (res == EOF)
+ die_horribly(AH, modulename, "unexpected end of file\n");
+ ctx->filePos += 1;
+ return res;
+}
+
+/*
+ * Write a buffer of data to the archive.
+ *
+ * Mandatory.
+ *
+ * Called by the archiver to write a block of bytes to the archive.
+ */
+static size_t
+_WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ size_t res;
+
+ res = fwrite(buf, 1, len, AH->FH);
+
+ if (res != len)
+ die_horribly(AH, modulename,
+ "could not write to output file: %s\n", strerror(errno));
+
+ ctx->filePos += res;
+ return res;
+}
+
+/*
+ * Read a block of bytes from the archive.
+ *
+ * Mandatory.
+ *
+ * Called by the archiver to read a block of bytes from the archive
+ */
+static size_t
+_ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ size_t res;
+
+ res = fread(buf, 1, len, AH->FH);
+ ctx->filePos += res;
+
+ return res;
+}
+
+/*
+ * Close the archive.
+ *
+ * Mandatory.
+ *
+ * When writing the archive, this is the routine that actually starts
+ * the process of saving it to files. No data should be written prior
+ * to this point, since the user could sort the TOC after creating it.
+ *
+ * If an archive is to be written, this toutine must call:
+ * WriteHead to save the archive header
+ * WriteToc to save the TOC entries
+ * WriteDataChunks to save all DATA & BLOBs.
+ *
+ */
+static void
+_CloseArchive(ArchiveHandle *AH)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ pgoff_t tpos;
+
+ if (AH->mode == archModeWrite)
+ {
+ WriteHead(AH);
+ tpos = ftello(AH->FH);
+ WriteToc(AH);
+ ctx->dataStart = _getFilePos(AH, ctx);
+ WriteDataChunks(AH);
+
+ /*
+ * If possible, re-write the TOC in order to update the data offset
+ * information. This is not essential, as pg_restore can cope in most
+ * cases without it; but it can make pg_restore significantly faster
+ * in some situations (especially parallel restore).
+ */
+ if (ctx->hasSeek &&
+ fseeko(AH->FH, tpos, SEEK_SET) == 0)
+ WriteToc(AH);
+ }
+
+ if (fclose(AH->FH) != 0)
+ die_horribly(AH, modulename, "could not close archive file: %s\n", strerror(errno));
+
+ AH->FH = NULL;
+}
+
+/*
+ * Reopen the archive's file handle.
+ *
+ * We close the original file handle, except on Windows. (The difference
+ * is because on Windows, this is used within a multithreading context,
+ * and we don't want a thread closing the parent file handle.)
+ */
+static void
+_ReopenArchive(ArchiveHandle *AH)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ pgoff_t tpos;
+
+ if (AH->mode == archModeWrite)
+ die_horribly(AH, modulename, "can only reopen input archives\n");
+
+ /*
+ * These two cases are user-facing errors since they represent unsupported
+ * (but not invalid) use-cases. Word the error messages appropriately.
+ */
+ if (AH->fSpec == NULL || strcmp(AH->fSpec, "") == 0)
+ die_horribly(AH, modulename, "parallel restore from stdin is not supported\n");
+ if (!ctx->hasSeek)
+ die_horribly(AH, modulename, "parallel restore from non-seekable file is not supported\n");
+
+ errno = 0;
+ tpos = ftello(AH->FH);
+ if (errno)
+ die_horribly(AH, modulename, "could not determine seek position in archive file: %s\n",
+ strerror(errno));
+
+#ifndef WIN32
+ if (fclose(AH->FH) != 0)
+ die_horribly(AH, modulename, "could not close archive file: %s\n",
+ strerror(errno));
+#endif
+
+ AH->FH = fopen(AH->fSpec, PG_BINARY_R);
+ if (!AH->FH)
+ die_horribly(AH, modulename, "could not open input file \"%s\": %s\n",
+ AH->fSpec, strerror(errno));
+
+ if (fseeko(AH->FH, tpos, SEEK_SET) != 0)
+ die_horribly(AH, modulename, "could not set seek position in archive file: %s\n",
+ strerror(errno));
+}
+
+/*
+ * Clone format-specific fields during parallel restoration.
+ */
+static void
+_Clone(ArchiveHandle *AH)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+
+ AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
+ memcpy(AH->formatData, ctx, sizeof(lclContext));
+ ctx = (lclContext *) AH->formatData;
+
+ /* sanity check, shouldn't happen */
+ if (ctx->cs != NULL)
+ die_horribly(AH, modulename, "compressor active\n");
+
+ /*
+ * Note: we do not make a local lo_buf because we expect at most one BLOBS
+ * entry per archive, so no parallelism is possible. Likewise,
+ * TOC-entry-local state isn't an issue because any one TOC entry is
+ * touched by just one worker child.
+ */
+}
+
+static void
+_DeClone(ArchiveHandle *AH)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+
+ free(ctx);
+}
+
+/*--------------------------------------------------
+ * END OF FORMAT CALLBACKS
+ *--------------------------------------------------
+ */
+
+/*
+ * Get the current position in the archive file.
+ */
+static pgoff_t
+_getFilePos(ArchiveHandle *AH, lclContext *ctx)
+{
+ pgoff_t pos;
+
+ if (ctx->hasSeek)
+ {
+ pos = ftello(AH->FH);
+ if (pos != ctx->filePos)
+ {
+ write_msg(modulename, "WARNING: ftell mismatch with expected position -- ftell used\n");
+
+ /*
+ * Prior to 1.7 (pg7.3) we relied on the internally maintained
+ * pointer. Now we rely on ftello() always, unless the file has
+ * been found to not support it.
+ */
+ }
+ }
+ else
+ pos = ctx->filePos;
+ return pos;
+}
+
+/*
+ * Read a data block header. The format changed in V1.3, so we
+ * centralize the code here for simplicity. Returns *type = EOF
+ * if at EOF.
+ */
+static void
+_readBlockHeader(ArchiveHandle *AH, int *type, int *id)
+{
+ lclContext *ctx = (lclContext *) AH->formatData;
+ int byt;
+
+ /*
+ * Note: if we are at EOF with a pre-1.3 input file, we'll die_horribly
+ * inside ReadInt rather than returning EOF. It doesn't seem worth
+ * jumping through hoops to deal with that case better, because no such
+ * files are likely to exist in the wild: only some 7.1 development
+ * versions of pg_dump ever generated such files.
+ */
+ if (AH->version < K_VERS_1_3)
+ *type = BLK_DATA;
+ else
+ {
+ byt = getc(AH->FH);
+ *type = byt;
+ if (byt == EOF)
+ {
+ *id = 0; /* don't return an uninitialized value */
+ return;
+ }
+ ctx->filePos += 1;
+ }
+
+ *id = ReadInt(AH);
+}
+
+/*
+ * Callback function for WriteDataToArchive. Writes one block of (compressed)
+ * data to the archive.
+ */
+static size_t
+_CustomWriteFunc(ArchiveHandle *AH, const char *buf, size_t len)
+{
+ /* never write 0-byte blocks (this should not happen) */
+ if (len == 0)
+ return 0;
+
+ WriteInt(AH, len);
+ return _WriteBuf(AH, buf, len);
+}
+
+/*
+ * Callback function for ReadDataFromArchive. To keep things simple, we
+ * always read one compressed block at a time.
+ */
+static size_t
+_CustomReadFunc(ArchiveHandle *AH, char **buf, size_t *buflen)
+{
+ size_t blkLen;
+ size_t cnt;
+
+ /* Read length */
+ blkLen = ReadInt(AH);
+ if (blkLen == 0)
+ return 0;
+
+ /* If the caller's buffer is not large enough, allocate a bigger one */
+ if (blkLen > *buflen)
+ {
+ free(*buf);
+ *buf = (char *) pg_malloc(blkLen);
+ *buflen = blkLen;
+ }
+
+ cnt = _ReadBuf(AH, *buf, blkLen);
+ if (cnt != blkLen)
+ {
+ if (feof(AH->FH))
+ die_horribly(AH, modulename,
+ "could not read from input file: end of file\n");
+ else
+ die_horribly(AH, modulename,
+ "could not read from input file: %s\n", strerror(errno));
+ }
+ return cnt;
+}
projects / postgresql / blobdiff