From 40bede5477bb5bce98ce9548841cb414634c26f7 Mon Sep 17 00:00:00 2001 From: Fujii Masao Date: Mon, 9 Feb 2015 15:15:24 +0900 Subject: [PATCH] Move pg_lzcompress.c to src/common. The meta data of PGLZ symbolized by PGLZ_Header is removed, to make the compression and decompression code independent on the backend-only varlena facility. PGLZ_Header is being used to store some meta data related to the data being compressed like the raw length of the uncompressed record or some varlena-related data, making it unpluggable once PGLZ is stored in src/common as it contains some backend-only code paths with the management of varlena structures. The APIs of PGLZ are reworked at the same time to do only compression and decompression of buffers without the meta-data layer, simplifying its use for a more general usage. On-disk format is preserved as well, so there is no incompatibility with previous major versions of PostgreSQL for TOAST entries. Exposing compression and decompression APIs of pglz makes possible its use by extensions and contrib modules. Especially this commit is required for upcoming WAL compression feature so that the WAL reader facility can decompress the WAL data by using pglz_decompress. Michael Paquier, reviewed by me. --- src/backend/access/heap/tuptoaster.c | 87 ++++++++++++----- src/backend/utils/adt/Makefile | 4 +- src/common/Makefile | 3 +- .../utils/adt => common}/pg_lzcompress.c | 97 +++++++++---------- src/include/{utils => common}/pg_lzcompress.h | 31 +----- src/tools/msvc/Mkvcbuild.pm | 3 +- src/tools/pgindent/typedefs.list | 1 - 7 files changed, 124 insertions(+), 102 deletions(-) rename src/{backend/utils/adt => common}/pg_lzcompress.c (92%) rename src/include/{utils => common}/pg_lzcompress.h (77%) diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c index 2d53c8e5ff..d19e7220c5 100644 --- a/src/backend/access/heap/tuptoaster.c +++ b/src/backend/access/heap/tuptoaster.c @@ -35,9 +35,9 @@ #include "access/tuptoaster.h" #include "access/xact.h" #include "catalog/catalog.h" +#include "common/pg_lzcompress.h" #include "miscadmin.h" #include "utils/fmgroids.h" -#include "utils/pg_lzcompress.h" #include "utils/rel.h" #include "utils/typcache.h" #include "utils/tqual.h" @@ -45,6 +45,26 @@ #undef TOAST_DEBUG +/* + * The information at the start of the compressed toast data. + */ +typedef struct toast_compress_header +{ + int32 vl_len_; /* varlena header (do not touch directly!) */ + int32 rawsize; +} toast_compress_header; + +/* + * Utilities for manipulation of header information for compressed + * toast entries. + */ +#define TOAST_COMPRESS_HDRSZ ((int32) sizeof(toast_compress_header)) +#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) ptr)->rawsize) +#define TOAST_COMPRESS_RAWDATA(ptr) \ + (((char *) ptr) + TOAST_COMPRESS_HDRSZ) +#define TOAST_COMPRESS_SET_RAWSIZE(ptr, len) \ + (((toast_compress_header *) ptr)->rawsize = len) + static void toast_delete_datum(Relation rel, Datum value); static Datum toast_save_datum(Relation rel, Datum value, struct varlena * oldexternal, int options); @@ -53,6 +73,7 @@ static bool toastid_valueid_exists(Oid toastrelid, Oid valueid); static struct varlena *toast_fetch_datum(struct varlena * attr); static struct varlena *toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length); +static struct varlena *toast_decompress_datum(struct varlena * attr); static int toast_open_indexes(Relation toastrel, LOCKMODE lock, Relation **toastidxs, @@ -138,11 +159,8 @@ heap_tuple_untoast_attr(struct varlena * attr) /* If it's compressed, decompress it */ if (VARATT_IS_COMPRESSED(attr)) { - PGLZ_Header *tmp = (PGLZ_Header *) attr; - - attr = (struct varlena *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ); - SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ); - pglz_decompress(tmp, VARDATA(attr)); + struct varlena *tmp = attr; + attr = toast_decompress_datum(tmp); pfree(tmp); } } @@ -163,11 +181,7 @@ heap_tuple_untoast_attr(struct varlena * attr) /* * This is a compressed value inside of the main tuple */ - PGLZ_Header *tmp = (PGLZ_Header *) attr; - - attr = (struct varlena *) palloc(PGLZ_RAW_SIZE(tmp) + VARHDRSZ); - SET_VARSIZE(attr, PGLZ_RAW_SIZE(tmp) + VARHDRSZ); - pglz_decompress(tmp, VARDATA(attr)); + attr = toast_decompress_datum(attr); } else if (VARATT_IS_SHORT(attr)) { @@ -234,14 +248,10 @@ heap_tuple_untoast_attr_slice(struct varlena * attr, if (VARATT_IS_COMPRESSED(preslice)) { - PGLZ_Header *tmp = (PGLZ_Header *) preslice; - Size size = PGLZ_RAW_SIZE(tmp) + VARHDRSZ; - - preslice = (struct varlena *) palloc(size); - SET_VARSIZE(preslice, size); - pglz_decompress(tmp, VARDATA(preslice)); + struct varlena *tmp = preslice; + preslice = toast_decompress_datum(tmp); - if (tmp != (PGLZ_Header *) attr) + if (tmp != attr) pfree(tmp); } @@ -1228,6 +1238,7 @@ toast_compress_datum(Datum value) { struct varlena *tmp; int32 valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + int32 len; Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value))); Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value))); @@ -1240,7 +1251,8 @@ toast_compress_datum(Datum value) valsize > PGLZ_strategy_default->max_input_size) return PointerGetDatum(NULL); - tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize)); + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + TOAST_COMPRESS_HDRSZ); /* * We recheck the actual size even if pglz_compress() reports success, @@ -1252,10 +1264,15 @@ toast_compress_datum(Datum value) * only one header byte and no padding if the value is short enough. So * we insist on a savings of more than 2 bytes to ensure we have a gain. */ - if (pglz_compress(VARDATA_ANY(DatumGetPointer(value)), valsize, - (PGLZ_Header *) tmp, PGLZ_strategy_default) && - VARSIZE(tmp) < valsize - 2) + len = pglz_compress(VARDATA_ANY(DatumGetPointer(value)), + valsize, + TOAST_COMPRESS_RAWDATA(tmp), + PGLZ_strategy_default); + if (len >= 0 && + len + TOAST_COMPRESS_HDRSZ < valsize - 2) { + TOAST_COMPRESS_SET_RAWSIZE(tmp, valsize); + SET_VARSIZE_COMPRESSED(tmp, len + TOAST_COMPRESS_HDRSZ); /* successful compression */ return PointerGetDatum(tmp); } @@ -2100,6 +2117,32 @@ toast_fetch_datum_slice(struct varlena * attr, int32 sliceoffset, int32 length) return result; } +/* ---------- + * toast_decompress_datum - + * + * Decompress a compressed version of a varlena datum + */ +static struct varlena * +toast_decompress_datum(struct varlena * attr) +{ + struct varlena *result; + + Assert(VARATT_IS_COMPRESSED(attr)); + + result = (struct varlena *) + palloc(TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); + SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); + + if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr), + VARSIZE(attr) - TOAST_COMPRESS_HDRSZ, + VARDATA(result), + TOAST_COMPRESS_RAWSIZE(attr)) < 0) + elog(ERROR, "compressed data is corrupted"); + + return result; +} + + /* ---------- * toast_open_indexes * diff --git a/src/backend/utils/adt/Makefile b/src/backend/utils/adt/Makefile index 3ea9bf435a..20e5ff10c7 100644 --- a/src/backend/utils/adt/Makefile +++ b/src/backend/utils/adt/Makefile @@ -25,8 +25,8 @@ OBJS = acl.o arrayfuncs.o array_selfuncs.o array_typanalyze.o \ jsonfuncs.o like.o lockfuncs.o mac.o misc.o nabstime.o name.o \ network.o network_gist.o network_selfuncs.o \ numeric.o numutils.o oid.o oracle_compat.o \ - orderedsetaggs.o pg_lzcompress.o pg_locale.o pg_lsn.o \ - pgstatfuncs.o pseudotypes.o quote.o rangetypes.o rangetypes_gist.o \ + orderedsetaggs.o pg_locale.o pg_lsn.o pgstatfuncs.o \ + pseudotypes.o quote.o rangetypes.o rangetypes_gist.o \ rangetypes_selfuncs.o rangetypes_spgist.o rangetypes_typanalyze.o \ regexp.o regproc.o ri_triggers.o rowtypes.o ruleutils.o \ selfuncs.o tid.o timestamp.o trigfuncs.o \ diff --git a/src/common/Makefile b/src/common/Makefile index e5c345d7de..5f24eb38de 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -23,7 +23,8 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -DFRONTEND $(CPPFLAGS) LIBS += $(PTHREAD_LIBS) -OBJS_COMMON = exec.o pgfnames.o psprintf.o relpath.o rmtree.o string.o username.o wait_error.o +OBJS_COMMON = exec.o pg_lzcompress.o pgfnames.o psprintf.o relpath.o \ + rmtree.o string.o username.o wait_error.o OBJS_FRONTEND = $(OBJS_COMMON) fe_memutils.o diff --git a/src/backend/utils/adt/pg_lzcompress.c b/src/common/pg_lzcompress.c similarity index 92% rename from src/backend/utils/adt/pg_lzcompress.c rename to src/common/pg_lzcompress.c index 65cb6a95c2..447a043ad7 100644 --- a/src/backend/utils/adt/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -8,8 +8,8 @@ * * Entry routines: * - * bool - * pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, + * int32 + * pglz_compress(const char *source, int32 slen, char *dest, * const PGLZ_Strategy *strategy); * * source is the input data to be compressed. @@ -23,44 +23,43 @@ * the compression algorithm. If NULL, the compiled * in default strategy is used. * - * The return value is TRUE if compression succeeded, - * FALSE if not; in the latter case the contents of dest - * are undefined. + * The return value is the number of bytes written in the + * buffer dest, or -1 if compression fails; in the latter + * case the contents of dest are undefined. * - * void - * pglz_decompress(const PGLZ_Header *source, char *dest) + * int32 + * pglz_decompress(const char *source, int32 slen, char *dest, + * int32 rawsize) * * source is the compressed input. * + * slen is the length of the compressed input. + * * dest is the area where the uncompressed data will be * written to. It is the callers responsibility to - * provide enough space. The required amount can be - * obtained with the macro PGLZ_RAW_SIZE(source). + * provide enough space. * * The data is written to buff exactly as it was handed * to pglz_compress(). No terminating zero byte is added. * - * The decompression algorithm and internal data format: + * rawsize is the length of the uncompressed data. * - * PGLZ_Header is defined as + * The return value is the number of bytes written in the + * buffer dest, or -1 if decompression fails. * - * typedef struct PGLZ_Header { - * int32 vl_len_; - * int32 rawsize; - * } + * The decompression algorithm and internal data format: * - * The header is followed by the compressed data itself. + * It is made with the compressed data itself. * * The data representation is easiest explained by describing * the process of decompression. * - * If VARSIZE(x) == rawsize + sizeof(PGLZ_Header), then the data + * If compressed_size == rawsize, then the data * is stored uncompressed as plain bytes. Thus, the decompressor - * simply copies rawsize bytes from the location after the - * header to the destination. + * simply copies rawsize bytes to the destination. * - * Otherwise the first byte after the header tells what to do - * the next 8 times. We call this the control byte. + * Otherwise the first byte tells what to do the next 8 times. + * We call this the control byte. * * An unset bit in the control byte means, that one uncompressed * byte follows, which is copied from input to output. @@ -169,14 +168,18 @@ * * Copyright (c) 1999-2015, PostgreSQL Global Development Group * - * src/backend/utils/adt/pg_lzcompress.c + * src/common/pg_lzcompress.c * ---------- */ +#ifndef FRONTEND #include "postgres.h" +#else +#include "postgres_fe.h" +#endif #include -#include "utils/pg_lzcompress.h" +#include "common/pg_lzcompress.h" /* ---------- @@ -492,14 +495,15 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, /* ---------- * pglz_compress - * - * Compresses source into dest using strategy. + * Compresses source into dest using strategy. Returns the number of + * bytes written in buffer dest, or -1 if compression fails. * ---------- */ -bool -pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, +int32 +pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy) { - unsigned char *bp = ((unsigned char *) dest) + sizeof(PGLZ_Header); + unsigned char *bp = (unsigned char *) dest; unsigned char *bstart = bp; int hist_next = 1; bool hist_recycle = false; @@ -533,12 +537,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, if (strategy->match_size_good <= 0 || slen < strategy->min_input_size || slen > strategy->max_input_size) - return false; - - /* - * Save the original source size in the header. - */ - dest->rawsize = slen; + return -1; /* * Limit the match parameters to the supported range. @@ -611,7 +610,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, * allow 4 slop bytes. */ if (bp - bstart >= result_max) - return false; + return -1; /* * If we've emitted more than first_success_by bytes without finding @@ -620,7 +619,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, * pre-compressed data). */ if (!found_match && bp - bstart >= strategy->first_success_by) - return false; + return -1; /* * Try to find a match in the history @@ -664,35 +663,34 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, *ctrlp = ctrlb; result_size = bp - bstart; if (result_size >= result_max) - return false; - - /* - * Success - need only fill in the actual length of the compressed datum. - */ - SET_VARSIZE_COMPRESSED(dest, result_size + sizeof(PGLZ_Header)); + return -1; - return true; + /* success */ + return result_size; } /* ---------- * pglz_decompress - * - * Decompresses source into dest. + * Decompresses source into dest. Returns the number of bytes + * decompressed in the destination buffer, or -1 if decompression + * fails. * ---------- */ -void -pglz_decompress(const PGLZ_Header *source, char *dest) +int32 +pglz_decompress(const char *source, int32 slen, char *dest, + int32 rawsize) { const unsigned char *sp; const unsigned char *srcend; unsigned char *dp; unsigned char *destend; - sp = ((const unsigned char *) source) + sizeof(PGLZ_Header); - srcend = ((const unsigned char *) source) + VARSIZE(source); + sp = (const unsigned char *) source; + srcend = ((const unsigned char *) source) + slen; dp = (unsigned char *) dest; - destend = dp + source->rawsize; + destend = dp + rawsize; while (sp < srcend && dp < destend) { @@ -771,9 +769,10 @@ pglz_decompress(const PGLZ_Header *source, char *dest) * Check we decompressed the right amount. */ if (dp != destend || sp != srcend) - elog(ERROR, "compressed data is corrupt"); + return -1; /* * That's it. */ + return rawsize; } diff --git a/src/include/utils/pg_lzcompress.h b/src/include/common/pg_lzcompress.h similarity index 77% rename from src/include/utils/pg_lzcompress.h rename to src/include/common/pg_lzcompress.h index 4af24a32a4..52bcaf14b1 100644 --- a/src/include/utils/pg_lzcompress.h +++ b/src/include/common/pg_lzcompress.h @@ -3,7 +3,7 @@ * * Definitions for the builtin LZ compressor * - * src/include/utils/pg_lzcompress.h + * src/include/common/pg_lzcompress.h * ---------- */ @@ -11,19 +11,6 @@ #define _PG_LZCOMPRESS_H_ -/* ---------- - * PGLZ_Header - - * - * The information at the start of the compressed data. - * ---------- - */ -typedef struct PGLZ_Header -{ - int32 vl_len_; /* varlena header (do not touch directly!) */ - int32 rawsize; -} PGLZ_Header; - - /* ---------- * PGLZ_MAX_OUTPUT - * @@ -31,16 +18,7 @@ typedef struct PGLZ_Header * We allow 4 bytes for overrun before detecting compression failure. * ---------- */ -#define PGLZ_MAX_OUTPUT(_dlen) ((_dlen) + 4 + sizeof(PGLZ_Header)) - -/* ---------- - * PGLZ_RAW_SIZE - - * - * Macro to determine the uncompressed data size contained - * in the entry. - * ---------- - */ -#define PGLZ_RAW_SIZE(_lzdata) ((_lzdata)->rawsize) +#define PGLZ_MAX_OUTPUT(_dlen) ((_dlen) + 4) /* ---------- @@ -105,8 +83,9 @@ extern const PGLZ_Strategy *const PGLZ_strategy_always; * Global function declarations * ---------- */ -extern bool pglz_compress(const char *source, int32 slen, PGLZ_Header *dest, +extern int32 pglz_compress(const char *source, int32 slen, char *dest, const PGLZ_Strategy *strategy); -extern void pglz_decompress(const PGLZ_Header *source, char *dest); +extern int32 pglz_decompress(const char *source, int32 slen, char *dest, + int32 rawsize); #endif /* _PG_LZCOMPRESS_H_ */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 4336f2eabb..830f56f0fa 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -76,7 +76,8 @@ sub mkvcbuild push(@pgportfiles, 'rint.c') if ($vsVersion < '12.00'); our @pgcommonallfiles = qw( - exec.c pgfnames.c psprintf.c relpath.c rmtree.c string.c username.c wait_error.c); + exec.c pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c + string.c username.c wait_error.c); our @pgcommonfrontendfiles = (@pgcommonallfiles, qw(fe_memutils.c)); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ab36aa3acb..cfd580c445 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1125,7 +1125,6 @@ PGEventResultCreate PGEventResultDestroy PGFInfoFunction PGFunction -PGLZ_Header PGLZ_HistEntry PGLZ_Strategy PGMessageField -- 2.40.0