From f9895e19e9c274e51d463b0a27cd382d5ce7aba8 Mon Sep 17 00:00:00 2001 From: Richard Russon Date: Fri, 13 Jul 2018 14:25:52 +0100 Subject: [PATCH] split out hcache serialisation code --- Makefile.autosetup | 2 +- hcache/hcache.c | 660 +------------------------------------------- hcache/hcache.h | 27 ++ hcache/serialize.c | 670 +++++++++++++++++++++++++++++++++++++++++++++ hcache/serialize.h | 61 +++++ 5 files changed, 764 insertions(+), 656 deletions(-) create mode 100644 hcache/serialize.c create mode 100644 hcache/serialize.h diff --git a/Makefile.autosetup b/Makefile.autosetup index 8ac1d973b..8699c51b3 100644 --- a/Makefile.autosetup +++ b/Makefile.autosetup @@ -210,7 +210,7 @@ ALLOBJS+= $(LIBCONNOBJS) # libhcache @if USE_HCACHE LIBHCACHE= libhcache.a -LIBHCACHEOBJS= hcache/hcache.o +LIBHCACHEOBJS= hcache/hcache.o hcache/serialize.o CLEANFILES+= $(LIBHCACHE) $(LIBHCACHEOBJS) MUTTLIBS+= $(LIBHCACHE) ALLOBJS+= $(LIBHCACHEOBJS) diff --git a/hcache/hcache.c b/hcache/hcache.c index 2c00e285c..ff0513f30 100644 --- a/hcache/hcache.c +++ b/hcache/hcache.c @@ -34,6 +34,7 @@ #include "config.h" #include "email/email.h" #include "muttlib.h" +#include "serialize.h" #if !(defined(HAVE_BDB) || defined(HAVE_GDBM) || defined(HAVE_KC) || \ defined(HAVE_LMDB) || defined(HAVE_QDBM) || defined(HAVE_TC)) @@ -60,29 +61,6 @@ char *HeaderCacheBackend; static unsigned int hcachever = 0x0; -/** - * struct HeaderCache - header cache structure - * - * This struct holds both the backend-agnostic and the backend-specific parts - * of the header cache. Backend code MUST initialize the fetch, store, - * delete and close function pointers in hcache_open, and MAY store - * backend-specific context in the ctx pointer. - */ -struct HeaderCache -{ - char *folder; - unsigned int crc; - void *ctx; -}; - -/** - * union Validate - Header cache validity - */ -union Validate { - struct timeval timeval; - unsigned int uidvalidity; -}; - #define HCACHE_BACKEND(name) extern const struct HcacheOps hcache_##name##_ops; HCACHE_BACKEND(bdb) HCACHE_BACKEND(gdbm) @@ -143,543 +121,18 @@ static const struct HcacheOps *hcache_get_backend_ops(const char *backend) return *ops; } -/** - * lazy_malloc - Allocate some memory - * @param size Minimum size to allocate - * @retval ptr Allocated memory - * - * This block is likely to be lazy_realloc()'d repeatedly. - * It starts off with a minimum size of 4KiB. - */ -static void *lazy_malloc(size_t size) -{ - if (size < 4096) - size = 4096; - - return mutt_mem_malloc(size); -} - -/** - * lazy_realloc - Reallocate some memory - * @param ptr Pointer to resize - * @param size Minimum size - * - * The minimum size is 4KiB to avoid repeated resizing. - */ -static void lazy_realloc(void *ptr, size_t size) -{ - void **p = (void **) ptr; - - if (p != NULL && size < 4096) - return; - - mutt_mem_realloc(ptr, size); -} - -/** - * dump_int - Pack an integer into a binary blob - * @param i Integer to save - * @param d Binary blob to add to - * @param off Offset into the blob - * @retval ptr End of the newly packed binary - */ -static unsigned char *dump_int(unsigned int i, unsigned char *d, int *off) -{ - lazy_realloc(&d, *off + sizeof(int)); - memcpy(d + *off, &i, sizeof(int)); - (*off) += sizeof(int); - - return d; -} - -/** - * restore_int - Unpack an integer from a binary blob - * @param i Integer to write to - * @param d Binary blob to read from - * @param off Offset into the blob - */ -static void restore_int(unsigned int *i, const unsigned char *d, int *off) -{ - memcpy(i, d + *off, sizeof(int)); - (*off) += sizeof(int); -} - -/** - * dump_char_size - Pack a fixed-length string into a binary blob - * @param c String to pack - * @param d Binary blob to add to - * @param off Offset into the blob - * @param size Size of the string - * @param convert If true, the strings will be converted to utf-8 - * @retval ptr End of the newly packed binary - */ -static unsigned char *dump_char_size(char *c, unsigned char *d, int *off, - ssize_t size, bool convert) -{ - char *p = c; - - if (!c) - { - size = 0; - d = dump_int(size, d, off); - return d; - } - - if (convert && !mutt_str_is_ascii(c, size)) - { - p = mutt_str_substr_dup(c, c + size); - if (mutt_ch_convert_string(&p, Charset, "utf-8", 0) == 0) - { - size = mutt_str_strlen(p) + 1; - } - } - - d = dump_int(size, d, off); - lazy_realloc(&d, *off + size); - memcpy(d + *off, p, size); - *off += size; - - if (p != c) - FREE(&p); - - return d; -} - -/** - * dump_char - Pack a variable-length string into a binary blob - * @param c String to pack - * @param d Binary blob to add to - * @param off Offset into the blob - * @param convert If true, the strings will be converted to utf-8 - * @retval ptr End of the newly packed binary - */ -static unsigned char *dump_char(char *c, unsigned char *d, int *off, bool convert) -{ - return dump_char_size(c, d, off, mutt_str_strlen(c) + 1, convert); -} - -/** - * restore_char - Unpack a variable-length string from a binary blob - * @param c Store the unpacked string here - * @param d Binary blob to read from - * @param off Offset into the blob - * @param convert If true, the strings will be converted to utf-8 - */ -static void restore_char(char **c, const unsigned char *d, int *off, bool convert) -{ - unsigned int size; - restore_int(&size, d, off); - - if (size == 0) - { - *c = NULL; - return; - } - - *c = mutt_mem_malloc(size); - memcpy(*c, d + *off, size); - if (convert && !mutt_str_is_ascii(*c, size)) - { - char *tmp = mutt_str_strdup(*c); - if (mutt_ch_convert_string(&tmp, "utf-8", Charset, 0) == 0) - { - FREE(c); - *c = tmp; - } - else - { - FREE(&tmp); - } - } - *off += size; -} - -/** - * dump_address - Pack an Address into a binary blob - * @param a Address to pack - * @param d Binary blob to add to - * @param off Offset into the blob - * @param convert If true, the strings will be converted to utf-8 - * @retval ptr End of the newly packed binary - */ -static unsigned char *dump_address(struct Address *a, unsigned char *d, int *off, bool convert) -{ - unsigned int counter = 0; - unsigned int start_off = *off; - - d = dump_int(0xdeadbeef, d, off); - - while (a) - { - d = dump_char(a->personal, d, off, convert); - d = dump_char(a->mailbox, d, off, false); - d = dump_int(a->group, d, off); - a = a->next; - counter++; - } - - memcpy(d + start_off, &counter, sizeof(int)); - - return d; -} - -/** - * restore_address - Unpack an Address from a binary blob - * @param a Store the unpacked Address here - * @param d Binary blob to read from - * @param off Offset into the blob - * @param convert If true, the strings will be converted from utf-8 - */ -static void restore_address(struct Address **a, const unsigned char *d, int *off, bool convert) -{ - unsigned int counter = 0; - unsigned int g = 0; - - restore_int(&counter, d, off); - - while (counter) - { - *a = mutt_addr_new(); - restore_char(&(*a)->personal, d, off, convert); - restore_char(&(*a)->mailbox, d, off, false); - restore_int(&g, d, off); - (*a)->group = g; - a = &(*a)->next; - counter--; - } - - *a = NULL; -} - -/** - * dump_stailq - Pack a STAILQ into a binary blob - * @param l List to read from - * @param d Binary blob to add to - * @param off Offset into the blob - * @param convert If true, the strings will be converted to utf-8 - * @retval ptr End of the newly packed binary - */ -static unsigned char *dump_stailq(struct ListHead *l, unsigned char *d, int *off, bool convert) -{ - unsigned int counter = 0; - unsigned int start_off = *off; - - d = dump_int(0xdeadbeef, d, off); - - struct ListNode *np = NULL; - STAILQ_FOREACH(np, l, entries) - { - d = dump_char(np->data, d, off, convert); - counter++; - } - - memcpy(d + start_off, &counter, sizeof(int)); - - return d; -} - -/** - * restore_stailq - Unpack a STAILQ from a binary blob - * @param l List to add to - * @param d Binary blob to read from - * @param off Offset into the blob - * @param convert If true, the strings will be converted from utf-8 - */ -static void restore_stailq(struct ListHead *l, const unsigned char *d, int *off, bool convert) -{ - unsigned int counter; - - restore_int(&counter, d, off); - - struct ListNode *np = NULL; - while (counter) - { - np = mutt_list_insert_tail(l, NULL); - restore_char(&np->data, d, off, convert); - counter--; - } -} - -/** - * dump_buffer - Pack a Buffer into a binary blob - * @param b Buffer to pack - * @param d Binary blob to add to - * @param off Offset into the blob - * @param convert If true, the strings will be converted to utf-8 - * @retval ptr End of the newly packed binary - */ -static unsigned char *dump_buffer(struct Buffer *b, unsigned char *d, int *off, bool convert) -{ - if (!b) - { - d = dump_int(0, d, off); - return d; - } - else - d = dump_int(1, d, off); - - d = dump_char_size(b->data, d, off, b->dsize + 1, convert); - d = dump_int(b->dptr - b->data, d, off); - d = dump_int(b->dsize, d, off); - d = dump_int(b->destroy, d, off); - - return d; -} - -/** - * restore_buffer - Unpack a Buffer from a binary blob - * @param b Store the unpacked Buffer here - * @param d Binary blob to read from - * @param off Offset into the blob - * @param convert If true, the strings will be converted from utf-8 - */ -static void restore_buffer(struct Buffer **b, const unsigned char *d, int *off, bool convert) -{ - unsigned int used; - unsigned int offset; - restore_int(&used, d, off); - if (!used) - { - return; - } - - *b = mutt_mem_malloc(sizeof(struct Buffer)); - - restore_char(&(*b)->data, d, off, convert); - restore_int(&offset, d, off); - (*b)->dptr = (*b)->data + offset; - restore_int(&used, d, off); - (*b)->dsize = used; - restore_int(&used, d, off); - (*b)->destroy = used; -} - -/** - * dump_parameter - Pack a Parameter into a binary blob - * @param p Parameter to pack - * @param d Binary blob to add to - * @param off Offset into the blob - * @param convert If true, the strings will be converted to utf-8 - * @retval ptr End of the newly packed binary - */ -static unsigned char *dump_parameter(struct ParameterList *p, unsigned char *d, - int *off, bool convert) -{ - unsigned int counter = 0; - unsigned int start_off = *off; - - d = dump_int(0xdeadbeef, d, off); - - struct Parameter *np = NULL; - TAILQ_FOREACH(np, p, entries) - { - d = dump_char(np->attribute, d, off, false); - d = dump_char(np->value, d, off, convert); - counter++; - } - - memcpy(d + start_off, &counter, sizeof(int)); - - return d; -} - -/** - * restore_parameter - Unpack a Parameter from a binary blob - * @param p Store the unpacked Parameter here - * @param d Binary blob to read from - * @param off Offset into the blob - * @param convert If true, the strings will be converted from utf-8 - */ -static void restore_parameter(struct ParameterList *p, const unsigned char *d, - int *off, bool convert) -{ - unsigned int counter; - - restore_int(&counter, d, off); - - struct Parameter *np = NULL; - while (counter) - { - np = mutt_param_new(); - restore_char(&np->attribute, d, off, false); - restore_char(&np->value, d, off, convert); - TAILQ_INSERT_TAIL(p, np, entries); - counter--; - } -} - -/** - * dump_body - Pack an Body into a binary blob - * @param c Body to pack - * @param d Binary blob to add to - * @param off Offset into the blob - * @param convert If true, the strings will be converted to utf-8 - * @retval ptr End of the newly packed binary - */ -static unsigned char *dump_body(struct Body *c, unsigned char *d, int *off, bool convert) -{ - struct Body nb; - - memcpy(&nb, c, sizeof(struct Body)); - - /* some fields are not safe to cache */ - nb.content = NULL; - nb.charset = NULL; - nb.next = NULL; - nb.parts = NULL; - nb.hdr = NULL; - nb.aptr = NULL; - - lazy_realloc(&d, *off + sizeof(struct Body)); - memcpy(d + *off, &nb, sizeof(struct Body)); - *off += sizeof(struct Body); - - d = dump_char(nb.xtype, d, off, false); - d = dump_char(nb.subtype, d, off, false); - - d = dump_parameter(&nb.parameter, d, off, convert); - - d = dump_char(nb.description, d, off, convert); - d = dump_char(nb.form_name, d, off, convert); - d = dump_char(nb.filename, d, off, convert); - d = dump_char(nb.d_filename, d, off, convert); - - return d; -} - -/** - * restore_body - Unpack a Body from a binary blob - * @param c Store the unpacked Body here - * @param d Binary blob to read from - * @param off Offset into the blob - * @param convert If true, the strings will be converted from utf-8 - */ -static void restore_body(struct Body *c, const unsigned char *d, int *off, bool convert) -{ - memcpy(c, d + *off, sizeof(struct Body)); - *off += sizeof(struct Body); - - restore_char(&c->xtype, d, off, false); - restore_char(&c->subtype, d, off, false); - - TAILQ_INIT(&c->parameter); - restore_parameter(&c->parameter, d, off, convert); - - restore_char(&c->description, d, off, convert); - restore_char(&c->form_name, d, off, convert); - restore_char(&c->filename, d, off, convert); - restore_char(&c->d_filename, d, off, convert); -} - -/** - * dump_envelope - Pack an Envelope into a binary blob - * @param e Envelope to pack - * @param d Binary blob to add to - * @param off Offset into the blob - * @param convert If true, the strings will be converted to utf-8 - * @retval ptr End of the newly packed binary - */ -static unsigned char *dump_envelope(struct Envelope *e, unsigned char *d, int *off, bool convert) -{ - d = dump_address(e->return_path, d, off, convert); - d = dump_address(e->from, d, off, convert); - d = dump_address(e->to, d, off, convert); - d = dump_address(e->cc, d, off, convert); - d = dump_address(e->bcc, d, off, convert); - d = dump_address(e->sender, d, off, convert); - d = dump_address(e->reply_to, d, off, convert); - d = dump_address(e->mail_followup_to, d, off, convert); - - d = dump_char(e->list_post, d, off, convert); - d = dump_char(e->subject, d, off, convert); - - if (e->real_subj) - d = dump_int(e->real_subj - e->subject, d, off); - else - d = dump_int(-1, d, off); - - d = dump_char(e->message_id, d, off, false); - d = dump_char(e->supersedes, d, off, false); - d = dump_char(e->date, d, off, false); - d = dump_char(e->x_label, d, off, convert); - - d = dump_buffer(e->spam, d, off, convert); - - d = dump_stailq(&e->references, d, off, false); - d = dump_stailq(&e->in_reply_to, d, off, false); - d = dump_stailq(&e->userhdrs, d, off, convert); - -#ifdef USE_NNTP - d = dump_char(e->xref, d, off, false); - d = dump_char(e->followup_to, d, off, false); - d = dump_char(e->x_comment_to, d, off, convert); -#endif - - return d; -} - -/** - * restore_envelope - Unpack an Envelope from a binary blob - * @param e Store the unpacked Envelope here - * @param d Binary blob to read from - * @param off Offset into the blob - * @param convert If true, the strings will be converted from utf-8 - */ -static void restore_envelope(struct Envelope *e, const unsigned char *d, int *off, bool convert) -{ - int real_subj_off; - - restore_address(&e->return_path, d, off, convert); - restore_address(&e->from, d, off, convert); - restore_address(&e->to, d, off, convert); - restore_address(&e->cc, d, off, convert); - restore_address(&e->bcc, d, off, convert); - restore_address(&e->sender, d, off, convert); - restore_address(&e->reply_to, d, off, convert); - restore_address(&e->mail_followup_to, d, off, convert); - - restore_char(&e->list_post, d, off, convert); - restore_char(&e->subject, d, off, convert); - restore_int((unsigned int *) (&real_subj_off), d, off); - - if (real_subj_off >= 0) - e->real_subj = e->subject + real_subj_off; - else - e->real_subj = NULL; - - restore_char(&e->message_id, d, off, false); - restore_char(&e->supersedes, d, off, false); - restore_char(&e->date, d, off, false); - restore_char(&e->x_label, d, off, convert); - - restore_buffer(&e->spam, d, off, convert); - - restore_stailq(&e->references, d, off, false); - restore_stailq(&e->in_reply_to, d, off, false); - restore_stailq(&e->userhdrs, d, off, convert); - -#ifdef USE_NNTP - restore_char(&e->xref, d, off, false); - restore_char(&e->followup_to, d, off, false); - restore_char(&e->x_comment_to, d, off, convert); -#endif -} - /** * crc_matches - Is the CRC number correct? * @param d Binary blob to read CRC from * @param crc CRC to compare * @retval num 1 if true, 0 if not */ -static int crc_matches(const char *d, unsigned int crc) +static bool crc_matches(const char *d, unsigned int crc) { - int off = sizeof(union Validate); - unsigned int mycrc = 0; - if (!d) - return 0; + return false; - restore_int(&mycrc, (unsigned char *) d, &off); + unsigned int mycrc = *(unsigned int *)(d + sizeof(union Validate)); return (crc == mycrc); } @@ -782,109 +235,6 @@ static const char *hcache_per_folder(const char *path, const char *folder, hcach return hcpath; } -/** - * hcache_dump - Serialise a Header object - * @param h Header cache handle - * @param header Header to serialise - * @param off Size of the binary blob - * @param uidvalidity IMAP server identifier - * @retval ptr Binary blob representing the Header - * - * This function transforms a header into a char so that it is useable by - * db_store. - */ -static void *hcache_dump(header_cache_t *h, struct Header *header, int *off, - unsigned int uidvalidity) -{ - struct Header nh; - bool convert = !CharsetIsUtf8; - - *off = 0; - unsigned char *d = lazy_malloc(sizeof(union Validate)); - - if (uidvalidity == 0) - { - struct timeval now; - gettimeofday(&now, NULL); - memcpy(d, &now, sizeof(struct timeval)); - } - else - memcpy(d, &uidvalidity, sizeof(uidvalidity)); - *off += sizeof(union Validate); - - d = dump_int(h->crc, d, off); - - lazy_realloc(&d, *off + sizeof(struct Header)); - memcpy(&nh, header, sizeof(struct Header)); - - /* some fields are not safe to cache */ - nh.tagged = false; - nh.changed = false; - nh.threaded = false; - nh.recip_valid = false; - nh.searched = false; - nh.matched = false; - nh.collapsed = false; - nh.limited = false; - nh.num_hidden = 0; - nh.recipient = 0; - nh.pair = 0; - nh.attach_valid = false; - nh.path = NULL; - nh.tree = NULL; - nh.thread = NULL; - STAILQ_INIT(&nh.tags); -#ifdef MIXMASTER - STAILQ_INIT(&nh.chain); -#endif - nh.data = NULL; - - memcpy(d + *off, &nh, sizeof(struct Header)); - *off += sizeof(struct Header); - - d = dump_envelope(nh.env, d, off, convert); - d = dump_body(nh.content, d, off, convert); - d = dump_char(nh.maildir_flags, d, off, convert); - - return d; -} - -/** - * mutt_hcache_restore - Deserialise a Header object - * @param d Binary blob - * @retval ptr Reconstructed Header - */ -struct Header *mutt_hcache_restore(const unsigned char *d) -{ - int off = 0; - struct Header *h = mutt_header_new(); - bool convert = !CharsetIsUtf8; - - /* skip validate */ - off += sizeof(union Validate); - - /* skip crc */ - off += sizeof(unsigned int); - - memcpy(h, d + off, sizeof(struct Header)); - off += sizeof(struct Header); - - STAILQ_INIT(&h->tags); -#ifdef MIXMASTER - STAILQ_INIT(&h->chain); -#endif - - h->env = mutt_env_new(); - restore_envelope(h->env, d, &off, convert); - - h->content = mutt_body_new(); - restore_body(h->content, d, &off, convert); - - restore_char(&h->maildir_flags, d, &off, convert); - - return h; -} - /** * get_foldername - Where should the cache be stored? * @param folder Path to be canonicalised @@ -1058,7 +408,7 @@ int mutt_hcache_store(header_cache_t *h, const char *key, size_t keylen, if (!h) return -1; - data = hcache_dump(h, header, &dlen, uidvalidity); + data = mutt_hcache_dump(h, header, &dlen, uidvalidity); ret = mutt_hcache_store_raw(h, key, keylen, data, dlen); FREE(&data); diff --git a/hcache/hcache.h b/hcache/hcache.h index 2fa6e7db8..bfa730c86 100644 --- a/hcache/hcache.h +++ b/hcache/hcache.h @@ -29,6 +29,8 @@ * This module defines the user-visible header cache API, which is used within * neomutt to cache and restore mail header data. * + * @subpage hc_serial + * * @subpage hc_hcache * * Backends: @@ -48,12 +50,37 @@ #include #include +#include struct Header; + +/** + * struct HeaderCache - header cache structure + * + * This struct holds both the backend-agnostic and the backend-specific parts + * of the header cache. Backend code MUST initialize the fetch, store, + * delete and close function pointers in hcache_open, and MAY store + * backend-specific context in the ctx pointer. + */ +struct HeaderCache +{ + char *folder; + unsigned int crc; + void *ctx; +}; + typedef struct HeaderCache header_cache_t; typedef int (*hcache_namer_t)(const char *path, char *dest, size_t dlen); +/** + * union Validate - Header cache validity + */ +union Validate { + struct timeval timeval; + unsigned int uidvalidity; +}; + /* These Config Variables are only used in hcache/hcache.c */ extern char *HeaderCacheBackend; diff --git a/hcache/serialize.c b/hcache/serialize.c new file mode 100644 index 000000000..3de8bebac --- /dev/null +++ b/hcache/serialize.c @@ -0,0 +1,670 @@ +/** + * @file + * Email-object serialiser + * + * @authors + * Copyright (C) 2004 Thomas Glanzmann + * Copyright (C) 2004 Tobias Werth + * Copyright (C) 2004 Brian Fundakowski Feldman + * Copyright (C) 2016 Pietro Cerutti + * + * @copyright + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +/** + * @page hc_serial Email-object serialiser + * + * Email-object serialiser + */ + +#include "config.h" +#include +#include +#include "email/email.h" +#include "hcache.h" +#include "muttlib.h" + +/** + * lazy_malloc - Allocate some memory + * @param size Minimum size to allocate + * @retval ptr Allocated memory + * + * This block is likely to be lazy_realloc()'d repeatedly. + * It starts off with a minimum size of 4KiB. + */ +static void *lazy_malloc(size_t size) +{ + if (size < 4096) + size = 4096; + + return mutt_mem_malloc(size); +} + +/** + * lazy_realloc - Reallocate some memory + * @param ptr Pointer to resize + * @param size Minimum size + * + * The minimum size is 4KiB to avoid repeated resizing. + */ +static void lazy_realloc(void *ptr, size_t size) +{ + void **p = (void **) ptr; + + if (p != NULL && size < 4096) + return; + + mutt_mem_realloc(ptr, size); +} + +/** + * serial_dump_int - Pack an integer into a binary blob + * @param i Integer to save + * @param d Binary blob to add to + * @param off Offset into the blob + * @retval ptr End of the newly packed binary + */ +unsigned char *serial_dump_int(unsigned int i, unsigned char *d, int *off) +{ + lazy_realloc(&d, *off + sizeof(int)); + memcpy(d + *off, &i, sizeof(int)); + (*off) += sizeof(int); + + return d; +} + +/** + * serial_restore_int - Unpack an integer from a binary blob + * @param i Integer to write to + * @param d Binary blob to read from + * @param off Offset into the blob + */ +void serial_restore_int(unsigned int *i, const unsigned char *d, int *off) +{ + memcpy(i, d + *off, sizeof(int)); + (*off) += sizeof(int); +} + +/** + * serial_dump_char_size - Pack a fixed-length string into a binary blob + * @param c String to pack + * @param d Binary blob to add to + * @param off Offset into the blob + * @param size Size of the string + * @param convert If true, the strings will be converted to utf-8 + * @retval ptr End of the newly packed binary + */ +unsigned char *serial_dump_char_size(char *c, unsigned char *d, int *off, + ssize_t size, bool convert) +{ + char *p = c; + + if (!c) + { + size = 0; + d = serial_dump_int(size, d, off); + return d; + } + + if (convert && !mutt_str_is_ascii(c, size)) + { + p = mutt_str_substr_dup(c, c + size); + if (mutt_ch_convert_string(&p, Charset, "utf-8", 0) == 0) + { + size = mutt_str_strlen(p) + 1; + } + } + + d = serial_dump_int(size, d, off); + lazy_realloc(&d, *off + size); + memcpy(d + *off, p, size); + *off += size; + + if (p != c) + FREE(&p); + + return d; +} + +/** + * serial_dump_char - Pack a variable-length string into a binary blob + * @param c String to pack + * @param d Binary blob to add to + * @param off Offset into the blob + * @param convert If true, the strings will be converted to utf-8 + * @retval ptr End of the newly packed binary + */ +unsigned char *serial_dump_char(char *c, unsigned char *d, int *off, bool convert) +{ + return serial_dump_char_size(c, d, off, mutt_str_strlen(c) + 1, convert); +} + +/** + * serial_restore_char - Unpack a variable-length string from a binary blob + * @param c Store the unpacked string here + * @param d Binary blob to read from + * @param off Offset into the blob + * @param convert If true, the strings will be converted to utf-8 + */ +void serial_restore_char(char **c, const unsigned char *d, int *off, bool convert) +{ + unsigned int size; + serial_restore_int(&size, d, off); + + if (size == 0) + { + *c = NULL; + return; + } + + *c = mutt_mem_malloc(size); + memcpy(*c, d + *off, size); + if (convert && !mutt_str_is_ascii(*c, size)) + { + char *tmp = mutt_str_strdup(*c); + if (mutt_ch_convert_string(&tmp, "utf-8", Charset, 0) == 0) + { + FREE(c); + *c = tmp; + } + else + { + FREE(&tmp); + } + } + *off += size; +} + +/** + * serial_dump_address - Pack an Address into a binary blob + * @param a Address to pack + * @param d Binary blob to add to + * @param off Offset into the blob + * @param convert If true, the strings will be converted to utf-8 + * @retval ptr End of the newly packed binary + */ +unsigned char *serial_dump_address(struct Address *a, unsigned char *d, + int *off, bool convert) +{ + unsigned int counter = 0; + unsigned int start_off = *off; + + d = serial_dump_int(0xdeadbeef, d, off); + + while (a) + { + d = serial_dump_char(a->personal, d, off, convert); + d = serial_dump_char(a->mailbox, d, off, false); + d = serial_dump_int(a->group, d, off); + a = a->next; + counter++; + } + + memcpy(d + start_off, &counter, sizeof(int)); + + return d; +} + +/** + * serial_restore_address - Unpack an Address from a binary blob + * @param a Store the unpacked Address here + * @param d Binary blob to read from + * @param off Offset into the blob + * @param convert If true, the strings will be converted from utf-8 + */ +void serial_restore_address(struct Address **a, const unsigned char *d, + int *off, bool convert) +{ + unsigned int counter = 0; + unsigned int g = 0; + + serial_restore_int(&counter, d, off); + + while (counter) + { + *a = mutt_addr_new(); + serial_restore_char(&(*a)->personal, d, off, convert); + serial_restore_char(&(*a)->mailbox, d, off, false); + serial_restore_int(&g, d, off); + (*a)->group = g; + a = &(*a)->next; + counter--; + } + + *a = NULL; +} + +/** + * serial_dump_stailq - Pack a STAILQ into a binary blob + * @param l List to read from + * @param d Binary blob to add to + * @param off Offset into the blob + * @param convert If true, the strings will be converted to utf-8 + * @retval ptr End of the newly packed binary + */ +unsigned char *serial_dump_stailq(struct ListHead *l, unsigned char *d, + int *off, bool convert) +{ + unsigned int counter = 0; + unsigned int start_off = *off; + + d = serial_dump_int(0xdeadbeef, d, off); + + struct ListNode *np = NULL; + STAILQ_FOREACH(np, l, entries) + { + d = serial_dump_char(np->data, d, off, convert); + counter++; + } + + memcpy(d + start_off, &counter, sizeof(int)); + + return d; +} + +/** + * serial_restore_stailq - Unpack a STAILQ from a binary blob + * @param l List to add to + * @param d Binary blob to read from + * @param off Offset into the blob + * @param convert If true, the strings will be converted from utf-8 + */ +void serial_restore_stailq(struct ListHead *l, const unsigned char *d, + int *off, bool convert) +{ + unsigned int counter; + + serial_restore_int(&counter, d, off); + + struct ListNode *np = NULL; + while (counter) + { + np = mutt_list_insert_tail(l, NULL); + serial_restore_char(&np->data, d, off, convert); + counter--; + } +} + +/** + * serial_dump_buffer - Pack a Buffer into a binary blob + * @param b Buffer to pack + * @param d Binary blob to add to + * @param off Offset into the blob + * @param convert If true, the strings will be converted to utf-8 + * @retval ptr End of the newly packed binary + */ +unsigned char *serial_dump_buffer(struct Buffer *b, unsigned char *d, + int *off, bool convert) +{ + if (!b) + { + d = serial_dump_int(0, d, off); + return d; + } + else + d = serial_dump_int(1, d, off); + + d = serial_dump_char_size(b->data, d, off, b->dsize + 1, convert); + d = serial_dump_int(b->dptr - b->data, d, off); + d = serial_dump_int(b->dsize, d, off); + d = serial_dump_int(b->destroy, d, off); + + return d; +} + +/** + * serial_restore_buffer - Unpack a Buffer from a binary blob + * @param b Store the unpacked Buffer here + * @param d Binary blob to read from + * @param off Offset into the blob + * @param convert If true, the strings will be converted from utf-8 + */ +void serial_restore_buffer(struct Buffer **b, const unsigned char *d, + int *off, bool convert) +{ + unsigned int used; + unsigned int offset; + serial_restore_int(&used, d, off); + if (!used) + { + return; + } + + *b = mutt_mem_malloc(sizeof(struct Buffer)); + + serial_restore_char(&(*b)->data, d, off, convert); + serial_restore_int(&offset, d, off); + (*b)->dptr = (*b)->data + offset; + serial_restore_int(&used, d, off); + (*b)->dsize = used; + serial_restore_int(&used, d, off); + (*b)->destroy = used; +} + +/** + * serial_dump_parameter - Pack a Parameter into a binary blob + * @param p Parameter to pack + * @param d Binary blob to add to + * @param off Offset into the blob + * @param convert If true, the strings will be converted to utf-8 + * @retval ptr End of the newly packed binary + */ +unsigned char *serial_dump_parameter(struct ParameterList *p, + unsigned char *d, int *off, bool convert) +{ + unsigned int counter = 0; + unsigned int start_off = *off; + + d = serial_dump_int(0xdeadbeef, d, off); + + struct Parameter *np = NULL; + TAILQ_FOREACH(np, p, entries) + { + d = serial_dump_char(np->attribute, d, off, false); + d = serial_dump_char(np->value, d, off, convert); + counter++; + } + + memcpy(d + start_off, &counter, sizeof(int)); + + return d; +} + +/** + * serial_restore_parameter - Unpack a Parameter from a binary blob + * @param p Store the unpacked Parameter here + * @param d Binary blob to read from + * @param off Offset into the blob + * @param convert If true, the strings will be converted from utf-8 + */ +void serial_restore_parameter(struct ParameterList *p, + const unsigned char *d, int *off, bool convert) +{ + unsigned int counter; + + serial_restore_int(&counter, d, off); + + struct Parameter *np = NULL; + while (counter) + { + np = mutt_param_new(); + serial_restore_char(&np->attribute, d, off, false); + serial_restore_char(&np->value, d, off, convert); + TAILQ_INSERT_TAIL(p, np, entries); + counter--; + } +} + +/** + * serial_dump_body - Pack an Body into a binary blob + * @param c Body to pack + * @param d Binary blob to add to + * @param off Offset into the blob + * @param convert If true, the strings will be converted to utf-8 + * @retval ptr End of the newly packed binary + */ +unsigned char *serial_dump_body(struct Body *c, unsigned char *d, int *off, bool convert) +{ + struct Body nb; + + memcpy(&nb, c, sizeof(struct Body)); + + /* some fields are not safe to cache */ + nb.content = NULL; + nb.charset = NULL; + nb.next = NULL; + nb.parts = NULL; + nb.hdr = NULL; + nb.aptr = NULL; + + lazy_realloc(&d, *off + sizeof(struct Body)); + memcpy(d + *off, &nb, sizeof(struct Body)); + *off += sizeof(struct Body); + + d = serial_dump_char(nb.xtype, d, off, false); + d = serial_dump_char(nb.subtype, d, off, false); + + d = serial_dump_parameter(&nb.parameter, d, off, convert); + + d = serial_dump_char(nb.description, d, off, convert); + d = serial_dump_char(nb.form_name, d, off, convert); + d = serial_dump_char(nb.filename, d, off, convert); + d = serial_dump_char(nb.d_filename, d, off, convert); + + return d; +} + +/** + * serial_restore_body - Unpack a Body from a binary blob + * @param c Store the unpacked Body here + * @param d Binary blob to read from + * @param off Offset into the blob + * @param convert If true, the strings will be converted from utf-8 + */ +void serial_restore_body(struct Body *c, const unsigned char *d, int *off, bool convert) +{ + memcpy(c, d + *off, sizeof(struct Body)); + *off += sizeof(struct Body); + + serial_restore_char(&c->xtype, d, off, false); + serial_restore_char(&c->subtype, d, off, false); + + TAILQ_INIT(&c->parameter); + serial_restore_parameter(&c->parameter, d, off, convert); + + serial_restore_char(&c->description, d, off, convert); + serial_restore_char(&c->form_name, d, off, convert); + serial_restore_char(&c->filename, d, off, convert); + serial_restore_char(&c->d_filename, d, off, convert); +} + +/** + * serial_dump_envelope - Pack an Envelope into a binary blob + * @param e Envelope to pack + * @param d Binary blob to add to + * @param off Offset into the blob + * @param convert If true, the strings will be converted to utf-8 + * @retval ptr End of the newly packed binary + */ +unsigned char *serial_dump_envelope(struct Envelope *e, unsigned char *d, + int *off, bool convert) +{ + d = serial_dump_address(e->return_path, d, off, convert); + d = serial_dump_address(e->from, d, off, convert); + d = serial_dump_address(e->to, d, off, convert); + d = serial_dump_address(e->cc, d, off, convert); + d = serial_dump_address(e->bcc, d, off, convert); + d = serial_dump_address(e->sender, d, off, convert); + d = serial_dump_address(e->reply_to, d, off, convert); + d = serial_dump_address(e->mail_followup_to, d, off, convert); + + d = serial_dump_char(e->list_post, d, off, convert); + d = serial_dump_char(e->subject, d, off, convert); + + if (e->real_subj) + d = serial_dump_int(e->real_subj - e->subject, d, off); + else + d = serial_dump_int(-1, d, off); + + d = serial_dump_char(e->message_id, d, off, false); + d = serial_dump_char(e->supersedes, d, off, false); + d = serial_dump_char(e->date, d, off, false); + d = serial_dump_char(e->x_label, d, off, convert); + + d = serial_dump_buffer(e->spam, d, off, convert); + + d = serial_dump_stailq(&e->references, d, off, false); + d = serial_dump_stailq(&e->in_reply_to, d, off, false); + d = serial_dump_stailq(&e->userhdrs, d, off, convert); + +#ifdef USE_NNTP + d = serial_dump_char(e->xref, d, off, false); + d = serial_dump_char(e->followup_to, d, off, false); + d = serial_dump_char(e->x_comment_to, d, off, convert); +#endif + + return d; +} + +/** + * serial_restore_envelope - Unpack an Envelope from a binary blob + * @param e Store the unpacked Envelope here + * @param d Binary blob to read from + * @param off Offset into the blob + * @param convert If true, the strings will be converted from utf-8 + */ +void serial_restore_envelope(struct Envelope *e, const unsigned char *d, + int *off, bool convert) +{ + int real_subj_off; + + serial_restore_address(&e->return_path, d, off, convert); + serial_restore_address(&e->from, d, off, convert); + serial_restore_address(&e->to, d, off, convert); + serial_restore_address(&e->cc, d, off, convert); + serial_restore_address(&e->bcc, d, off, convert); + serial_restore_address(&e->sender, d, off, convert); + serial_restore_address(&e->reply_to, d, off, convert); + serial_restore_address(&e->mail_followup_to, d, off, convert); + + serial_restore_char(&e->list_post, d, off, convert); + serial_restore_char(&e->subject, d, off, convert); + serial_restore_int((unsigned int *) (&real_subj_off), d, off); + + if (real_subj_off >= 0) + e->real_subj = e->subject + real_subj_off; + else + e->real_subj = NULL; + + serial_restore_char(&e->message_id, d, off, false); + serial_restore_char(&e->supersedes, d, off, false); + serial_restore_char(&e->date, d, off, false); + serial_restore_char(&e->x_label, d, off, convert); + + serial_restore_buffer(&e->spam, d, off, convert); + + serial_restore_stailq(&e->references, d, off, false); + serial_restore_stailq(&e->in_reply_to, d, off, false); + serial_restore_stailq(&e->userhdrs, d, off, convert); + +#ifdef USE_NNTP + serial_restore_char(&e->xref, d, off, false); + serial_restore_char(&e->followup_to, d, off, false); + serial_restore_char(&e->x_comment_to, d, off, convert); +#endif +} + +/** + * mutt_hcache_dump - Serialise a Header object + * @param h Header cache handle + * @param header Header to serialise + * @param off Size of the binary blob + * @param uidvalidity IMAP server identifier + * @retval ptr Binary blob representing the Header + * + * This function transforms a header into a char so that it is useable by + * db_store. + */ +void *mutt_hcache_dump(header_cache_t *h, const struct Header *header, int *off, + unsigned int uidvalidity) +{ + struct Header nh; + bool convert = !CharsetIsUtf8; + + *off = 0; + unsigned char *d = lazy_malloc(sizeof(union Validate)); + + if (uidvalidity == 0) + { + struct timeval now; + gettimeofday(&now, NULL); + memcpy(d, &now, sizeof(struct timeval)); + } + else + memcpy(d, &uidvalidity, sizeof(uidvalidity)); + *off += sizeof(union Validate); + + d = serial_dump_int(h->crc, d, off); + + lazy_realloc(&d, *off + sizeof(struct Header)); + memcpy(&nh, header, sizeof(struct Header)); + + /* some fields are not safe to cache */ + nh.tagged = false; + nh.changed = false; + nh.threaded = false; + nh.recip_valid = false; + nh.searched = false; + nh.matched = false; + nh.collapsed = false; + nh.limited = false; + nh.num_hidden = 0; + nh.recipient = 0; + nh.pair = 0; + nh.attach_valid = false; + nh.path = NULL; + nh.tree = NULL; + nh.thread = NULL; + STAILQ_INIT(&nh.tags); +#ifdef MIXMASTER + STAILQ_INIT(&nh.chain); +#endif + nh.data = NULL; + + memcpy(d + *off, &nh, sizeof(struct Header)); + *off += sizeof(struct Header); + + d = serial_dump_envelope(nh.env, d, off, convert); + d = serial_dump_body(nh.content, d, off, convert); + d = serial_dump_char(nh.maildir_flags, d, off, convert); + + return d; +} + +/** + * mutt_hcache_restore - Deserialise a Header object + * @param d Binary blob + * @retval ptr Reconstructed Header + */ +struct Header *mutt_hcache_restore(const unsigned char *d) +{ + int off = 0; + struct Header *h = mutt_header_new(); + bool convert = !CharsetIsUtf8; + + /* skip validate */ + off += sizeof(union Validate); + + /* skip crc */ + off += sizeof(unsigned int); + + memcpy(h, d + off, sizeof(struct Header)); + off += sizeof(struct Header); + + STAILQ_INIT(&h->tags); +#ifdef MIXMASTER + STAILQ_INIT(&h->chain); +#endif + + h->env = mutt_env_new(); + serial_restore_envelope(h->env, d, &off, convert); + + h->content = mutt_body_new(); + serial_restore_body(h->content, d, &off, convert); + + serial_restore_char(&h->maildir_flags, d, &off, convert); + + return h; +} diff --git a/hcache/serialize.h b/hcache/serialize.h new file mode 100644 index 000000000..ddf3c1868 --- /dev/null +++ b/hcache/serialize.h @@ -0,0 +1,61 @@ +/** + * @file + * Email-object serialiser + * + * @authors + * Copyright (C) 2004 Thomas Glanzmann + * Copyright (C) 2004 Tobias Werth + * Copyright (C) 2004 Brian Fundakowski Feldman + * Copyright (C) 2016 Pietro Cerutti + * + * @copyright + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +#ifndef _MUTT_HCACHE_SERIALIZE_H +#define _MUTT_HCACHE_SERIALIZE_H + +#include "hcache.h" + +struct Address; +struct Body; +struct Buffer; +struct Envelope; +struct Header; +struct ListHead; +struct ParameterList; + +unsigned char *serial_dump_address(struct Address *a, unsigned char *d, int *off, bool convert); +unsigned char *serial_dump_body(struct Body *c, unsigned char *d, int *off, bool convert); +unsigned char *serial_dump_buffer(struct Buffer *b, unsigned char *d, int *off, bool convert); +unsigned char *serial_dump_char(char *c, unsigned char *d, int *off, bool convert); +unsigned char *serial_dump_char_size(char *c, unsigned char *d, int *off, ssize_t size, bool convert); +unsigned char *serial_dump_envelope(struct Envelope *e, unsigned char *d, int *off, bool convert); +unsigned char *serial_dump_int(unsigned int i, unsigned char *d, int *off); +unsigned char *serial_dump_parameter(struct ParameterList *p, unsigned char *d, int *off, bool convert); +unsigned char *serial_dump_stailq(struct ListHead *l, unsigned char *d, int *off, bool convert); + +void serial_restore_address(struct Address **a, const unsigned char *d, int *off, bool convert); +void serial_restore_body(struct Body *c, const unsigned char *d, int *off, bool convert); +void serial_restore_buffer(struct Buffer **b, const unsigned char *d, int *off, bool convert); +void serial_restore_char(char **c, const unsigned char *d, int *off, bool convert); +void serial_restore_envelope(struct Envelope *e, const unsigned char *d, int *off, bool convert); +void serial_restore_int(unsigned int *i, const unsigned char *d, int *off); +void serial_restore_parameter(struct ParameterList *p, const unsigned char *d, int *off, bool convert); +void serial_restore_stailq(struct ListHead *l, const unsigned char *d, int *off, bool convert); + +void * mutt_hcache_dump(header_cache_t *h, const struct Header *header, int *off, unsigned int uidvalidity); +struct Header *mutt_hcache_restore(const unsigned char *d); + +#endif /* _MUTT_HCACHE_SERIALIZE_H */ -- 2.50.1