From: Thomas Glanzmann Date: Wed, 26 Jan 2005 13:36:44 +0000 (+0000) Subject: IMAP header cache. X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=a70dfb5d9509bfc67e014562e039645fe0d6e293;p=neomutt IMAP header cache. --- diff --git a/Makefile.am b/Makefile.am index 7b7876424..060a9bda5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -63,7 +63,7 @@ EXTRA_mutt_SOURCES = account.c md5c.c mutt_sasl.c mutt_socket.c mutt_ssl.c \ pgplib.c sha1.c pgpmicalg.c gnupgparse.c resize.c dotlock.c remailer.c \ browser.h mbyte.h remailer.h url.h mutt_ssl_nss.c \ crypt-mod-pgp-classic.c crypt-mod-smime-classic.c \ - pgppacket.c mutt_idna.h + pgppacket.c mutt_idna.h hcache.c EXTRA_DIST = COPYRIGHT GPL OPS OPS.PGP OPS.CRYPT OPS.SMIME TODO \ configure acconfig.h account.h \ diff --git a/configure.in b/configure.in index 227f5bac0..3d3f8bc70 100644 --- a/configure.in +++ b/configure.in @@ -694,11 +694,6 @@ if test "x$with_idn" != "xno"; then AC_CHECK_FUNCS(idna_to_ascii_lz idna_to_ascii_from_locale) fi -if test "$need_md5" = "yes" -then - MUTT_LIB_OBJECTS="$MUTT_LIB_OBJECTS md5c.o" -fi - dnl -- end socket -- AC_ARG_ENABLE(debug, [ --enable-debug Enable debugging support], @@ -770,6 +765,96 @@ AC_ARG_ENABLE(exact-address, [ --enable-exact-address Enable regeneration o fi]) +dnl -- start cache -- +AC_ARG_ENABLE(hcache, [ --enable-hcache Enable header caching], +[if test x$enableval = xyes; then + AC_DEFINE(USE_HCACHE, 1, [Enable header caching]) + MUTT_LIB_OBJECTS="$MUTT_LIB_OBJECTS hcache.o" + + OLDCPPFLAGS="$CPPFLAGS" + OLDLIBS="$LIBS" + + need_md5="yes" + ac_prefer_gdbm=yes + AC_ARG_WITH(gdbm, [ --without-gdbm Get rid of gdbm even if it is available ], + ac_prefer_gdbm=$withval) + if test x$ac_prefer_gdbm != xno; then + CPPFLAGS="$OLDCPPFLAGS" + LIBS="$OLDLIBS -lgdbm"; + AC_CACHE_CHECK(for gdbm_open, ac_cv_gdbmopen,[ + ac_cv_gdbmopen=no + AC_TRY_LINK([#include ],[gdbm_open(0,0,0,0,0);],[ac_cv_gdbmopen=yes]) + ]) + fi + + ac_bdb_prefix=yes + AC_ARG_WITH(bdb, [ --with-bdb[=DIR] Use BerkeleyDB4 if gdbm is not available ], + ac_bdb_prefix=$withval) + if test x$ac_bdb_prefix != xno -a x$ac_cv_gdbmopen != xyes; then + test x$ac_bdb_prefix = xyes && ac_bdb_prefix="$mutt_cv_prefix /opt/csw/bdb4 /opt /usr/local /usr" + for d in $ac_bdb_prefix; do + bdbpfx="$bdbpfx $d" + for v in BerkeleyDB.4.3 BerkeleyDB.4.2 BerkeleyDB.4.1; do + bdbpfx="$bdbpfx $d/$v" + done + done + BDB_VERSIONS="db-4 db4 db-4.3 db4.3 db43 db-4.2 db4.2 db42 db-4.1 db4.1 db41 db ''" + AC_MSG_CHECKING([for BerkeleyDB > 4.0]) + for d in $bdbpfx; do + BDB_INCLUDE_DIR="" + BDB_LIB_DIR="" + for v in / $BDB_VERSIONS; do + if test -r "$d/include/$v/db.h"; then + BDB_INCLUDE_DIR="$d/include/$v" + if test -d "$d/lib/$v"; then + BDB_LIB_DIR="$d/lib/$v" + for l in `echo $BDB_VERSIONS`; do + CPPFLAGS="-I$BDB_INCLUDE_DIR $OLDCPPFLAGS" + LIBS="$OLDLIBS -L$BDB_LIB_DIR -l$l" + AC_TRY_LINK([ + #include + #include + ],[ + DB *db = NULL; + db->open(db,NULL,NULL,NULL,0,0,0); + ],[ + ac_cv_dbcreate=yes + BDB_LIB="$l" + break + ]) + done + test x$ac_dbcreate = xyes && break + fi + fi + done + test x$BDB_LIB != x && break + done + if test x$ac_cv_dbcreate = xyes; then + AC_MSG_RESULT(yes) + else + AC_MSG_RESULT(no) + fi + fi + + if test x$ac_cv_gdbmopen = xyes; then + CPPFLAGS="$OLDCPPFLAGS" + LIBS="$OLDLIBS -lgdbm"; + AC_DEFINE(HAVE_GDBM, 1, [GDBM Support]) + elif test x$ac_cv_dbcreate = xyes; then + CPPFLAGS="-I$BDB_INCLUDE_DIR $OLDCPPFLAGS" + LIBS="$OLDLIBS -L$BDB_LIB_DIR -l$BDB_LIB" + AC_DEFINE(HAVE_DB4, 1, [Sleepycat DB4 Support]) + else + AC_MSG_ERROR(You need Sleepycat DB4 or GDBM for --enable-hcache) + fi +fi]) +dnl -- end cache -- + +if test "$need_md5" = "yes" +then + MUTT_LIB_OBJECTS="$MUTT_LIB_OBJECTS md5c.o" +fi + AC_SUBST(MUTTLIBS) AC_SUBST(MUTT_LIB_OBJECTS) AC_SUBST(LIBIMAP) diff --git a/globals.h b/globals.h index 327328c2c..2d18651ec 100644 --- a/globals.h +++ b/globals.h @@ -65,6 +65,10 @@ WHERE char *Ispell; WHERE char *Locale; WHERE char *MailcapPath; WHERE char *Maildir; +#if USE_HCACHE +WHERE char *HeaderCache; +WHERE char *HeaderCachePageSize; +#endif WHERE char *MhFlagged; WHERE char *MhReplied; WHERE char *MhUnseen; diff --git a/hcache.c b/hcache.c new file mode 100644 index 000000000..828c302c7 --- /dev/null +++ b/hcache.c @@ -0,0 +1,901 @@ +/* + * Copyright (C) 2004 Thomas Glanzmann + * Copyright (C) 2004 Tobias Werth + * Copyright (C) 2004 Brian Fundakowski Feldman + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111, USA. + */ + +#if HAVE_CONFIG_H +#include "config.h" +#endif /* HAVE_CONFIG_H */ + +#if HAVE_GDBM +#include +#elif HAVE_DB4 +#include +#endif + +#include +#include +#include "mutt.h" +#ifdef USE_IMAP +#include "message.h" +#endif +#include "mime.h" +#include "mx.h" +#include "lib.h" +#include "md5.h" + +#if HAVE_GDBM +static struct +header_cache +{ + GDBM_FILE db; + char *folder; + unsigned int crc; +} HEADER_CACHE; +#elif HAVE_DB4 +static struct +header_cache +{ + DB_ENV *env; + DB *db; + unsigned int crc; + int fd; + char lockfile[_POSIX_PATH_MAX]; +} HEADER_CACHE; +#endif + +typedef union +{ + struct timeval timeval; + unsigned long long uid_validity; +} validate; + +static void * +lazy_malloc(size_t siz) +{ + if (0 < siz && siz < 4096) { + siz = 4096; + } + + return safe_malloc(siz); +} + +static void +lazy_realloc(void *ptr, size_t siz) +{ + void **p = (void **)ptr; + + if ( p != NULL + && 0 < siz + && siz < 4096) { + return; + } + + safe_realloc(ptr, siz); +} + +static unsigned char * +dump_int(unsigned int i, unsigned char *d, int *off) +{ + lazy_realloc(&d, *off + sizeof(int)); + memcpy(d + *off, &i, sizeof(int)); + (*off) += sizeof(int); + + return d; +} + +static void +restore_int(unsigned int *i, const unsigned char *d, int *off) +{ + memcpy(i, d + *off, sizeof(int)); + (*off) += sizeof(int); +} + +static unsigned char * +dump_char(char *c, unsigned char *d, int *off) +{ + unsigned int size; + + if (c == NULL) { + size = 0; + d = dump_int(size, d, off); + return d; + } + + size = mutt_strlen(c) + 1; + d = dump_int(size, d, off); + lazy_realloc(&d, *off + size); + memcpy(d + *off, c, size); + *off += size; + + return d; +} + +static unsigned char * +dump_char_size(char *c, unsigned char *d, int *off, ssize_t size) +{ + if (c == NULL) { + size = 0; + d = dump_int(size, d, off); + return d; + } + + d = dump_int(size, d, off); + lazy_realloc(&d, *off + size); + memcpy(d + *off, c, size); + *off += size; + + return d; +} + +static void +restore_char(char **c, const unsigned char *d, int *off) +{ + unsigned int size; + restore_int(&size, d, off); + + if (size == 0) { + *c = NULL; + return; + } + + *c = safe_malloc(size); + memcpy(*c, d + *off, size); + *off += size; +} + +static unsigned char * +dump_address(ADDRESS *a, unsigned char *d, int *off) +{ + unsigned int counter = 0; + unsigned int start_off = *off; + + d = dump_int(0xdeadbeef, d, off); + + while (a) { +#ifdef EXACT_ADDRESS + d = dump_char(a->val, d, off); +#endif + d = dump_char(a->personal, d, off); + d = dump_char(a->mailbox, d, off); + d = dump_int(a->group, d, off); + a = a->next; + counter++; + } + + memcpy(d + start_off, &counter, sizeof(int)); + + return d; +} + +static void +restore_address(ADDRESS **a, const unsigned char *d, int *off) +{ + unsigned int counter; + + restore_int(&counter, d, off); + + while (counter) { + *a = safe_malloc(sizeof(ADDRESS)); +#ifdef EXACT_ADDRESS + restore_char(&(*a)->val, d, off); +#endif + restore_char(&(*a)->personal, d, off); + restore_char(&(*a)->mailbox, d, off); + restore_int((unsigned int *)&(*a)->group, d, off); + a = &(*a)->next; + counter--; + } + + *a = NULL; +} + +static unsigned char * +dump_list(LIST *l, unsigned char *d, int *off) +{ + unsigned int counter = 0; + unsigned int start_off = *off; + + d = dump_int(0xdeadbeef, d, off); + + while (l) { + d = dump_char(l->data, d, off); + l = l->next; + counter++; + } + + memcpy(d + start_off, &counter, sizeof(int)); + + return d; +} + +static void +restore_list(LIST **l, const unsigned char *d, int *off) +{ + unsigned int counter; + + restore_int(&counter, d, off); + + while (counter) { + *l = safe_malloc(sizeof(LIST)); + restore_char(&(*l)->data, d, off); + l = &(*l)->next; + counter--; + } + + *l = NULL; +} + +static unsigned char * +dump_buffer(BUFFER *b, unsigned char *d, int *off) +{ + if (! b) { + d = dump_int(0, d, off); + return d; + } else { + d = dump_int(1, d, off); + } + + d = dump_char_size(b->data, d, off, b->dsize + 1); + d = dump_int(b->dptr - b->data, d, off); + d = dump_int(b->dsize, d, off); + d = dump_int(b->destroy, d, off); + + return d; +} + +static void +restore_buffer(BUFFER **b, const unsigned char *d, int *off) +{ + unsigned int used; + unsigned int offset; + restore_int(&used, d, off); + if (! used) { + return; + } + + *b = safe_malloc(sizeof(BUFFER)); + + restore_char(& (*b)->data, d, off); + restore_int(& offset, d, off); + (*b)->dptr = (*b)->data + offset; + restore_int(& (*b)->dsize, d, off); + restore_int((unsigned int *) & (*b)->destroy, d, off); +} + +static unsigned char * +dump_parameter(PARAMETER *p, unsigned char *d, int *off) +{ + unsigned int counter = 0; + unsigned int start_off = *off; + + d = dump_int(0xdeadbeef, d, off); + + while (p) { + d = dump_char(p->attribute, d, off); + d = dump_char(p->value, d, off); + p = p->next; + counter++; + } + + memcpy(d + start_off, &counter, sizeof(int)); + + return d; +} + +static void +restore_parameter(PARAMETER **p, const unsigned char *d, int *off) +{ + unsigned int counter; + + restore_int(&counter, d, off); + + while (counter) { + *p = safe_malloc(sizeof(PARAMETER)); + restore_char(&(*p)->attribute, d, off); + restore_char(&(*p)->value, d, off); + p = &(*p)->next; + counter--; + } + + *p = NULL; +} + +static unsigned char * +dump_body(BODY *c, unsigned char *d, int *off) +{ + lazy_realloc(&d, *off + sizeof(BODY)); + memcpy(d + *off, c, sizeof(BODY)); + *off += sizeof(BODY); + + d = dump_char(c->xtype, d, off); + d = dump_char(c->subtype, d, off); + + d = dump_parameter(c->parameter, d, off); + + d = dump_char(c->description, d, off); + d = dump_char(c->form_name, d, off); + d = dump_char(c->filename, d, off); + d = dump_char(c->d_filename, d, off); + + return d; +} + +static void +restore_body(BODY *c, const unsigned char *d, int *off) +{ + memcpy(c, d + *off, sizeof(BODY)); + *off += sizeof(BODY); + + restore_char(& c->xtype, d, off); + restore_char(& c->subtype, d, off); + + restore_parameter(& c->parameter, d, off); + + restore_char(& c->description, d, off); + restore_char(& c->form_name, d, off); + restore_char(& c->filename, d, off); + restore_char(& c->d_filename, d, off); +} + +static unsigned char * +dump_envelope(ENVELOPE *e, unsigned char *d, int *off) +{ + d = dump_address(e->return_path, d, off); + d = dump_address(e->from, d, off); + d = dump_address(e->to, d, off); + d = dump_address(e->cc, d, off); + d = dump_address(e->bcc, d, off); + d = dump_address(e->sender, d, off); + d = dump_address(e->reply_to, d, off); + d = dump_address(e->mail_followup_to, d, off); + + d = dump_char(e->list_post, d, off); + d = dump_char(e->subject, d, off); + if (e->real_subj) { + d = dump_int(e->real_subj - e->subject, d, off); + } else { + d = dump_int(-1, d, off); + } + d = dump_char(e->message_id, d, off); + d = dump_char(e->supersedes, d, off); + d = dump_char(e->date, d, off); + d = dump_char(e->x_label, d, off); + + d = dump_buffer(e->spam, d, off); + + d = dump_list(e->references, d, off); + d = dump_list(e->in_reply_to, d, off); + d = dump_list(e->userhdrs, d, off); + + return d; +} + +static void +restore_envelope(ENVELOPE *e, const unsigned char *d, int *off) +{ + int real_subj_off; + + restore_address(& e->return_path, d, off); + restore_address(& e->from, d, off); + restore_address(& e->to, d, off); + restore_address(& e->cc, d, off); + restore_address(& e->bcc, d, off); + restore_address(& e->sender, d, off); + restore_address(& e->reply_to, d, off); + restore_address(& e->mail_followup_to, d, off); + + restore_char(& e->list_post, d, off); + restore_char(& e->subject, d, off); + restore_int((unsigned int *) (& real_subj_off), d, off); + if (0 <= real_subj_off) { + e->real_subj = e->subject + real_subj_off; + } else { + e->real_subj = NULL; + } + restore_char(& e->message_id, d, off); + restore_char(& e->supersedes, d, off); + restore_char(& e->date, d, off); + restore_char(& e->x_label, d, off); + + restore_buffer(& e->spam, d, off); + + restore_list(& e->references, d, off); + restore_list(& e->in_reply_to, d, off); + restore_list(& e->userhdrs, d, off); +} + +static +unsigned int crc32(unsigned int crc, unsigned char const *p, size_t len) +{ + int i; + while (len--) { + crc ^= *p++; + for (i = 0; i < 8; i++) + crc = (crc >> 1) ^ ((crc & 1) ? 0xedb88320 : 0); + } + return crc; +} + +static int +generate_crc32() +{ + int crc = 0; + SPAM_LIST *sp = SpamList; + RX_LIST *rx = NoSpamList; + + crc = crc32(crc, (unsigned char const *) "sithglan@stud.uni-erlangen.de[sithglan]|hcache.c|20041125191652|14839", mutt_strlen("sithglan@stud.uni-erlangen.de[sithglan]|hcache.c|20041125191652|14839")); + +#if HAVE_LANGINFO_CODESET + crc = crc32(crc, (unsigned char const *) Charset, mutt_strlen(Charset)); + crc = crc32(crc, (unsigned char const *) "HAVE_LANGINFO_CODESET", mutt_strlen("HAVE_LANGINFO_CODESET")); +#endif + +#if EXACT_ADDRESS + crc = crc32(crc, (unsigned char const *) "EXACT_ADDRESS", mutt_strlen("EXACT_ADDRESS")); +#endif + +#ifdef USE_POP + crc = crc32(crc, (unsigned char const *) "USE_POP", mutt_strlen("USE_POP")); +#endif + +#ifdef MIXMASTER + crc = crc32(crc, (unsigned char const *) "MIXMASTER", mutt_strlen("MIXMASTER")); +#endif + +#ifdef USE_IMAP + crc = crc32(crc, (unsigned char const *) "USE_IMAP", mutt_strlen("USE_IMAP")); + crc = crc32(crc, (unsigned char const *) ImapHeaders, mutt_strlen(ImapHeaders)); +#endif + while(sp) { + crc = crc32(crc, (unsigned char const *) sp->rx->pattern, mutt_strlen(sp->rx->pattern)); + sp = sp->next; + } + + crc = crc32(crc, (unsigned char const *) "SPAM_SEPERATOR", mutt_strlen("SPAM_SEPERATOR")); + + while(rx) { + crc = crc32(crc, (unsigned char const *) rx->rx->pattern, mutt_strlen(rx->rx->pattern)); + rx = rx->next; + } + + return crc; +} + +static int +crc32_matches(const char *d, unsigned int crc) +{ + int off = sizeof(validate); + unsigned int mycrc = 0; + + if (! d) { + return 0; + } + + restore_int(&mycrc, (unsigned char *) d, &off); + + return (crc == mycrc); +} + +/* Append md5sumed folder to path if path is a directory. */ +static const char * +mutt_hcache_per_folder(const char *path, const char *folder) +{ + static char mutt_hcache_per_folder_path[_POSIX_PATH_MAX]; + struct stat path_stat; + MD5_CTX md5; + unsigned char md5sum[16]; + int ret; + + ret = stat(path, &path_stat); + if (ret < 0) { + return path; + } + + if (! S_ISDIR(path_stat.st_mode)) { + return path; + } + + MD5Init(&md5); + MD5Update(&md5, (unsigned char *) folder, strlen(folder)); + MD5Final(md5sum, &md5); + + ret = snprintf(mutt_hcache_per_folder_path, _POSIX_PATH_MAX, + "%s/%02x%02x%02x%02x%02x%02x%02x%02x" + "%02x%02x%02x%02x%02x%02x%02x%02x", + path, md5sum[0], md5sum[1], md5sum[2], md5sum[3], + md5sum[4], md5sum[5], md5sum[6], md5sum[7], md5sum[8], + md5sum[9], md5sum[10], md5sum[11], md5sum[12], + md5sum[13], md5sum[14], md5sum[15]); + + if (ret <= 0) { + return path; + } + + return mutt_hcache_per_folder_path; +} + +/* This function transforms a header into a char so that it is useable by + * db_store */ +static void * +mutt_hcache_dump(void *_db, HEADER *h, int *off, unsigned long long uid_validity) +{ + struct header_cache *db = _db; + unsigned char *d = NULL; + *off = 0; + + d = lazy_malloc(sizeof(validate)); + + if (uid_validity) { + memcpy(d, &uid_validity, sizeof(long long)); + } else { + struct timeval now; + gettimeofday(&now, NULL); + memcpy(d, &now, sizeof(struct timeval)); + } + *off += sizeof(validate); + + d = dump_int(db->crc, d, off); + + lazy_realloc(&d, *off + sizeof(HEADER)); + memcpy(d + *off, h, sizeof(HEADER)); + *off += sizeof(HEADER); + + d = dump_envelope(h->env, d, off); + d = dump_body(h->content, d, off); + d = dump_char(h->maildir_flags, d, off); + + return d; +} + +HEADER * +mutt_hcache_restore(const unsigned char *d, HEADER **oh) +{ + int off = 0; + HEADER *h = mutt_new_header(); + + /* skip validate */ + off += sizeof(validate); + + /* skip crc */ + off += sizeof(unsigned int); + + memcpy(h, d + off, sizeof(HEADER)); + off += sizeof(HEADER); + + h->env = mutt_new_envelope(); + restore_envelope(h->env, d, &off); + + h->content = mutt_new_body(); + restore_body(h->content, d, &off); + + restore_char(&h->maildir_flags, d, &off); + + /* this is needed for maildir style mailboxes */ + if (oh) { + h->old = (*oh)->old; + h->path = safe_strdup((*oh)->path); + mutt_free_header (oh); + } + + return h; +} + +#if HAVE_GDBM + +void * +mutt_hcache_open(const char *path, const char *folder) +{ + struct header_cache *h = safe_calloc(1, sizeof(HEADER_CACHE)); + int pagesize = atoi(HeaderCachePageSize) ? atoi(HeaderCachePageSize) : 16384; + h->db = NULL; + h->folder = safe_strdup (folder); + h->crc = generate_crc32(); + + if (! path || path[0] == '\0') { + FREE(& h->folder); + FREE(& h); + return NULL; + } + + path = mutt_hcache_per_folder(path, folder); + + h->db = gdbm_open((char *) path, pagesize, GDBM_WRCREAT, 00600, NULL); + if (h->db) { + return h; + } + + /* if rw failed try ro */ + h->db = gdbm_open((char *) path, pagesize, GDBM_READER, 00600, NULL); + if(h->db) { + return h; + } else { + FREE(& h->folder); + FREE(& h); + + return NULL; + } +} + +void +mutt_hcache_close(void *db) +{ + struct header_cache *h = db; + + if (! h) { + return; + } + + gdbm_close(h->db); + FREE(& h->folder); + FREE(& h); +} + +void * +mutt_hcache_fetch(void *db, const char *filename, size_t (*keylen)(const char *fn)) +{ + struct header_cache *h = db; + datum key; + datum data; + char path[_POSIX_PATH_MAX]; + + if (! h) { + return NULL; + } + + strncpy(path, h->folder, sizeof(path)); + safe_strcat (path, sizeof (path), filename); + + key.dptr = path; + key.dsize = keylen(path); + + data = gdbm_fetch(h->db, key); + + if (! crc32_matches(data.dptr, h->crc)) { + FREE(&data.dptr); + return NULL; + } + + return data.dptr; +} + +int +mutt_hcache_store(void *db, const char *filename, HEADER *header, unsigned long long uid_validity, size_t (*keylen)(const char *fn)) +{ + struct header_cache *h = db; + datum key; + datum data; + char path[_POSIX_PATH_MAX]; + int ret; + + if (! h) { + return -1; + } + + strncpy(path, h->folder, sizeof(path)); + safe_strcat (path, sizeof (path), filename); + + key.dptr = path; + key.dsize = keylen(path); + + data.dptr = mutt_hcache_dump(db, header, &data.dsize, uid_validity); + + ret = gdbm_store(h->db, key, data, GDBM_REPLACE); + + FREE(& data.dptr); + + return ret; +} + +int +mutt_hcache_delete(void *db, const char *filename, size_t (*keylen)(const char *fn)) +{ + datum key; + struct header_cache *h = db; + char path[_POSIX_PATH_MAX]; + + if (! h) { + return -1; + } + + strncpy(path, h->folder, sizeof(path)); + safe_strcat (path, sizeof (path), filename); + + key.dptr = path; + key.dsize = keylen(path); + + return gdbm_delete(h->db, key); +} +#elif HAVE_DB4 + +static void +mutt_hcache_dbt_init(DBT *dbt, void *data, size_t len) +{ + dbt->data = data; + dbt->size = dbt->ulen = len; + dbt->dlen = dbt->doff = 0; + dbt->flags = DB_DBT_USERMEM; +} + +static void +mutt_hcache_dbt_empty_init(DBT *dbt) +{ + dbt->data = NULL; + dbt->size = dbt->ulen = dbt->dlen = dbt->doff = 0; + dbt->flags = 0; +} + +void * +mutt_hcache_open(const char *path, const char *folder) +{ + struct stat sb; + u_int32_t createflags = DB_CREATE; + int ret; + struct header_cache *h = calloc(1, sizeof(HEADER_CACHE)); + int pagesize = atoi(HeaderCachePageSize); + + + h->crc = generate_crc32(); + + if (! path || path[0] == '\0') { + FREE(& h); + return NULL; + } + + path = mutt_hcache_per_folder(path, folder); + + snprintf (h->lockfile, _POSIX_PATH_MAX, "%s-lock-hack", path); + + h->fd = open(h->lockfile, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); + if (h->fd < 0) { + FREE (&h); + return NULL; + } + + if (mx_lock_file(h->lockfile, h->fd, 1, 0, 5)) { + close(h->fd); + FREE (&h); + return NULL; + } + + ret = db_env_create(&h->env, 0); + if (ret) { + mx_unlock_file(h->lockfile, h->fd, 0); + close(h->fd); + FREE(& h); + return NULL; + } + + ret = h->env->open(h->env, NULL, DB_INIT_MPOOL | DB_CREATE | DB_PRIVATE, 0600); + if (! ret) { + ret = db_create(&h->db, h->env, 0); + if (ret) { + h->env->close(h->env, 0); + mx_unlock_file(h->lockfile, h->fd, 0); + close(h->fd); + FREE(& h); + return NULL; + } + } + + if (stat(path, &sb) != 0 && errno == ENOENT) { + createflags |= DB_EXCL; + h->db->set_pagesize(h->db, pagesize); + } + + ret = h->db->open(h->db, NULL, path, folder, DB_BTREE, createflags, 0600); + if (ret) { + h->db->close(h->db, 0); + h->env->close(h->env, 0); + mx_unlock_file(h->lockfile, h->fd, 0); + close(h->fd); + FREE(& h); + return NULL; + } + + return h; +} + +void +mutt_hcache_close(void *db) +{ + struct header_cache *h = db; + int ret; + + if (! h) { + return; + } + + h->db->close(h->db, 0); + h->env->close(h->env, 0); + mx_unlock_file(h->lockfile, h->fd, 0); + close(h->fd); + FREE(& h); +} + +void * +mutt_hcache_fetch(void *db, const char *filename, size_t (*keylen)(const char *fn)) +{ + DBT key; + DBT data; + struct header_cache *h = db; + + if (! h) { + return NULL; + } + + filename++; /* skip '/' */ + + mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); + mutt_hcache_dbt_empty_init(&data); + data.flags = DB_DBT_MALLOC; + + h->db->get(h->db, NULL, &key, &data, 0); + + if (! crc32_matches(data.data, h->crc)) { + FREE(&data.data); + return NULL; + } + + return data.data; +} + +int +mutt_hcache_store(void *db, const char *filename, HEADER *header, unsigned long long uid_validity, size_t (*keylen)(const char *fn)) +{ + DBT key; + DBT data; + int ret; + struct header_cache *h = db; + + if (! h) { + return -1; + } + + filename++; /* skip '/' */ + + mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); + + mutt_hcache_dbt_empty_init(&data); + data.flags = DB_DBT_USERMEM; + data.data = mutt_hcache_dump(db, header, (signed int *) &data.size, uid_validity); + data.ulen = data.size; + + ret = h->db->put(h->db, NULL, &key, &data, 0); + + FREE(& data.data); + + return ret; +} + +int +mutt_hcache_delete(void *db, const char *filename, size_t (*keylen)(const char *fn)) +{ + DBT key; + struct header_cache *h = db; + + if (! h) { + return -1; + } + + filename++; /* skip '/' */ + + mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); + return h->db->del(h->db, NULL, &key, 0); +} +#endif diff --git a/imap/imap.c b/imap/imap.c index ed70258a6..c3c4ddba2 100644 --- a/imap/imap.c +++ b/imap/imap.c @@ -603,6 +603,16 @@ int imap_open_mailbox (CONTEXT* ctx) if ((pc = imap_get_flags (&(idata->flags), pc)) == NULL) goto fail; } +#ifdef USE_HCACHE + /* save UIDVALIDITY for the header cache */ + else if (ascii_strncasecmp("OK [UIDVALIDITY", pc, 14) == 0) + { + dprint(2, (debugfile, "Getting mailbox UIDVALIDITY\n")); + pc += 3; + pc = imap_next_word(pc); + sscanf(pc, "%u", &(idata->uid_validity)); + } +#endif else { pc = imap_next_word (pc); diff --git a/imap/imap_private.h b/imap/imap_private.h index 9905f5b99..c29bca20b 100644 --- a/imap/imap_private.h +++ b/imap/imap_private.h @@ -179,6 +179,9 @@ typedef struct unsigned int newMailCount; IMAP_CACHE cache[IMAP_CACHE_LEN]; unsigned int noclose : 1; +#ifdef USE_HCACHE + unsigned long long uid_validity; +#endif /* all folder flags - system flags AND keywords */ LIST *flags; diff --git a/imap/message.c b/imap/message.c index 8e7a99c8a..4eae3b849 100644 --- a/imap/message.c +++ b/imap/message.c @@ -39,6 +39,12 @@ static int msg_has_flag (LIST* flag_list, const char* flag); static int msg_parse_fetch (IMAP_HEADER* h, char* s); static char* msg_parse_flags (IMAP_HEADER* h, char* s); +#if USE_HCACHE +static int msg_fetch_header_fetch (CONTEXT* ctx, IMAP_HEADER* h, char* buf, + FILE* fp); +static size_t imap_hcache_keylen (const char *fn); +#endif /* USE_HCACHE */ + /* imap_read_headers: * Changed to read many headers instead of just one. It will return the * msgno of the last message read. It will return a value other than @@ -57,8 +63,18 @@ int imap_read_headers (IMAP_DATA* idata, int msgbegin, int msgend) int fetchlast = 0; const char *want_headers = "DATE FROM SUBJECT TO CC MESSAGE-ID REFERENCES CONTENT-TYPE CONTENT-DESCRIPTION IN-REPLY-TO REPLY-TO LINES LIST-POST X-LABEL"; +#if USE_HCACHE + void *hc = NULL; + unsigned long long *uid_validity = NULL; + char uid_buf[64]; +#endif /* USE_HCACHE */ + ctx = idata->ctx; +#if USE_HCACHE + hc = mutt_hcache_open (HeaderCache, ctx->path); +#endif /* USE_HCACHE */ + if (mutt_bit_isset (idata->capabilities,IMAP4REV1)) { snprintf (hdrreq, sizeof (hdrreq), "BODY.PEEK[HEADER.FIELDS (%s%s%s)]", @@ -73,6 +89,9 @@ int imap_read_headers (IMAP_DATA* idata, int msgbegin, int msgend) { /* Unable to fetch headers for lower versions */ mutt_error _("Unable to fetch headers from this IMAP server version."); mutt_sleep (2); /* pause a moment to let the user see the error */ +#if USE_HCACHE + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ return -1; } @@ -83,6 +102,9 @@ int imap_read_headers (IMAP_DATA* idata, int msgbegin, int msgend) { mutt_error (_("Could not create temporary file %s"), tempfile); mutt_sleep (2); +#if USE_HCACHE + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ return -1; } unlink (tempfile); @@ -95,14 +117,96 @@ int imap_read_headers (IMAP_DATA* idata, int msgbegin, int msgend) idata->reopen &= ~IMAP_NEWMAIL_PENDING; idata->newMailCount = 0; +#if USE_HCACHE + snprintf (buf, sizeof (buf), + "FETCH %d:%d (UID FLAGS)", msgbegin + 1, msgend + 1); + fetchlast = msgend + 1; + + imap_cmd_start (idata, buf); + + for (msgno = msgbegin; msgno <= msgend ; msgno++) + { + if (ReadInc && (!msgno || ((msgno+1) % ReadInc == 0))) + mutt_message (_("Evaluating cache... [%d/%d]"), msgno + 1, + msgend + 1); + + rewind (fp); + memset (&h, 0, sizeof (h)); + h.data = safe_calloc (1, sizeof (IMAP_HEADER_DATA)); + do + { + mfhrc = 0; + + rc = imap_cmd_step (idata); + if (rc != IMAP_CMD_CONTINUE) + break; + + if ((mfhrc = msg_fetch_header_fetch (idata->ctx, &h, idata->cmd.buf, fp)) == -1) + continue; + else if (mfhrc < 0) + break; + + /* make sure we don't get remnants from older larger message headers */ + fputs ("\n\n", fp); + + sprintf(uid_buf, "/%u", h.data->uid); /* XXX --tg 21:41 04-07-11 */ + uid_validity = (unsigned long long *) mutt_hcache_fetch (hc, uid_buf, &imap_hcache_keylen); + + if (uid_validity != NULL + && *uid_validity == idata->uid_validity) { + ctx->hdrs[msgno] = mutt_hcache_restore((unsigned char *) uid_validity, 0); + ctx->hdrs[msgno]->index = h.sid - 1; + if (h.sid != ctx->msgcount + 1) + dprint (1, (debugfile, "imap_read_headers: msgcount and sequence ID are inconsistent!")); + /* messages which have not been expunged are ACTIVE (borrowed from mh + * folders) */ + ctx->hdrs[msgno]->active = 1; + ctx->hdrs[msgno]->read = h.read; + ctx->hdrs[msgno]->old = h.old; + ctx->hdrs[msgno]->deleted = h.deleted; + ctx->hdrs[msgno]->flagged = h.flagged; + ctx->hdrs[msgno]->replied = h.replied; + ctx->hdrs[msgno]->changed = h.changed; + /* ctx->hdrs[msgno]->received is restored from mutt_hcache_restore */ + ctx->hdrs[msgno]->data = (void *) (h.data); + + ctx->msgcount++; + } + rewind (fp); + + FREE(&uid_validity); + + } + while ((rc != IMAP_CMD_OK) && ((mfhrc == -1) || + ((msgno + 1) >= fetchlast))); + + if ((mfhrc < -1) || ((rc != IMAP_CMD_CONTINUE) && (rc != IMAP_CMD_OK))) + { + imap_free_header_data ((void**) &h.data); + fclose (fp); + mutt_hcache_close (hc); + return -1; + } + } + + fetchlast = msgbegin; +#endif /* USE_HCACHE */ + for (msgno = msgbegin; msgno <= msgend ; msgno++) { if (ReadInc && (!msgno || ((msgno+1) % ReadInc == 0))) mutt_message (_("Fetching message headers... [%d/%d]"), msgno + 1, msgend + 1); + if (ctx->hdrs[msgno]) + continue; + if (msgno + 1 > fetchlast) { + fetchlast = msgno + 1; + while((fetchlast <= msgend) && (! ctx->hdrs[fetchlast])) + fetchlast++; + /* * Make one request for everything. This makes fetching headers an * order of magnitude faster if you have a large mailbox. @@ -112,11 +216,9 @@ int imap_read_headers (IMAP_DATA* idata, int msgbegin, int msgend) */ snprintf (buf, sizeof (buf), "FETCH %d:%d (UID FLAGS INTERNALDATE RFC822.SIZE %s)", msgno + 1, - msgend + 1, hdrreq); + fetchlast, hdrreq); imap_cmd_start (idata, buf); - - fetchlast = msgend + 1; } /* freshen fp, h */ @@ -170,6 +272,11 @@ int imap_read_headers (IMAP_DATA* idata, int msgbegin, int msgend) /* content built as a side-effect of mutt_read_rfc822_header */ ctx->hdrs[msgno]->content->length = h.content_length; +#if USE_HCACHE + sprintf(uid_buf, "/%u", h.data->uid); + mutt_hcache_store(hc, uid_buf, ctx->hdrs[msgno], idata->uid_validity, &imap_hcache_keylen); +#endif /* USE_HCACHE */ + ctx->msgcount++; } while ((rc != IMAP_CMD_OK) && ((mfhrc == -1) || @@ -179,7 +286,9 @@ int imap_read_headers (IMAP_DATA* idata, int msgbegin, int msgend) { imap_free_header_data ((void**) &h.data); fclose (fp); - +#if USE_HCACHE + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ return -1; } @@ -194,6 +303,10 @@ int imap_read_headers (IMAP_DATA* idata, int msgbegin, int msgend) } } +#if USE_HCACHE + mutt_hcache_close (hc); +#endif /* USE_HCACHE */ + fclose(fp); if (ctx->msgcount > oldmsgcount) @@ -724,6 +837,7 @@ char* imap_set_flags (IMAP_DATA* idata, HEADER* h, char* s) return s; } + /* msg_fetch_header: import IMAP FETCH response into an IMAP_HEADER. * Expects string beginning with * n FETCH. * Returns: @@ -784,6 +898,56 @@ static int msg_fetch_header (CONTEXT* ctx, IMAP_HEADER* h, char* buf, FILE* fp) return rc; } +#if USE_HCACHE +static size_t imap_hcache_keylen (const char *fn) +{ + return mutt_strlen(fn); +} + +/* msg_fetch_header: import IMAP FETCH response into an IMAP_HEADER. + * Expects string beginning with * n FETCH. + * Returns: + * 0 on success + * -1 if the string is not a fetch response + * -2 if the string is a corrupt fetch response */ +static int msg_fetch_header_fetch (CONTEXT* ctx, IMAP_HEADER* h, char* buf, FILE* fp) +{ + IMAP_DATA* idata; + long bytes; + int rc = -1; /* default now is that string isn't FETCH response*/ + + idata = (IMAP_DATA*) ctx->data; + + if (buf[0] != '*') + return rc; + + /* skip to message number */ + buf = imap_next_word (buf); + h->sid = atoi (buf); + + /* find FETCH tag */ + buf = imap_next_word (buf); + if (ascii_strncasecmp ("FETCH", buf, 5)) + return rc; + + rc = -2; /* we've got a FETCH response, for better or worse */ + if (!(buf = strchr (buf, '('))) + return rc; + buf++; + + if (msg_parse_fetch (h, buf) < 0) { + return -2; + } + + if (!(buf = strchr (buf, ')'))) + return rc; + buf++; + + return 0; +} +#endif /* USE_HCACHE */ + + /* msg_has_flag: do a caseless comparison of the flag against a flag list, * return 1 if found or flag list has '\*', 0 otherwise */ static int msg_has_flag (LIST* flag_list, const char* flag) diff --git a/init.h b/init.h index 7e08426de..6cfabb247 100644 --- a/init.h +++ b/init.h @@ -1022,6 +1022,25 @@ struct option_t MuttVars[] = { ** \fBDON'T CHANGE THIS SETTING UNLESS YOU ARE REALLY SURE WHAT YOU ARE ** DOING!\fP */ +#if USE_HCACHE + { "header_cache", DT_PATH, R_NONE, UL &HeaderCache, 0 }, + /* + ** .pp + ** The header_cache variable points to the header cache database. If + ** header_cache points to a directory there will be created one header cache + ** database per folder within this directory. If it doesn't point to a directory a + ** global header cache for all folders is used. Per default it is unset and so + ** no header caching will be used. + */ + { "header_cache_pagesize", DT_STR, R_NONE, UL &HeaderCachePageSize, UL "16384" }, + /* + ** .pp + ** Change the header cache database page size. Too large or to small values + ** can waste space, memory, or CPU time. The default should be more or + ** less the best you can get. For details google after mutt header + ** cache (first hit). + */ +#endif /* USE_HCACHE */ { "maildir_trash", DT_BOOL, R_NONE, OPTMAILDIRTRASH, 0 }, /* ** .pp diff --git a/main.c b/main.c index 5341f2ac7..45d81c5a3 100644 --- a/main.c +++ b/main.c @@ -412,6 +412,12 @@ static void show_version (void) "-HAVE_GETADDRINFO " #endif +#if USE_HCACHE + "+USE_HCACHE " +#else + "-USE_HCACHE " +#endif + ); #ifdef ISPELL diff --git a/protos.h b/protos.h index a481e4f4b..366fe6048 100644 --- a/protos.h +++ b/protos.h @@ -102,6 +102,16 @@ LIST *mutt_make_references(ENVELOPE *e); ENVELOPE *mutt_read_rfc822_header (FILE *, HEADER *, short, short); HEADER *mutt_dup_header (HEADER *); +#if USE_HCACHE +void *mutt_hcache_open(const char *path, const char *folder); +void mutt_hcache_close(void *db); +HEADER *mutt_hcache_restore(const unsigned char *d, HEADER **oh); +void *mutt_hcache_fetch(void *db, const char *filename, size_t (*keylen)(const char *fn)); +int mutt_hcache_store(void *db, const char *filename, HEADER *h, unsigned long long uid_validity, size_t (*keylen)(const char *fn)); +int mutt_hcache_delete(void *db, const char *filename, size_t (*keylen)(const char *fn)); +#endif /* USE_HCACHE */ + + ATTACHPTR **mutt_gen_attach_list (BODY *, int, ATTACHPTR **, short *, short *, int, int); time_t mutt_decrease_mtime (const char *, struct stat *);