From 5e03a680b4f264e010cc56a7ca177eb0a9c80040 Mon Sep 17 00:00:00 2001 From: Rocco Rutte Date: Tue, 4 Jul 2006 17:11:03 +0000 Subject: [PATCH] pdmef.cache.24: Generalise IMAP body caching and add POP support. $imap_cachedir is now a synonym for $message_cachedir, and should be dropped soon since it hasn't been in an official release. --- Makefile.am | 2 +- configure.in | 4 + doc/manual.xml.head | 89 ++++++++++++++++++++++ globals.h | 4 +- imap/imap.c | 3 + imap/imap_private.h | 4 +- imap/message.c | 86 ++++++++------------- imap/util.c | 1 + init.h | 15 ++-- mh.c | 1 + pop.c | 177 ++++++++++++++++++++++++++++++++++++++++---- pop.h | 2 + 12 files changed, 311 insertions(+), 77 deletions(-) diff --git a/Makefile.am b/Makefile.am index aee88f0a..9dcbb1e3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -64,7 +64,7 @@ EXTRA_mutt_SOURCES = account.c md5c.c mutt_sasl.c mutt_socket.c mutt_ssl.c \ pgplib.c sha1.c pgpmicalg.c gnupgparse.c resize.c dotlock.c remailer.c \ browser.h mbyte.h remailer.h url.h \ crypt-mod-pgp-classic.c crypt-mod-smime-classic.c \ - pgppacket.c mutt_idna.h hcache.h hcache.c mutt_ssl_gnutls.c \ + pgppacket.c mutt_idna.h hcache.h hcache.c bcache.c bcache.h mutt_ssl_gnutls.c \ crypt-gpgme.c crypt-mod-pgp-gpgme.c crypt-mod-smime-gpgme.c EXTRA_DIST = COPYRIGHT GPL OPS OPS.PGP OPS.CRYPT OPS.SMIME TODO UPDATING \ diff --git a/configure.in b/configure.in index e994b8cf..d6092ea7 100644 --- a/configure.in +++ b/configure.in @@ -517,6 +517,10 @@ AC_ARG_ENABLE(imap, AC_HELP_STRING([--enable-imap], [Enable IMAP support]), ]) AM_CONDITIONAL(BUILD_IMAP, test x$need_imap = xyes) +if test x"$need_imap" = xyes -o x"$need_pop" = xyes ; then + MUTT_LIB_OBJECTS="$MUTT_LIB_OBJECTS bcache.o" +fi + dnl -- end socket dependencies -- if test "$need_socket" = "yes" diff --git a/doc/manual.xml.head b/doc/manual.xml.head index 2e3b0bcd..206960c7 100644 --- a/doc/manual.xml.head +++ b/doc/manual.xml.head @@ -4476,6 +4476,95 @@ macro pager \cb |urlview\n + +Local caching (OPTIONAL) + + +Mutt contains two types of local caching: (1) +the so-called ``header caching'' and (2) the +so-called ``body caching'' which are both described in this section. + + + +These are optional which means they're not enabled by default. +Details on how to enable either of these techniques are given in the +following subsections. + + + +Header caching + + +Mutt provides optional support for caching message headers for the +following types of folders: IMAP, POP, Maildir and MH. Header caching +greatly improves speed because for remote folders, headers +usually only need to be downloaded once. For Maildir and MH, reading the +headers from a single file is much faster than looking at possibly +thousands of single files (since Maildir uses one file per message.) + + + +Header caching can be enabled via the configure script and the +--enable--hcache option. It's not turned on +by default because external database libraries are required: one +of qdbm, gdbm or bdb must be present. + + + +If enabled, $header_cache can be +used to either point to a file or a directory. If set to point to +a file, one database file for all folders will be used (which may +result in lower performance), but one file per folder if it points +to a directory. + + + +For the one-file-per-folder case, database files will be named by +MD5 sums and can be removed if a system is short on space. However, +there currently is no easy to find out which database file is used +for which folder. + + + + + +Body caching + + +In addition to caching message headers only, mutt can also cache +whole message bodies. This results in faster display of messages +for POP and IMAP folders because messages usually have to be +downloaded only once. + + + +If the configure script is called with --enable-pop +and/or --enable-imap, body caching will be +built in as it does not require additional software packages such +as database libraries. + + + +For configuration, the variable $message_cachedir must point to a +directory. There, mutt will create a hierarchy of subdirectories +named like: proto:user@hostname where +proto is either ``pop'' or ``imap.'' Within +there for each folder, mutt stores messages in single files (just +like Maildir) so that with manual symlink creation these cache +directories can be examined with mutt as read-only Maildir folders. + + + +All files can be removed as needed if the consumed disk space +becomes an issue as mutt will silently fetch missing items again. + + + + + + diff --git a/globals.h b/globals.h index d7fbc383..6abd75e0 100644 --- a/globals.h +++ b/globals.h @@ -55,7 +55,6 @@ WHERE char *Homedir; WHERE char *Hostname; #ifdef USE_IMAP WHERE char *ImapAuthenticators INITVAL (NULL); -WHERE char *ImapCachedir; WHERE char *ImapDelimChars INITVAL (NULL); WHERE char *ImapHeaders; WHERE char *ImapHomeNamespace INITVAL (NULL); @@ -68,6 +67,9 @@ WHERE char *Ispell; WHERE char *Locale; WHERE char *MailcapPath; WHERE char *Maildir; +#if defined(USE_IMAP) || defined(USE_POP) +WHERE char *MessageCachedir; +#endif #if USE_HCACHE WHERE char *HeaderCache; #if HAVE_GDBM || HAVE_DB4 diff --git a/imap/imap.c b/imap/imap.c index 99b0c470..4a5d6009 100644 --- a/imap/imap.c +++ b/imap/imap.c @@ -804,6 +804,7 @@ void imap_logout (IMAP_DATA* idata) ; FREE(& idata->buf); + mutt_bcache_close (&idata->bcache); FREE(& idata); } @@ -1307,6 +1308,8 @@ void imap_close_mailbox (CONTEXT* ctx) FREE (&idata->cache[i].path); } } + + mutt_bcache_close (&idata->bcache); } /* use the NOOP or IDLE command to poll for new mail diff --git a/imap/imap_private.h b/imap/imap_private.h index 82d9bfc5..ac97deaf 100644 --- a/imap/imap_private.h +++ b/imap/imap_private.h @@ -23,6 +23,7 @@ #include "imap.h" #include "mutt_curses.h" #include "mutt_socket.h" +#include "bcache.h" /* -- symbols -- */ #define IMAP_PORT 143 @@ -222,7 +223,8 @@ typedef struct IMAP_CACHE cache[IMAP_CACHE_LEN]; unsigned int uid_validity; unsigned int uidnext; - + body_cache_t *bcache; + /* all folder flags - system flags AND keywords */ LIST *flags; } IMAP_DATA; diff --git a/imap/message.c b/imap/message.c index 562140d7..75ffa529 100644 --- a/imap/message.c +++ b/imap/message.c @@ -41,6 +41,8 @@ #include "hcache.h" #endif +#include "bcache.h" + static FILE* msg_cache_get (IMAP_DATA* idata, HEADER* h); static FILE* msg_cache_put (IMAP_DATA* idata, HEADER* h); @@ -839,33 +841,19 @@ int imap_copy_messages (CONTEXT* ctx, HEADER* h, char* dest, int delete) return -1; } -/* create file system path for idata/h */ -static int msg_cache_path (IMAP_DATA* idata, HEADER* h, char* buf, size_t len) +static body_cache_t *msg_cache_open (IMAP_DATA *idata) { - ACCOUNT* account; - char* s, *p; - int slen; - - if (!ImapCachedir) - return -1; + char *s; + char *p = idata->mailbox; + char mailbox[_POSIX_PATH_MAX]; + size_t mlen = sizeof (mailbox); - account = &idata->conn->account; + if (idata->bcache) + return idata->bcache; - snprintf (buf, len, "%s/", ImapCachedir); - slen = mutt_strlen (buf); - if (account->flags & M_ACCT_USER) - snprintf (buf + slen, len - slen, "%s@", account->user); - safe_strcat (buf, len, account->host); - if (account->flags & M_ACCT_PORT) - { - slen = mutt_strlen (buf); - snprintf (buf + slen, len - slen, ":%hu", account->port); - } - safe_strcat (buf, len, "/"); + mailbox[0] = '\0'; - slen = len - mutt_strlen (buf) - 2; - p = idata->mailbox; - for (s = buf + mutt_strlen (buf); *p && slen; slen--) + for (s = mailbox; p && *p && mlen; mlen--) { if (*p == idata->delim) { @@ -873,9 +861,9 @@ static int msg_cache_path (IMAP_DATA* idata, HEADER* h, char* buf, size_t len) /* simple way to avoid collisions with UIDs */ if (*(p + 1) >= '0' && *(p + 1) <= '9') { - slen--; - if (slen) - *++s = '_'; + mlen--; + if (mlen) + *++s = '_'; } } else @@ -885,55 +873,43 @@ static int msg_cache_path (IMAP_DATA* idata, HEADER* h, char* buf, size_t len) } *s = '\0'; - slen = mutt_strlen (buf); - snprintf (buf + slen, len - slen, "/%u-%u", idata->uid_validity, - HEADER_DATA(h)->uid); - - return 0; + return mutt_bcache_open (&idata->conn->account, mailbox); } static FILE* msg_cache_get (IMAP_DATA* idata, HEADER* h) { - char path[_POSIX_PATH_MAX]; + char id[_POSIX_PATH_MAX]; - if (msg_cache_path (idata, h, path, sizeof (path)) < 0) + if (!idata || !h) return NULL; - return fopen (path, "r"); + idata->bcache = msg_cache_open (idata); + snprintf (id, sizeof (id), "%u-%u", idata->uid_validity, HEADER_DATA(h)->uid); + return mutt_bcache_get (idata->bcache, id); } static FILE* msg_cache_put (IMAP_DATA* idata, HEADER* h) { - char path[_POSIX_PATH_MAX]; - FILE* fp; - char* s; - struct stat sb; + char id[_POSIX_PATH_MAX]; - if (msg_cache_path (idata, h, path, sizeof (path)) < 0) + if (!idata || !h) return NULL; - s = strchr (path + 1, '/'); - while (!(fp = safe_fopen (path, "w+")) && errno == ENOENT && s) - { - /* create missing path components */ - *s = '\0'; - if (stat (path, &sb) < 0 && (errno != ENOENT || mkdir (path, 0777) < 0)) - return NULL; - *s = '/'; - s = strchr (s + 1, '/'); - } - - return fp; + idata->bcache = msg_cache_open (idata); + snprintf (id, sizeof (id), "%u-%u", idata->uid_validity, HEADER_DATA(h)->uid); + return mutt_bcache_put (idata->bcache, id); } int imap_cache_del (IMAP_DATA* idata, HEADER* h) { - char path[_POSIX_PATH_MAX]; - - if (msg_cache_path (idata, h, path, sizeof (path)) < 0) + char id[_POSIX_PATH_MAX]; + + if (!idata || !h) return -1; - return unlink (path); + idata->bcache = msg_cache_open (idata); + snprintf (id, sizeof (id), "%u-%u", idata->uid_validity, HEADER_DATA(h)->uid); + return mutt_bcache_del (idata->bcache, id); } /* imap_add_keywords: concatenate custom IMAP tags to list, if they diff --git a/imap/util.c b/imap/util.c index 5bfc8511..c8a18b2e 100644 --- a/imap/util.c +++ b/imap/util.c @@ -278,6 +278,7 @@ void imap_free_idata (IMAP_DATA** idata) imap_mboxcache_free (*idata); mutt_buffer_free(&(*idata)->cmdbuf); FREE (&(*idata)->buf); + mutt_bcache_close (&(*idata)->bcache); FREE (idata); /* __FREE_CHECKED__ */ } diff --git a/init.h b/init.h index d1a9bc7e..12a521d5 100644 --- a/init.h +++ b/init.h @@ -822,12 +822,8 @@ struct option_t MuttVars[] = { ** the previous methods are unavailable. If a method is available but ** authentication fails, mutt will not connect to the IMAP server. */ - { "imap_cachedir", DT_PATH, R_NONE, UL &ImapCachedir, 0 }, + { "imap_cachedir", DT_SYN, R_NONE, UL "message_cachedir", 0 }, /* - ** .pp - ** Set this to a directory and mutt will cache copies of messages from - ** your IMAP servers here. You are free to remove entries at any time - ** if space becomes an issue. */ { "imap_check_subscribed", DT_BOOL, R_NONE, OPTIMAPCHECKSUBSCRIBED, 0 }, /* @@ -1281,6 +1277,15 @@ struct option_t MuttVars[] = { ** from your spool mailbox to your ``$$mbox'' mailbox, or as a result of ** a ``$mbox-hook'' command. */ +#if defined(USE_IMAP) || defined(USE_POP) + { "message_cachedir", DT_PATH, R_NONE, UL &MessageCachedir, 0 }, + /* + ** .pp + ** Set this to a directory and mutt will cache copies of messages from + ** your IMAP and POP servers here. You are free to remove entries at any time + ** if space becomes an issue. + */ +#endif { "message_format", DT_STR, R_NONE, UL &MsgFmt, UL "%s" }, /* ** .pp diff --git a/mh.c b/mh.c index c05bdcd3..13fdb7f1 100644 --- a/mh.c +++ b/mh.c @@ -32,6 +32,7 @@ #include "copy.h" #include "buffy.h" #include "sort.h" +#include "account.h" #if USE_HCACHE #include "hcache.h" #endif diff --git a/pop.c b/pop.c index 20cccb4a..bdd5360b 100644 --- a/pop.c +++ b/pop.c @@ -1,5 +1,6 @@ /* * Copyright (C) 2000-2002 Vsevolod Volkov + * Copyright (C) 2006 Rocco Rutte * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -25,6 +26,10 @@ #include "mx.h" #include "pop.h" #include "mutt_crypt.h" +#include "bcache.h" +#if USE_HCACHE +#include "hcache.h" +#endif #include #include @@ -157,6 +162,28 @@ static int fetch_uidl (char *line, void *data) return 0; } +static int msg_cache_check (const char *id, body_cache_t *bcache, void *data) +{ + CONTEXT *ctx; + POP_DATA *pop_data; + int i; + + if (!(ctx = (CONTEXT *)data)) + return -1; + if (!(pop_data = (POP_DATA *)ctx->data)) + return -1; + + for (i = 0; i < ctx->msgcount; i++) + /* if the id we get is known for a header: done (i.e. keep in cache) */ + if (ctx->hdrs[i]->data && mutt_strcmp (ctx->hdrs[i]->data, id) == 0) + return 0; + + /* message not found in context -> remove it from cache + * return the result of bcache, so we stop upon its first error + */ + return mutt_bcache_del (bcache, id); +} + /* * Read headers * returns: @@ -168,8 +195,16 @@ static int fetch_uidl (char *line, void *data) static int pop_fetch_headers (CONTEXT *ctx) { int i, ret, old_count, new_count; + unsigned short hcached = 0, bcached; POP_DATA *pop_data = (POP_DATA *)ctx->data; +#ifdef USE_HCACHE + header_cache_t *hc = NULL; + void *data; + + hc = mutt_hcache_open (HeaderCache, ctx->path); +#endif + time (&pop_data->check_time); pop_data->clear_cache = 0; @@ -211,9 +246,70 @@ static int pop_fetch_headers (CONTEXT *ctx) mutt_message (_("Fetching message headers... [%d/%d]"), i + 1 - old_count, new_count - old_count); - ret = pop_read_header (pop_data, ctx->hdrs[i]); - if (ret < 0) +#if USE_HCACHE + if ((data = mutt_hcache_fetch (hc, ctx->hdrs[i]->data, strlen))) + { + char *uidl = safe_strdup (ctx->hdrs[i]->data); + int refno = ctx->hdrs[i]->refno; + int index = ctx->hdrs[i]->index; + /* + * - POP dynamically numbers headers and relies on h->refno + * to map messages; so restore header and overwrite restored + * refno with current refno, same for index + * - h->data needs to a separate pointer as it's driver-specific + * data freed separately elsewhere + * (the old h->data should point inside a malloc'd block from + * hcache so there shouldn't be a memleak here) + */ + HEADER *h = mutt_hcache_restore ((unsigned char *) data, NULL); + mutt_free_header (&ctx->hdrs[i]); + ctx->hdrs[i] = h; + ctx->hdrs[i]->refno = refno; + ctx->hdrs[i]->index = index; + ctx->hdrs[i]->data = uidl; + ret = 0; + hcached = 1; + } + else +#endif + if ((ret = pop_read_header (pop_data, ctx->hdrs[i])) < 0) break; +#if USE_HCACHE + else + { + mutt_hcache_store (hc, ctx->hdrs[i]->data, ctx->hdrs[i], 0, strlen); + } + + FREE(&data); +#endif + + /* + * faked support for flags works like this: + * - if 'hcached' is 1, we have the message in our hcache: + * - if we also have a body: read + * - if we don't have a body: old + * (if $mark_old is set which is maybe wrong as + * $mark_old should be considered for syncing the + * folder and not when opening it XXX) + * - if 'hcached' is 0, we don't have the message in our hcache: + * - if we also have a body: read + * - if we don't have a body: new + */ + bcached = mutt_bcache_exists (pop_data->bcache, ctx->hdrs[i]->data) == 0; + ctx->hdrs[i]->old = 0; + ctx->hdrs[i]->read = 0; + if (hcached) + { + if (bcached) + ctx->hdrs[i]->read = 1; + else if (option (OPTMARKOLD)) + ctx->hdrs[i]->old = 1; + } + else + { + if (bcached) + ctx->hdrs[i]->read = 1; + } ctx->msgcount++; } @@ -222,6 +318,10 @@ static int pop_fetch_headers (CONTEXT *ctx) mx_update_context (ctx, i - old_count); } +#if USE_HCACHE + mutt_hcache_close (hc); +#endif + if (ret < 0) { for (i = ctx->msgcount; i < new_count; i++) @@ -229,6 +329,12 @@ static int pop_fetch_headers (CONTEXT *ctx) return ret; } + /* after putting the result into our structures, + * clean up cache, i.e. wipe messages deleted outside + * the availability of our cache + */ + mutt_bcache_list (pop_data->bcache, msg_cache_check, (void*)ctx); + mutt_clear_error (); return (new_count - old_count); } @@ -268,6 +374,7 @@ int pop_open_mailbox (CONTEXT *ctx) return -1; conn->data = pop_data; + pop_data->bcache = mutt_bcache_open (&acct, NULL); FOREVER { @@ -332,6 +439,8 @@ void pop_close_mailbox (CONTEXT *ctx) if (!pop_data->conn->data) mutt_socket_free (pop_data->conn); + mutt_bcache_close (&pop_data->bcache); + return; } @@ -346,8 +455,16 @@ int pop_fetch_message (MESSAGE* msg, CONTEXT* ctx, int msgno) POP_DATA *pop_data = (POP_DATA *)ctx->data; POP_CACHE *cache; HEADER *h = ctx->hdrs[msgno]; + unsigned short bcache = 1; + + /* see if we already have the message in body cache */ + if ((msg->fp = mutt_bcache_get (pop_data->bcache, h->data))) + return 0; - /* see if we already have the message in our cache */ + /* + * see if we already have the message in our cache in + * case $message_cachedir is unset + */ cache = &pop_data->cache[h->index % POP_CACHE_LEN]; if (cache->path) @@ -358,7 +475,7 @@ int pop_fetch_message (MESSAGE* msg, CONTEXT* ctx, int msgno) msg->fp = fopen (cache->path, "r"); if (msg->fp) return 0; - + mutt_perror (cache->path); mutt_sleep (2); return -1; @@ -388,13 +505,18 @@ int pop_fetch_message (MESSAGE* msg, CONTEXT* ctx, int msgno) progressbar.msg = _("Fetching message..."); mutt_progress_bar (&progressbar, 0); - mutt_mktemp (path); - msg->fp = safe_fopen (path, "w+"); - if (!msg->fp) + /* see if we can put in body cache; use our cache as fallback */ + if (!(msg->fp = mutt_bcache_put (pop_data->bcache, h->data))) { - mutt_perror (path); - mutt_sleep (2); - return -1; + /* no */ + bcache = 0; + mutt_mktemp (path); + if (!(msg->fp = safe_fopen (path, "w+"))) + { + mutt_perror (path); + mutt_sleep (2); + return -1; + } } snprintf (buf, sizeof (buf), "RETR %d\r\n", h->refno); @@ -404,7 +526,14 @@ int pop_fetch_message (MESSAGE* msg, CONTEXT* ctx, int msgno) break; safe_fclose (&msg->fp); - unlink (path); + + /* if RETR failed (e.g. connection closed), be sure to remove either + * the file in bcache or from POP's own cache since the next iteration + * of the loop will re-attempt to put() the message */ + if (bcache) + mutt_bcache_del (pop_data->bcache, h->data); + else + unlink (path); if (ret == -2) { @@ -424,8 +553,11 @@ int pop_fetch_message (MESSAGE* msg, CONTEXT* ctx, int msgno) /* Update the header information. Previously, we only downloaded a * portion of the headers, those required for the main display. */ - cache->index = h->index; - cache->path = safe_strdup (path); + if (!bcache) + { + cache->index = h->index; + cache->path = safe_strdup (path); + } rewind (msg->fp); uidl = h->data; mutt_free_envelope (&h->env); @@ -457,6 +589,9 @@ int pop_sync_mailbox (CONTEXT *ctx, int *index_hint) int i, ret; char buf[LONG_STRING]; POP_DATA *pop_data = (POP_DATA *)ctx->data; +#ifdef USE_HCACHE + header_cache_t *hc = NULL; +#endif pop_data->check_time = 0; @@ -467,15 +602,29 @@ int pop_sync_mailbox (CONTEXT *ctx, int *index_hint) mutt_message (_("Marking %d messages deleted..."), ctx->deleted); +#if USE_HCACHE + hc = mutt_hcache_open (HeaderCache, ctx->path); +#endif + for (i = 0, ret = 0; ret == 0 && i < ctx->msgcount; i++) { if (ctx->hdrs[i]->deleted) { snprintf (buf, sizeof (buf), "DELE %d\r\n", ctx->hdrs[i]->refno); - ret = pop_query (pop_data, buf, sizeof (buf)); + if ((ret = pop_query (pop_data, buf, sizeof (buf))) == 0) + { + mutt_bcache_del (pop_data->bcache, ctx->hdrs[i]->data); +#if USE_HCACHE + mutt_hcache_delete (hc, ctx->hdrs[i]->data, strlen); +#endif + } } } +#if USE_HCACHE + mutt_hcache_close (hc); +#endif + if (ret == 0) { strfcpy (buf, "QUIT\r\n", sizeof (buf)); diff --git a/pop.h b/pop.h index ab48a07a..13bd1212 100644 --- a/pop.h +++ b/pop.h @@ -22,6 +22,7 @@ #include "mailbox.h" #include "mutt_socket.h" #include "mutt_curses.h" +#include "bcache.h" #define POP_PORT 110 #define POP_SSL_PORT 995 @@ -74,6 +75,7 @@ typedef struct time_t login_delay; /* minimal login delay capability */ char *auth_list; /* list of auth mechanisms */ char *timestamp; + body_cache_t *bcache; /* body cache */ char err_msg[POP_CMD_RESPONSE]; POP_CACHE cache[POP_CACHE_LEN]; } POP_DATA; -- 2.50.1