From 2165222122f40cdb93f2cd4227fa4ec1013e0c1a Mon Sep 17 00:00:00 2001 From: Pietro Cerutti Date: Thu, 21 Mar 2019 12:30:36 +0000 Subject: [PATCH] Make NotMuch hcache-aware Closes: #1600 --- globals.h | 4 +- imap/imap.c | 2 - imap/imap_private.h | 9 ---- imap/message.c | 2 - imap/util.c | 8 ++- maildir/lib.h | 6 --- maildir/maildir.c | 2 - maildir/shared.c | 31 ++--------- mx.h | 2 - nntp/newsrc.c | 7 +-- nntp/nntp.c | 16 +----- nntp/nntp_private.h | 5 -- notmuch/mutt_notmuch.c | 119 +++++++++++++++++++++++++++++++---------- pop/pop.c | 4 -- 14 files changed, 103 insertions(+), 114 deletions(-) diff --git a/globals.h b/globals.h index cb1030621..763cb0540 100644 --- a/globals.h +++ b/globals.h @@ -220,8 +220,8 @@ WHERE bool C_ForwardQuote; ///< Config: Automatically quote a #ifdef USE_HCACHE #if defined(HAVE_QDBM) || defined(HAVE_TC) || defined(HAVE_KC) WHERE bool C_HeaderCacheCompress; ///< Config: (hcache) Enable database compression (qdbm,tokyocabinet,kyotocabinet) -#endif /* HAVE_QDBM */ -#endif +#endif /* HAVE_QDBM | HAVE_TC | HAVE_KC */ +#endif /* USE_HCACHE */ WHERE bool C_Header; ///< Config: Include the message headers in the reply email (Weed applies) WHERE bool C_Help; ///< Config: Display a help line with common key bindings #ifdef USE_IMAP diff --git a/imap/imap.c b/imap/imap.c index e05a33b0c..c0e88cb06 100644 --- a/imap/imap.c +++ b/imap/imap.c @@ -59,9 +59,7 @@ #include "pattern.h" #include "progress.h" #include "sort.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif #ifdef ENABLE_NLS #include #endif diff --git a/imap/imap_private.h b/imap/imap_private.h index e4526e6db..a3ec947e1 100644 --- a/imap/imap_private.h +++ b/imap/imap_private.h @@ -33,9 +33,7 @@ #include "mutt/mutt.h" #include "config/lib.h" #include "conn/conn.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif struct Email; struct Mailbox; @@ -239,10 +237,7 @@ struct ImapMboxData unsigned int max_msn; /**< the largest MSN fetched so far */ struct BodyCache *bcache; -#ifdef USE_HCACHE header_cache_t *hcache; -#endif - }; /** @@ -345,8 +340,4 @@ void imap_utf_decode(bool unicode, char **s); void imap_allow_reopen(struct Mailbox *m); void imap_disallow_reopen(struct Mailbox *m); -#ifdef USE_HCACHE -#define imap_hcache_keylen mutt_str_strlen -#endif /* USE_HCACHE */ - #endif /* MUTT_IMAP_IMAP_PRIVATE_H */ diff --git a/imap/message.c b/imap/message.c index 33f6a3c21..f5c490855 100644 --- a/imap/message.c +++ b/imap/message.c @@ -55,9 +55,7 @@ #include "mx.h" #include "progress.h" #include "protos.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif #ifdef ENABLE_NLS #include #endif diff --git a/imap/util.c b/imap/util.c index 76e250d10..8d87a3cc3 100644 --- a/imap/util.c +++ b/imap/util.c @@ -56,9 +56,7 @@ #include "message.h" #include "mutt_account.h" #include "options.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif /* These Config Variables are only used in imap/util.c */ char *C_ImapDelimChars; ///< Config: (imap) Characters that denote separators in IMAP folders @@ -467,7 +465,7 @@ struct Email *imap_hcache_get(struct ImapMboxData *mdata, unsigned int uid) return NULL; sprintf(key, "/%u", uid); - uv = mutt_hcache_fetch(mdata->hcache, key, imap_hcache_keylen(key)); + uv = mutt_hcache_fetch(mdata->hcache, key, mutt_str_strlen(key)); if (uv) { if (*(unsigned int *) uv == mdata->uid_validity) @@ -495,7 +493,7 @@ int imap_hcache_put(struct ImapMboxData *mdata, struct Email *e) return -1; sprintf(key, "/%u", imap_edata_get(e)->uid); - return mutt_hcache_store(mdata->hcache, key, imap_hcache_keylen(key), e, mdata->uid_validity); + return mutt_hcache_store(mdata->hcache, key, mutt_str_strlen(key), e, mdata->uid_validity); } /** @@ -513,7 +511,7 @@ int imap_hcache_del(struct ImapMboxData *mdata, unsigned int uid) return -1; sprintf(key, "/%u", uid); - return mutt_hcache_delete(mdata->hcache, key, imap_hcache_keylen(key)); + return mutt_hcache_delete(mdata->hcache, key, mutt_str_strlen(key)); } /** diff --git a/maildir/lib.h b/maildir/lib.h index 0fb17232d..3b4291077 100644 --- a/maildir/lib.h +++ b/maildir/lib.h @@ -40,9 +40,7 @@ #include #include "config/lib.h" #include "mx.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif struct Mailbox; struct Email; @@ -67,10 +65,6 @@ struct Email *maildir_parse_message (enum MailboxType magic, const char *fnam struct Email *maildir_parse_stream (enum MailboxType magic, FILE *fp, const char *fname, bool is_old, struct Email *e); bool maildir_update_flags (struct Mailbox *m, struct Email *o, struct Email *n); int mh_check_empty (const char *path); -#ifdef USE_HCACHE int mh_sync_mailbox_message (struct Mailbox *m, int msgno, header_cache_t *hc); -#else -int mh_sync_mailbox_message (struct Mailbox *m, int msgno); -#endif #endif /* MUTT_MAILDIR_LIB_H */ diff --git a/maildir/maildir.c b/maildir/maildir.c index 86c1e23e3..e6cf8ddad 100644 --- a/maildir/maildir.c +++ b/maildir/maildir.c @@ -52,9 +52,7 @@ #include "monitor.h" #include "muttlib.h" #include "mx.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif // Flags for maildir_mbox_check() #define MMC_NO_DIRS 0 ///< No directories changed diff --git a/maildir/shared.c b/maildir/shared.c index 891fee667..6c1d3e0ca 100644 --- a/maildir/shared.c +++ b/maildir/shared.c @@ -64,9 +64,6 @@ #ifdef USE_NOTMUCH #include "notmuch/mutt_notmuch.h" #endif -#ifdef USE_HCACHE -#include "hcache/hcache.h" -#endif struct Account; @@ -481,7 +478,6 @@ int maildir_move_to_mailbox(struct Mailbox *m, struct Maildir **md) return num; } -#ifdef USE_HCACHE /** * maildir_hcache_keylen - Calculate the length of the Maildir path * @param fn File name @@ -494,7 +490,6 @@ size_t maildir_hcache_keylen(const char *fn) const char *p = strrchr(fn, ':'); return p ? (size_t)(p - fn) : mutt_str_strlen(fn); } -#endif /** * md_cmp_inode - Compare two Maildirs by inode number @@ -710,12 +705,6 @@ void maildir_delayed_parsing(struct Mailbox *m, struct Maildir **md, struct Prog char fn[PATH_MAX]; int count; bool sort = false; -#ifdef USE_HCACHE - const char *key = NULL; - size_t keylen; - struct stat lastchanged; - int ret; -#endif #ifdef USE_HCACHE header_cache_t *hc = mutt_hcache_open(C_HeaderCache, m->path, NULL); @@ -748,16 +737,15 @@ void maildir_delayed_parsing(struct Mailbox *m, struct Maildir **md, struct Prog snprintf(fn, sizeof(fn), "%s/%s", m->path, p->email->path); #ifdef USE_HCACHE + struct stat lastchanged = { 0 }; + int ret = 0; if (C_MaildirHeaderCacheVerify) { ret = stat(fn, &lastchanged); } - else - { - lastchanged.st_mtime = 0; - ret = 0; - } + const char *key = NULL; + size_t keylen = 0; if (m->magic == MUTT_MH) { key = p->email->path; @@ -1430,11 +1418,7 @@ struct Email *maildir_parse_message(enum MailboxType magic, const char *fname, * @retval 0 Success * @retval -1 Error */ -#ifdef USE_HCACHE int mh_sync_mailbox_message(struct Mailbox *m, int msgno, header_cache_t *hc) -#else -int mh_sync_mailbox_message(struct Mailbox *m, int msgno) -#endif { if (!m || !m->emails) return -1; @@ -1758,9 +1742,7 @@ int mh_mbox_sync(struct Mailbox *m, int *index_hint) return -1; int i, j; -#ifdef USE_HCACHE header_cache_t *hc = NULL; -#endif char msgbuf[PATH_MAX + 64]; struct Progress progress; @@ -1788,13 +1770,8 @@ int mh_mbox_sync(struct Mailbox *m, int *index_hint) if (!m->quiet) mutt_progress_update(&progress, i, -1); -#ifdef USE_HCACHE if (mh_sync_mailbox_message(m, i, hc) == -1) goto err; -#else - if (mh_sync_mailbox_message(m, i) == -1) - goto err; -#endif } #ifdef USE_HCACHE diff --git a/mx.h b/mx.h index f7d326991..cbf7006ef 100644 --- a/mx.h +++ b/mx.h @@ -29,9 +29,7 @@ #include #include #include "config/lib.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif struct Email; struct Context; diff --git a/nntp/newsrc.c b/nntp/newsrc.c index 5cfc4530b..b5e1fe66d 100644 --- a/nntp/newsrc.c +++ b/nntp/newsrc.c @@ -56,9 +56,7 @@ #include "nntp.h" #include "protos.h" #include "sort.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif /* These Config Variables are only used in nntp/newsrc.c */ char *C_NewsCacheDir; ///< Config: (nntp) Directory for cached news articles @@ -992,9 +990,6 @@ const char *nntp_format_str(char *buf, size_t buflen, size_t col, int cols, char struct NntpAccountData *nntp_select_server(struct Mailbox *m, char *server, bool leave_lock) { char file[PATH_MAX]; -#ifdef USE_HCACHE - char *p = NULL; -#endif int rc; struct ConnAccount acct = { { 0 } }; struct NntpAccountData *adata = NULL; @@ -1117,7 +1112,7 @@ struct NntpAccountData *nntp_select_server(struct Mailbox *m, char *server, bool void *hdata = NULL; char *group = entry->d_name; - p = group + strlen(group) - 7; + char *p = group + strlen(group) - 7; if ((strlen(group) < 8) || (strcmp(p, ".hcache") != 0)) continue; *p = '\0'; diff --git a/nntp/nntp.c b/nntp/nntp.c index 257975503..2a38c0ec3 100644 --- a/nntp/nntp.c +++ b/nntp/nntp.c @@ -60,9 +60,7 @@ #include "progress.h" #include "protos.h" #include "sort.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif #ifdef USE_SASL #include #include @@ -98,9 +96,7 @@ struct FetchCtx bool restore; unsigned char *messages; struct Progress progress; -#ifdef USE_HCACHE header_cache_t *hc; -#endif }; /** @@ -1260,9 +1256,6 @@ static int nntp_fetch_headers(struct Mailbox *m, void *hc, anum_t first, anum_t int rc = 0; anum_t current; anum_t first_over = first; -#ifdef USE_HCACHE - void *hdata = NULL; -#endif /* if empty group or nothing to do */ if (!last || (first > last)) @@ -1276,9 +1269,7 @@ static int nntp_fetch_headers(struct Mailbox *m, void *hc, anum_t first, anum_t fc.messages = mutt_mem_calloc(last - first + 1, sizeof(unsigned char)); if (!fc.messages) return -1; -#ifdef USE_HCACHE fc.hc = hc; -#endif if (!m->emails) { @@ -1358,7 +1349,7 @@ static int nntp_fetch_headers(struct Mailbox *m, void *hc, anum_t first, anum_t #ifdef USE_HCACHE /* try to fetch header from cache */ - hdata = mutt_hcache_fetch(fc.hc, buf, strlen(buf)); + void *hdata = mutt_hcache_fetch(fc.hc, buf, strlen(buf)); if (hdata) { mutt_debug(LL_DEBUG2, "mutt_hcache_fetch %s\n", buf); @@ -2623,9 +2614,6 @@ static int nntp_mbox_sync(struct Mailbox *m, int *index_hint) struct NntpMboxData *mdata = m->mdata; int rc; -#ifdef USE_HCACHE - header_cache_t *hc = NULL; -#endif /* check for new articles */ mdata->adata->check_time = 0; @@ -2635,7 +2623,7 @@ static int nntp_mbox_sync(struct Mailbox *m, int *index_hint) #ifdef USE_HCACHE mdata->last_cached = 0; - hc = nntp_hcache_open(mdata); + header_cache_t *hc = nntp_hcache_open(mdata); #endif for (int i = 0; i < m->msg_count; i++) diff --git a/nntp/nntp_private.h b/nntp/nntp_private.h index b1ae04503..90a8f2877 100644 --- a/nntp/nntp_private.h +++ b/nntp/nntp_private.h @@ -26,9 +26,7 @@ #include "config.h" #include #include "nntp.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif struct Connection; struct Email; @@ -61,10 +59,7 @@ void nntp_mdata_free(void **ptr); void nntp_newsrc_gen_entries(struct Mailbox *m); int nntp_open_connection(struct NntpAccountData *adata); void nntp_article_status(struct Mailbox *m, struct Email *e, char *group, anum_t anum); - -#ifdef USE_HCACHE header_cache_t *nntp_hcache_open(struct NntpMboxData *mdata); void nntp_hcache_update(struct NntpMboxData *mdata, header_cache_t *hc); -#endif #endif /* MUTT_NNTP_NNTP_PRIVATE_H */ diff --git a/notmuch/mutt_notmuch.c b/notmuch/mutt_notmuch.c index 885831619..37706c617 100644 --- a/notmuch/mutt_notmuch.c +++ b/notmuch/mutt_notmuch.c @@ -55,6 +55,7 @@ #include "mutt/mutt.h" #include "config/lib.h" #include "email/lib.h" +#include "hcache/hcache.h" #include "mutt.h" #include "mutt_notmuch.h" #include "account.h" @@ -84,6 +85,31 @@ char *C_NmUnreadTag; ///< Config: (notmuch) Tag to use for unread messages char *C_NmFlaggedTag; ///< Config: (notmuch) Tag to use for flagged messages char *C_NmRepliedTag; ///< Config: (notmuch) Tag to use for replied messages +/** + * nm_hcache_open - Open a header cache + * @param m Mailbox + * @retval ptr Header cache handle + */ +static header_cache_t *nm_hcache_open(struct Mailbox *m) +{ +#ifdef USE_HCACHE + return mutt_hcache_open(C_HeaderCache, m->path, NULL); +#else + return NULL; +#endif +} + +/** + * nm_hcache_close - Close the header cache + * @param h Header cache handle + */ +void nm_hcache_close(header_cache_t *h) +{ +#ifdef USE_HCACHE + mutt_hcache_close(h); +#endif +} + /** * string_to_query_type - Lookup a query type * @param str String to lookup @@ -873,8 +899,8 @@ static struct Email *get_mutt_email(struct Mailbox *m, notmuch_message_t *msg) * @param msg Notmuch message * @param dedup De-duplicate results */ -static void append_message(struct Mailbox *m, notmuch_query_t *q, - notmuch_message_t *msg, bool dedup) +static void append_message(header_cache_t *h, struct Mailbox *m, + notmuch_query_t *q, notmuch_message_t *msg, bool dedup) { char *newpath = NULL; struct Email *e = NULL; @@ -905,25 +931,36 @@ static void append_message(struct Mailbox *m, notmuch_query_t *q, mutt_debug(LL_DEBUG2, "nm: allocate mx memory\n"); mx_alloc_memory(m); } - if (access(path, F_OK) == 0) - e = maildir_parse_message(MUTT_MAILDIR, path, false, NULL); + +#ifdef USE_HCACHE + void *from_cache = mutt_hcache_fetch(h, path, mutt_str_strlen(path)); + if (from_cache) + { + e = mutt_hcache_restore(from_cache); + } else +#endif { - /* maybe moved try find it... */ - char *folder = get_folder_from_path(path); - - if (folder) + if (access(path, F_OK) == 0) + e = maildir_parse_message(MUTT_MAILDIR, path, false, NULL); + else { - FILE *fp = maildir_open_find_message(folder, path, &newpath); - if (fp) + /* maybe moved try find it... */ + char *folder = get_folder_from_path(path); + + if (folder) { - e = maildir_parse_stream(MUTT_MAILDIR, fp, newpath, false, NULL); - fclose(fp); + FILE *fp = maildir_open_find_message(folder, path, &newpath); + if (fp) + { + e = maildir_parse_stream(MUTT_MAILDIR, fp, newpath, false, NULL); + fclose(fp); - mutt_debug(LL_DEBUG1, "nm: not up-to-date: %s -> %s\n", path, newpath); + mutt_debug(LL_DEBUG1, "nm: not up-to-date: %s -> %s\n", path, newpath); + } } + FREE(&folder); } - FREE(&folder); } if (!e) @@ -931,6 +968,19 @@ static void append_message(struct Mailbox *m, notmuch_query_t *q, mutt_debug(LL_DEBUG1, "nm: failed to parse message: %s\n", path); goto done; } + +#ifdef USE_HCACHE + + if (from_cache) + { + mutt_hcache_free(h, &from_cache); + } + else + { + mutt_hcache_store(h, newpath ? newpath : path, + mutt_str_strlen(newpath ? newpath : path), e, 0); + } +#endif if (init_email(e, newpath ? newpath : path, msg) != 0) { mutt_email_free(&e); @@ -969,7 +1019,7 @@ done: * * Careful, this calls itself recursively to make sure we get everything. */ -static void append_replies(struct Mailbox *m, notmuch_query_t *q, +static void append_replies(header_cache_t *h, struct Mailbox *m, notmuch_query_t *q, notmuch_message_t *top, bool dedup) { notmuch_messages_t *msgs = NULL; @@ -978,9 +1028,9 @@ static void append_replies(struct Mailbox *m, notmuch_query_t *q, notmuch_messages_move_to_next(msgs)) { notmuch_message_t *nm = notmuch_messages_get(msgs); - append_message(m, q, nm, dedup); + append_message(h, m, q, nm, dedup); /* recurse through all the replies to this message too */ - append_replies(m, q, nm, dedup); + append_replies(h, m, q, nm, dedup); notmuch_message_destroy(nm); } } @@ -995,7 +1045,7 @@ static void append_replies(struct Mailbox *m, notmuch_query_t *q, * add each top level reply in the thread, and then add each reply to the top * level replies */ -static void append_thread(struct Mailbox *m, notmuch_query_t *q, +static void append_thread(header_cache_t *h, struct Mailbox *m, notmuch_query_t *q, notmuch_thread_t *thread, bool dedup) { notmuch_messages_t *msgs = NULL; @@ -1004,8 +1054,8 @@ static void append_thread(struct Mailbox *m, notmuch_query_t *q, notmuch_messages_valid(msgs); notmuch_messages_move_to_next(msgs)) { notmuch_message_t *nm = notmuch_messages_get(msgs); - append_message(m, q, nm, dedup); - append_replies(m, q, nm, dedup); + append_message(h, m, q, nm, dedup); + append_replies(h, m, q, nm, dedup); notmuch_message_destroy(nm); } } @@ -1060,18 +1110,23 @@ static bool read_mesgs_query(struct Mailbox *m, notmuch_query_t *q, bool dedup) if (!msgs) return false; + header_cache_t *h = nm_hcache_open(m); + for (; notmuch_messages_valid(msgs) && ((limit == 0) || (m->msg_count < limit)); notmuch_messages_move_to_next(msgs)) { if (SigInt == 1) { + nm_hcache_close(h); SigInt = 0; return false; } notmuch_message_t *nm = notmuch_messages_get(msgs); - append_message(m, q, nm, dedup); + append_message(h, m, q, nm, dedup); notmuch_message_destroy(nm); } + + nm_hcache_close(h); return true; } @@ -1122,6 +1177,8 @@ static bool read_threads_query(struct Mailbox *m, notmuch_query_t *q, bool dedup if (!threads) return false; + header_cache_t *h = nm_hcache_open(m); + for (; notmuch_threads_valid(threads) && ((limit == 0) || (m->msg_count < limit)); notmuch_threads_move_to_next(threads)) { @@ -1131,9 +1188,11 @@ static bool read_threads_query(struct Mailbox *m, notmuch_query_t *q, bool dedup return false; } notmuch_thread_t *thread = notmuch_threads_get(threads); - append_thread(m, q, thread, dedup); + append_thread(h, m, q, thread, dedup); notmuch_thread_destroy(thread); } + + nm_hcache_close(h); return true; } @@ -2220,6 +2279,8 @@ static int nm_mbox_check(struct Mailbox *m, int *index_hint) goto done; #endif + header_cache_t *h = nm_hcache_open(m); + for (int i = 0; notmuch_messages_valid(msgs) && ((limit == 0) || (i < limit)); notmuch_messages_move_to_next(msgs), i++) { @@ -2232,7 +2293,7 @@ static int nm_mbox_check(struct Mailbox *m, int *index_hint) if (!e) { /* new email */ - append_message(m, NULL, msg, 0); + append_message(h, m, NULL, msg, 0); notmuch_message_destroy(msg); continue; } @@ -2263,6 +2324,8 @@ static int nm_mbox_check(struct Mailbox *m, int *index_hint) notmuch_message_destroy(msg); } + nm_hcache_close(h); + for (int i = 0; i < m->msg_count; i++) { if (!m->emails[i]->active) @@ -2318,6 +2381,8 @@ static int nm_mbox_sync(struct Mailbox *m, int *index_hint) mutt_progress_init(&progress, msgbuf, MUTT_PROGRESS_MSG, C_WriteInc, m->msg_count); } + header_cache_t *h = nm_hcache_open(m); + for (int i = 0; i < m->msg_count; i++) { char old[PATH_MAX], new[PATH_MAX]; @@ -2341,11 +2406,7 @@ static int nm_mbox_sync(struct Mailbox *m, int *index_hint) mutt_str_strfcpy(m->path, edata->folder, sizeof(m->path)); m->magic = edata->magic; -#ifdef USE_HCACHE - rc = mh_sync_mailbox_message(m, i, NULL); -#else - rc = mh_sync_mailbox_message(m, i); -#endif + rc = mh_sync_mailbox_message(m, i, h); mutt_str_strfcpy(m->path, uri, sizeof(m->path)); m->magic = MUTT_NOTMUCH; @@ -2377,6 +2438,8 @@ static int nm_mbox_sync(struct Mailbox *m, int *index_hint) m->mtime.tv_nsec = 0; } + nm_hcache_close(h); + FREE(&uri); mutt_debug(LL_DEBUG1, "nm: .... sync done [rc=%d]\n", rc); return rc; diff --git a/pop/pop.c b/pop/pop.c index 70d1ba445..9ee0c0c37 100644 --- a/pop/pop.c +++ b/pop/pop.c @@ -58,9 +58,7 @@ #include "mx.h" #include "ncrypt/ncrypt.h" #include "progress.h" -#ifdef USE_HCACHE #include "hcache/hcache.h" -#endif #ifdef ENABLE_NLS #include #endif @@ -73,10 +71,8 @@ unsigned char C_PopDelete; ///< Config: (pop) After downloading POP messages, de char *C_PopHost; ///< Config: (pop) Url of the POP server bool C_PopLast; ///< Config: (pop) Use the 'LAST' command to fetch new mail -#ifdef USE_HCACHE #define HC_FNAME "neomutt" /* filename for hcache as POP lacks paths */ #define HC_FEXT "hcache" /* extension for hcache as POP lacks paths */ -#endif /** * cache_id - Make a message-cache-compatible id -- 2.40.0