From: Kevin McCarthy Date: Tue, 15 May 2018 01:12:39 +0000 (-0700) Subject: Add basic CONDSTORE support when fetching initial messages. X-Git-Tag: 2019-10-25~665^2~7 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=e41b2e47e6a777e814c5826e06c644cf671eb402;p=neomutt Add basic CONDSTORE support when fetching initial messages. Store MODSEQ in the header cache, and use that to perform a "FETCH CHANGEDSINCE" for header updates when initially downloading messages. Further improvements could be made to add support when syncing. Handling MODSEQ for FLAG updates while the mailbox is open would be complicated by the fact that Mutt supports locally modified headers, so we couldn't accept the new (or subsequent) MODSEQ. However, this initial step may at least provide some benefit when opening the mailbox, which is generally the most time and data intensive. --- diff --git a/globals.h b/globals.h index d71b67b47..46aa9ccb3 100644 --- a/globals.h +++ b/globals.h @@ -224,6 +224,7 @@ WHERE bool Header; ///< Config: Include the message head WHERE bool Help; ///< Config: Display a help line with common key bindings #ifdef USE_IMAP WHERE bool ImapCheckSubscribed; ///< Config: (imap) When opening a mailbox, ask the server for a list of subscribed folders +WHERE bool ImapCondStore; WHERE bool ImapListSubscribed; ///< Config: (imap) When browsing a mailbox, only display subscribed folders WHERE bool ImapPassive; ///< Config: (imap) Reuse an existing IMAP connection to check for new mail WHERE bool ImapPeek; ///< Config: (imap) Don't mark messages as read when fetching them from the server diff --git a/imap/command.c b/imap/command.c index f29ad56bc..b7d4a0af1 100644 --- a/imap/command.c +++ b/imap/command.c @@ -67,23 +67,12 @@ bool ImapServernoise; ///< Config: (imap) Display server warnings as error messa * @note Gmail documents one string but use another, so we support both. */ static const char *const Capabilities[] = { - "IMAP4", - "IMAP4rev1", - "STATUS", - "ACL", - "NAMESPACE", - "AUTH=CRAM-MD5", - "AUTH=GSSAPI", - "AUTH=ANONYMOUS", - "AUTH=OAUTHBEARER", - "STARTTLS", - "LOGINDISABLED", - "IDLE", - "SASL-IR", - "ENABLE", - "X-GM-EXT-1", - "X-GM-EXT1", - NULL, + "IMAP4", "IMAP4rev1", "STATUS", + "ACL", "NAMESPACE", "AUTH=CRAM-MD5", + "AUTH=GSSAPI", "AUTH=ANONYMOUS", "AUTH=OAUTHBEARER", + "STARTTLS", "LOGINDISABLED", "IDLE", + "SASL-IR", "ENABLE", "CONDSTORE", + "X-GM-EXT-1", "X-GM-EXT1", NULL, }; /** @@ -382,6 +371,26 @@ static void cmd_parse_fetch(struct ImapData *idata, char *s) break; s = imap_next_word(s); } + else if (mutt_str_strncasecmp("MODSEQ", s, 6) == 0) + { + s += 6; + SKIPWS(s); + if (*s != '(') + { + mutt_debug(1, "cmd_parse_fetch: bogus MODSEQ response: %s\n", s); + return; + } + s++; + while (*s && *s != ')') + s++; + if (*s == ')') + s++; + else + { + mutt_debug(1, "cmd_parse_fetch: Unterminated MODSEQ response: %s\n", s); + return; + } + } else if (*s == ')') break; /* end of request */ else if (*s) diff --git a/imap/imap.c b/imap/imap.c index 5cb4b1de6..c3869ea86 100644 --- a/imap/imap.c +++ b/imap/imap.c @@ -1594,6 +1594,7 @@ struct ImapStatus *imap_mboxcache_get(struct ImapData *idata, const char *mbox, header_cache_t *hc = NULL; void *uidvalidity = NULL; void *uidnext = NULL; + unsigned long long *modseq = NULL; #endif struct ListNode *np = NULL; @@ -1622,22 +1623,26 @@ struct ImapStatus *imap_mboxcache_get(struct ImapData *idata, const char *mbox, { uidvalidity = mutt_hcache_fetch_raw(hc, "/UIDVALIDITY", 12); uidnext = mutt_hcache_fetch_raw(hc, "/UIDNEXT", 8); + modseq = mutt_hcache_fetch_raw(hc, "/MODSEQ", 7); if (uidvalidity) { if (!status) { mutt_hcache_free(hc, &uidvalidity); mutt_hcache_free(hc, &uidnext); + mutt_hcache_free(hc, (void **) &modseq); mutt_hcache_close(hc); return imap_mboxcache_get(idata, mbox, 1); } status->uidvalidity = *(unsigned int *) uidvalidity; status->uidnext = uidnext ? *(unsigned int *) uidnext : 0; - mutt_debug(3, "hcache uidvalidity %u, uidnext %u\n", status->uidvalidity, - status->uidnext); + status->modseq = modseq ? *modseq : 0; + mutt_debug(3, "mboxcache: hcache uidvalidity %u, uidnext %u, modseq %llu\n", + status->uidvalidity, status->uidnext, status->modseq); } mutt_hcache_free(hc, &uidvalidity); mutt_hcache_free(hc, &uidnext); + mutt_hcache_free(hc, (void **) &modseq); mutt_hcache_close(hc); } #endif @@ -2060,7 +2065,15 @@ static int imap_mbox_open(struct Context *ctx) if (ImapCheckSubscribed) imap_exec(idata, "LSUB \"\" \"*\"", IMAP_CMD_QUEUE); - snprintf(bufout, sizeof(bufout), "%s %s", ctx->readonly ? "EXAMINE" : "SELECT", buf); + + snprintf(bufout, sizeof(bufout), "%s %s%s", ctx->readonly ? "EXAMINE" : "SELECT", buf, +#if USE_HCACHE + mutt_bit_isset(idata->capabilities, CONDSTORE) && ImapCondStore ? + " (CONDSTORE)" : + ""); +#else + ""); +#endif idata->state = IMAP_SELECTED; @@ -2122,6 +2135,20 @@ static int imap_mbox_open(struct Context *ctx) goto fail; status->uidnext = idata->uidnext; } + else if (mutt_str_strncasecmp("OK [HIGHESTMODSEQ", pc, 17) == 0) + { + mutt_debug(3, "Getting mailbox HIGHESTMODSEQ\n"); + pc += 3; + pc = imap_next_word(pc); + if (mutt_str_atoull(pc, &idata->modseq) < 0) + goto fail; + status->modseq = idata->modseq; + } + else if (mutt_str_strncasecmp("OK [NOMODSEQ", pc, 12) == 0) + { + mutt_debug(3, "Mailbox has NOMODSEQ set\n"); + status->modseq = idata->modseq = 0; + } else { pc = imap_next_word(pc); diff --git a/imap/imap_private.h b/imap/imap_private.h index 78691db83..d9880f39f 100644 --- a/imap/imap_private.h +++ b/imap/imap_private.h @@ -135,6 +135,7 @@ enum ImapCaps IDLE, /**< RFC2177: IDLE */ SASL_IR, /**< SASL initial response draft */ ENABLE, /**< RFC5161 */ + CONDSTORE, /**< RFC7162 */ X_GM_EXT1, /**< https://developers.google.com/gmail/imap/imap-extensions */ X_GM_ALT1 = X_GM_EXT1, /**< Alternative capability string */ @@ -166,6 +167,7 @@ struct ImapStatus unsigned int uidnext; unsigned int uidvalidity; unsigned int unseen; + unsigned long long modseq; /* Used by CONDSTORE. 1 <= modseq < 2^63 */ }; /** @@ -254,6 +256,7 @@ struct ImapData struct Hash *uid_hash; unsigned int uid_validity; unsigned int uidnext; + unsigned long long modseq; struct Header **msn_index; /**< look up headers by (MSN-1) */ size_t msn_index_size; /**< allocation size */ unsigned int max_msn; /**< the largest MSN fetched so far */ diff --git a/imap/message.c b/imap/message.c index c1414b1df..be3ebf104 100644 --- a/imap/message.c +++ b/imap/message.c @@ -361,6 +361,26 @@ static int msg_parse_fetch(struct ImapHeader *h, char *s) /* handle above, in msg_fetch_header */ return -2; } + else if (mutt_str_strncasecmp("MODSEQ", s, 6) == 0) + { + s += 6; + SKIPWS(s); + if (*s != '(') + { + mutt_debug(1, "msg_parse_flags: bogus MODSEQ response: %s\n", s); + return -1; + } + s++; + while (*s && *s != ')') + s++; + if (*s == ')') + s++; + else + { + mutt_debug(1, "msg_parse_flags: Unterminated MODSEQ response: %s\n", s); + return -1; + } + } else if (*s == ')') s++; /* end of request */ else if (*s) @@ -630,6 +650,11 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i void *uid_validity = NULL; void *puidnext = NULL; unsigned int uidnext = 0; + int save_modseq = 0; + int has_condstore = 0; + int eval_condstore = 0; + unsigned long long *pmodseq = NULL; + unsigned long long hc_modseq = 0; #endif /* USE_HCACHE */ struct Context *ctx = idata->ctx; @@ -681,8 +706,30 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i uidnext = *(unsigned int *) puidnext; mutt_hcache_free(idata->hcache, &puidnext); } + /* Always save the MODSEQ, even if the server sent NOMODSEQ. */ + if (mutt_bit_isset(idata->capabilities, CONDSTORE) && ImapCondStore) + { + save_modseq = 1; + if (idata->modseq) + has_condstore = 1; + } if (uid_validity && uidnext && *(unsigned int *) uid_validity == idata->uid_validity) + { evalhc = true; + if (has_condstore) + { + pmodseq = mutt_hcache_fetch_raw(idata->hcache, "/MODSEQ", 7); + if (pmodseq) + { + hc_modseq = *pmodseq; + mutt_hcache_free(idata->hcache, (void **) &pmodseq); + } + /* The RFC doesn't allow a 0 value for CHANGEDSINCE, so we only + * do the CONDSTORE FETCH if we have a modseq to compare against. */ + if (hc_modseq) + eval_condstore = 1; + } + } mutt_hcache_free(idata->hcache, &uid_validity); } if (evalhc) @@ -692,7 +739,11 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i mutt_progress_init(&progress, _("Evaluating cache..."), MUTT_PROGRESS_MSG, ReadInc, msn_end); - snprintf(buf, sizeof(buf), "UID FETCH 1:%u (UID FLAGS)", uidnext - 1); + /* If we are using CONDSTORE's "FETCH CHANGEDSINCE", then we keep + * the flags in the header cache, and update them further below. + * Otherwise, we fetch the current state of the flags here. */ + snprintf(buf, sizeof(buf), "UID FETCH 1:%u (UID%s)", uidnext - 1, + eval_condstore ? "" : " FLAGS"); imap_cmd_start(idata, buf); @@ -746,12 +797,24 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i /* messages which have not been expunged are ACTIVE (borrowed from mh * folders) */ ctx->hdrs[idx]->active = true; - ctx->hdrs[idx]->read = h.data->read; - ctx->hdrs[idx]->old = h.data->old; - ctx->hdrs[idx]->deleted = h.data->deleted; - ctx->hdrs[idx]->flagged = h.data->flagged; - ctx->hdrs[idx]->replied = h.data->replied; - ctx->hdrs[idx]->changed = h.data->changed; + ctx->hdrs[idx]->changed = 0; + if (!eval_condstore) + { + ctx->hdrs[idx]->read = h.data->read; + ctx->hdrs[idx]->old = h.data->old; + ctx->hdrs[idx]->deleted = h.data->deleted; + ctx->hdrs[idx]->flagged = h.data->flagged; + ctx->hdrs[idx]->replied = h.data->replied; + } + else + { + h.data->read = ctx->hdrs[idx]->read; + h.data->old = ctx->hdrs[idx]->old; + h.data->deleted = ctx->hdrs[idx]->deleted; + h.data->flagged = ctx->hdrs[idx]->flagged; + h.data->replied = ctx->hdrs[idx]->replied; + } + /* ctx->hdrs[msgno]->received is restored from mutt_hcache_restore */ ctx->hdrs[idx]->data = (void *) (h.data); STAILQ_INIT(&ctx->hdrs[idx]->tags); @@ -760,6 +823,11 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i ctx->msgcount++; ctx->size += ctx->hdrs[idx]->content->length; + /* If this is the first time we are fetching, we need to + * store the current state of flags back into the header cache */ + if (has_condstore && !eval_condstore) + imap_hcache_put(idata, ctx->hdrs[idx]); + h.data = NULL; idx++; } @@ -774,6 +842,57 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i } } + if (eval_condstore && (hc_modseq != idata->modseq)) + { + unsigned int header_msn; + char *fetch_buf; + + /* L10N: + Fetching IMAP flag changes, using the CONDSTORE extension */ + mutt_progress_init(&progress, _("Fetching flag updates..."), + MUTT_PROGRESS_MSG, ReadInc, msn_end); + + snprintf(buf, sizeof(buf), "UID FETCH 1:%u (FLAGS) (CHANGEDSINCE %llu)", + uidnext - 1, hc_modseq); + + imap_cmd_start(idata, buf); + + rc = IMAP_CMD_CONTINUE; + for (msgno = 1; rc == IMAP_CMD_CONTINUE; msgno++) + { + mutt_progress_update(&progress, msgno, -1); + + /* cmd_parse_fetch will update the flags */ + rc = imap_cmd_step(idata); + if (rc != IMAP_CMD_CONTINUE) + break; + + /* so we just need to grab the header and persist it back into + * the header cache */ + fetch_buf = idata->buf; + if (fetch_buf[0] != '*') + continue; + + fetch_buf = imap_next_word(fetch_buf); + if (mutt_str_atoui(fetch_buf, &header_msn) < 0) + continue; + + if (header_msn < 1 || header_msn > msn_end || !idata->msn_index[header_msn - 1]) + { + mutt_debug(1, "imap_read_headers: skipping CONDSTORE flag update for unknown message number %u\n", + header_msn); + continue; + } + + imap_hcache_put(idata, idata->msn_index[header_msn - 1]); + } + + /* The IMAP flag setting as part of cmd_parse_fetch() ends up + * flipping these on. */ + idata->check_status &= ~IMAP_FLAGS_PENDING; + ctx->changed = 0; + } + /* Look for the first empty MSN and start there */ while (msn_begin <= msn_end) { @@ -863,12 +982,12 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i /* messages which have not been expunged are ACTIVE (borrowed from mh * folders) */ ctx->hdrs[idx]->active = true; + ctx->hdrs[idx]->changed = 0; ctx->hdrs[idx]->read = h.data->read; ctx->hdrs[idx]->old = h.data->old; ctx->hdrs[idx]->deleted = h.data->deleted; ctx->hdrs[idx]->flagged = h.data->flagged; ctx->hdrs[idx]->replied = h.data->replied; - ctx->hdrs[idx]->changed = h.data->changed; ctx->hdrs[idx]->received = h.received; ctx->hdrs[idx]->data = (void *) (h.data); STAILQ_INIT(&ctx->hdrs[idx]->tags); @@ -945,6 +1064,11 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i mutt_hcache_store_raw(idata->hcache, "/UIDNEXT", 8, &idata->uidnext, sizeof(idata->uidnext)); } + if (save_modseq) + { + mutt_hcache_store_raw(idata->hcache, "/MODSEQ", 7, &idata->modseq, + sizeof(idata->modseq)); + } imap_hcache_close(idata); #endif /* USE_HCACHE */ diff --git a/imap/message.h b/imap/message.h index b99b2bbe9..e32b4cc7d 100644 --- a/imap/message.h +++ b/imap/message.h @@ -41,7 +41,6 @@ struct ImapHeaderData bool deleted : 1; bool flagged : 1; bool replied : 1; - bool changed : 1; bool parsed : 1; diff --git a/init.h b/init.h index 688010043..dc58aa059 100644 --- a/init.h +++ b/init.h @@ -1421,6 +1421,14 @@ struct ConfigDef MuttVars[] = { ** of mailboxes it polls for new mail just as if you had issued individual ** ``$mailboxes'' commands. */ + { "imap_condstore", DT_BOOL, R_NONE, &ImapCondStore, 0 }, + /* + ** .pp + ** + ** When \fIset\fP, mutt will use the CONDSTORE extension (RFC 7162) + ** if advertised by the server. Mutt's current implementation is basic, + ** used only for initial message fetching and flag updates. + */ { "imap_delim_chars", DT_STRING, R_NONE, &ImapDelimChars, IP "/." }, /* ** .pp diff --git a/mutt/string.c b/mutt/string.c index d68ad435e..a776e41f5 100644 --- a/mutt/string.c +++ b/mutt/string.c @@ -277,6 +277,34 @@ int mutt_str_atoul(const char *str, unsigned long *dst) return 0; } +/* NOTE: this function's return value is different from mutt_atol. + * + * returns: 1 - successful conversion, with trailing characters + * 0 - successful conversion + * -1 - invalid input + */ +int mutt_str_atoull(const char *str, unsigned long long *dst) +{ + unsigned long long r; + unsigned long long *res = dst ? dst : &r; + char *e = NULL; + + /* no input: 0 */ + if (!str || !*str) + { + *res = 0; + return 0; + } + + errno = 0; + *res = strtoull(str, &e, 10); + if (*res == ULLONG_MAX && errno == ERANGE) + return -1; + if (e && *e != '\0') + return 1; + return 0; +} + /** * mutt_str_strdup - Copy a string, safely * @param str String to copy diff --git a/mutt/string2.h b/mutt/string2.h index ce67d6ed1..c7aa71f78 100644 --- a/mutt/string2.h +++ b/mutt/string2.h @@ -68,6 +68,7 @@ int mutt_str_atol(const char *str, long *dst); int mutt_str_atos(const char *str, short *dst); int mutt_str_atoui(const char *str, unsigned int *dst); int mutt_str_atoul(const char *str, unsigned long *dst); +int mutt_str_atoull(const char *str, unsigned long long *dst); void mutt_str_dequote_comment(char *s); const char *mutt_str_find_word(const char *src); const char *mutt_str_getenv(const char *name);