From: Kevin McCarthy Date: Tue, 15 May 2018 01:12:39 +0000 (-0700) Subject: Add basic CONDSTORE support when fetching initial messages. X-Git-Tag: mutt-1-11-rel~99 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=9079c2a88fa1cbd4a61d223ce5cc8f0d84c855f8;p=mutt Add basic CONDSTORE support when fetching initial messages. Store MODSEQ in the header cache, and use that to perform a "FETCH CHANGEDSINCE" for header updates when initially downloading messages. Further improvements could be made to add support when syncing. Handling MODSEQ for FLAG updates while the mailbox is open would be complicated by the fact that Mutt supports locally modified headers, so we couldn't accept the new (or subsequent) MODSEQ. However, this initial step may at least provide some benefit when opening the mailbox, which is generally the most time and data intensive. --- diff --git a/imap/command.c b/imap/command.c index 9b27f5dd..3a2a4933 100644 --- a/imap/command.c +++ b/imap/command.c @@ -70,6 +70,7 @@ static const char * const Capabilities[] = { "IDLE", "SASL-IR", "ENABLE", + "CONDSTORE", NULL }; @@ -732,6 +733,28 @@ static void cmd_parse_fetch (IMAP_DATA* idata, char* s) } s = imap_next_word (s); } + else if (ascii_strncasecmp ("MODSEQ", s, 6) == 0) + { + s += 6; + SKIPWS(s); + if (*s != '(') + { + dprint (1, (debugfile, "cmd_parse_fetch: bogus MODSEQ response: %s\n", + s)); + return; + } + s++; + while (*s && *s != ')') + s++; + if (*s == ')') + s++; + else + { + dprint (1, (debugfile, + "cmd_parse_fetch: Unterminated MODSEQ response: %s\n", s)); + return; + } + } else if (*s == ')') s++; /* end of request */ else if (*s) diff --git a/imap/imap.c b/imap/imap.c index be766e04..47a836ac 100644 --- a/imap/imap.c +++ b/imap/imap.c @@ -657,8 +657,15 @@ static int imap_open_mailbox (CONTEXT* ctx) imap_status (Postponed, 1); FREE (&pmx.mbox); - snprintf (bufout, sizeof (bufout), "%s %s", - ctx->readonly ? "EXAMINE" : "SELECT", buf); + snprintf (bufout, sizeof (bufout), "%s %s%s", + ctx->readonly ? "EXAMINE" : "SELECT", + buf, +#if USE_HCACHE + mutt_bit_isset (idata->capabilities, CONDSTORE) && + option (OPTIMAPCONDSTORE) ? " (CONDSTORE)" : ""); +#else + ""); +#endif idata->state = IMAP_SELECTED; @@ -717,6 +724,20 @@ static int imap_open_mailbox (CONTEXT* ctx) goto fail; status->uidnext = idata->uidnext; } + else if (ascii_strncasecmp ("OK [HIGHESTMODSEQ", pc, 17) == 0) + { + dprint (3, (debugfile, "Getting mailbox HIGHESTMODSEQ\n")); + pc += 3; + pc = imap_next_word (pc); + if (mutt_atoull (pc, &idata->modseq) < 0) + goto fail; + status->modseq = idata->modseq; + } + else if (ascii_strncasecmp ("OK [NOMODSEQ", pc, 12) == 0) + { + dprint (3, (debugfile, "Mailbox has NOMODSEQ set\n")); + status->modseq = idata->modseq = 0; + } else { pc = imap_next_word (pc); @@ -1710,6 +1731,7 @@ IMAP_STATUS* imap_mboxcache_get (IMAP_DATA* idata, const char* mbox, int create) header_cache_t *hc = NULL; unsigned int *uidvalidity = NULL; unsigned int *uidnext = NULL; + unsigned long long *modseq = NULL; #endif for (cur = idata->mboxcache; cur; cur = cur->next) @@ -1738,22 +1760,26 @@ IMAP_STATUS* imap_mboxcache_get (IMAP_DATA* idata, const char* mbox, int create) { uidvalidity = mutt_hcache_fetch_raw (hc, "/UIDVALIDITY", imap_hcache_keylen); uidnext = mutt_hcache_fetch_raw (hc, "/UIDNEXT", imap_hcache_keylen); + modseq = mutt_hcache_fetch_raw (hc, "/MODSEQ", imap_hcache_keylen); if (uidvalidity) { if (!status) { mutt_hcache_free ((void **)&uidvalidity); mutt_hcache_free ((void **)&uidnext); + mutt_hcache_free ((void **)&modseq); mutt_hcache_close (hc); return imap_mboxcache_get (idata, mbox, 1); } status->uidvalidity = *uidvalidity; status->uidnext = uidnext ? *uidnext: 0; - dprint (3, (debugfile, "mboxcache: hcache uidvalidity %u, uidnext %u\n", - status->uidvalidity, status->uidnext)); + status->modseq = modseq ? *modseq: 0; + dprint (3, (debugfile, "mboxcache: hcache uidvalidity %u, uidnext %u, modseq %llu\n", + status->uidvalidity, status->uidnext, status->modseq)); } mutt_hcache_free ((void **)&uidvalidity); mutt_hcache_free ((void **)&uidnext); + mutt_hcache_free ((void **)&modseq); mutt_hcache_close (hc); } #endif diff --git a/imap/imap_private.h b/imap/imap_private.h index 8d9d4fcc..fed8bfe6 100644 --- a/imap/imap_private.h +++ b/imap/imap_private.h @@ -118,6 +118,7 @@ enum IDLE, /* RFC 2177: IDLE */ SASL_IR, /* SASL initial response draft */ ENABLE, /* RFC 5161 */ + CONDSTORE, /* RFC 7162 */ CAPMAX }; @@ -142,6 +143,7 @@ typedef struct unsigned int uidnext; unsigned int uidvalidity; unsigned int unseen; + unsigned long long modseq; /* Used by CONDSTORE. 1 <= modseq < 2^63 */ } IMAP_STATUS; typedef struct @@ -219,6 +221,7 @@ typedef struct HASH *uid_hash; unsigned int uid_validity; unsigned int uidnext; + unsigned long long modseq; HEADER **msn_index; /* look up headers by (MSN-1) */ unsigned int msn_index_size; /* allocation size */ unsigned int max_msn; /* the largest MSN fetched so far */ diff --git a/imap/message.c b/imap/message.c index e6056555..85399929 100644 --- a/imap/message.c +++ b/imap/message.c @@ -183,6 +183,11 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms unsigned int *uid_validity = NULL; unsigned int *puidnext = NULL; unsigned int uidnext = 0; + int save_modseq = 0; + int has_condstore = 0; + int eval_condstore = 0; + unsigned long long *pmodseq = NULL; + unsigned long long hc_modseq = 0; #endif /* USE_HCACHE */ ctx = idata->ctx; @@ -237,8 +242,30 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms uidnext = *puidnext; mutt_hcache_free ((void **)&puidnext); } + /* Always save the MODSEQ, even if the server sent NOMODSEQ. */ + if (mutt_bit_isset (idata->capabilities, CONDSTORE) && option (OPTIMAPCONDSTORE)) + { + save_modseq = 1; + if (idata->modseq) + has_condstore = 1; + } if (uid_validity && uidnext && *uid_validity == idata->uid_validity) + { evalhc = 1; + if (has_condstore) + { + pmodseq = mutt_hcache_fetch_raw (idata->hcache, "/MODSEQ", imap_hcache_keylen); + if (pmodseq) + { + hc_modseq = *pmodseq; + mutt_hcache_free ((void **)&pmodseq); + } + /* The RFC doesn't allow a 0 value for CHANGEDSINCE, so we only + * do the CONDSTORE FETCH if we have a modseq to compare against. */ + if (hc_modseq) + eval_condstore = 1; + } + } mutt_hcache_free ((void **)&uid_validity); } if (evalhc) @@ -248,8 +275,12 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms mutt_progress_init (&progress, _("Evaluating cache..."), MUTT_PROGRESS_MSG, ReadInc, msn_end); + /* If we are using CONDSTORE's "FETCH CHANGEDSINCE", then we keep + * the flags in the header cache, and update them further below. + * Otherwise, we fetch the current state of the flags here. */ snprintf (buf, sizeof (buf), - "UID FETCH 1:%u (UID FLAGS)", uidnext - 1); + "UID FETCH 1:%u (UID%s)", uidnext - 1, + eval_condstore ? "" : " FLAGS"); imap_cmd_start (idata, buf); @@ -300,18 +331,35 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms /* messages which have not been expunged are ACTIVE (borrowed from mh * folders) */ ctx->hdrs[idx]->active = 1; - ctx->hdrs[idx]->read = h.data->read; - ctx->hdrs[idx]->old = h.data->old; - ctx->hdrs[idx]->deleted = h.data->deleted; - ctx->hdrs[idx]->flagged = h.data->flagged; - ctx->hdrs[idx]->replied = h.data->replied; - ctx->hdrs[idx]->changed = h.data->changed; + ctx->hdrs[idx]->changed = 0; + if (!eval_condstore) + { + ctx->hdrs[idx]->read = h.data->read; + ctx->hdrs[idx]->old = h.data->old; + ctx->hdrs[idx]->deleted = h.data->deleted; + ctx->hdrs[idx]->flagged = h.data->flagged; + ctx->hdrs[idx]->replied = h.data->replied; + } + else + { + h.data->read = ctx->hdrs[idx]->read; + h.data->old = ctx->hdrs[idx]->old; + h.data->deleted = ctx->hdrs[idx]->deleted; + h.data->flagged = ctx->hdrs[idx]->flagged; + h.data->replied = ctx->hdrs[idx]->replied; + } + /* ctx->hdrs[msgno]->received is restored from mutt_hcache_restore */ ctx->hdrs[idx]->data = (void *) (h.data); ctx->msgcount++; ctx->size += ctx->hdrs[idx]->content->length; + /* If this is the first time we are fetching, we need to + * store the current state of flags back into the header cache */ + if (has_condstore && !eval_condstore) + imap_hcache_put (idata, ctx->hdrs[idx]); + h.data = NULL; idx++; } @@ -327,6 +375,59 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms } } + if (eval_condstore && (hc_modseq != idata->modseq)) + { + unsigned int header_msn; + char *fetch_buf; + + /* L10N: + Fetching IMAP flag changes, using the CONDSTORE extension */ + mutt_progress_init (&progress, _("Fetching flag updates..."), + MUTT_PROGRESS_MSG, ReadInc, msn_end); + + snprintf (buf, sizeof (buf), + "UID FETCH 1:%u (FLAGS) (CHANGEDSINCE %llu)", + uidnext - 1, hc_modseq); + + imap_cmd_start (idata, buf); + + rc = IMAP_CMD_CONTINUE; + for (msgno = 1; rc == IMAP_CMD_CONTINUE; msgno++) + { + mutt_progress_update (&progress, msgno, -1); + + /* cmd_parse_fetch will update the flags */ + rc = imap_cmd_step (idata); + if (rc != IMAP_CMD_CONTINUE) + break; + + /* so we just need to grab the header and persist it back into + * the header cache */ + fetch_buf = idata->buf; + if (fetch_buf[0] != '*') + continue; + + fetch_buf = imap_next_word (fetch_buf); + if (mutt_atoui (fetch_buf, &header_msn) < 0) + continue; + + if (header_msn < 1 || header_msn > msn_end || + !idata->msn_index[header_msn - 1]) + { + dprint (1, (debugfile, "imap_read_headers: skipping CONDSTORE flag " + "update for unknown message number %u\n", header_msn)); + continue; + } + + imap_hcache_put (idata, idata->msn_index[header_msn - 1]); + } + + /* The IMAP flag setting as part of cmd_parse_fetch() ends up + * flipping these on. */ + idata->check_status &= ~IMAP_FLAGS_PENDING; + ctx->changed = 0; + } + /* Look for the first empty MSN and start there */ while (msn_begin <= msn_end) { @@ -417,12 +518,12 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms /* messages which have not been expunged are ACTIVE (borrowed from mh * folders) */ ctx->hdrs[idx]->active = 1; + ctx->hdrs[idx]->changed = 0; ctx->hdrs[idx]->read = h.data->read; ctx->hdrs[idx]->old = h.data->old; ctx->hdrs[idx]->deleted = h.data->deleted; ctx->hdrs[idx]->flagged = h.data->flagged; ctx->hdrs[idx]->replied = h.data->replied; - ctx->hdrs[idx]->changed = h.data->changed; ctx->hdrs[idx]->received = h.received; ctx->hdrs[idx]->data = (void *) (h.data); @@ -495,6 +596,9 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms if (idata->uidnext > 1) mutt_hcache_store_raw (idata->hcache, "/UIDNEXT", &idata->uidnext, sizeof (idata->uidnext), imap_hcache_keylen); + if (save_modseq) + mutt_hcache_store_raw (idata->hcache, "/MODSEQ", &idata->modseq, + sizeof (idata->modseq), imap_hcache_keylen); imap_hcache_close (idata); #endif /* USE_HCACHE */ @@ -1412,6 +1516,28 @@ static int msg_parse_fetch (IMAP_HEADER *h, char *s) /* handle above, in msg_fetch_header */ return -2; } + else if (ascii_strncasecmp ("MODSEQ", s, 6) == 0) + { + s += 6; + SKIPWS(s); + if (*s != '(') + { + dprint (1, (debugfile, "msg_parse_flags: bogus MODSEQ response: %s\n", + s)); + return -1; + } + s++; + while (*s && *s != ')') + s++; + if (*s == ')') + s++; + else + { + dprint (1, (debugfile, + "msg_parse_flags: Unterminated MODSEQ response: %s\n", s)); + return -1; + } + } else if (*s == ')') s++; /* end of request */ else if (*s) diff --git a/imap/message.h b/imap/message.h index 3c5d2cb2..864b7746 100644 --- a/imap/message.h +++ b/imap/message.h @@ -32,7 +32,6 @@ typedef struct imap_header_data unsigned int deleted : 1; unsigned int flagged : 1; unsigned int replied : 1; - unsigned int changed : 1; unsigned int parsed : 1; diff --git a/init.h b/init.h index c1724c13..30e1c766 100644 --- a/init.h +++ b/init.h @@ -1265,6 +1265,14 @@ struct option_t MuttVars[] = { ** it polls for new mail just as if you had issued individual ``$mailboxes'' ** commands. */ + { "imap_condstore", DT_BOOL, R_NONE, OPTIMAPCONDSTORE, 0 }, + /* + ** .pp + ** + ** When \fIset\fP, mutt will use the CONDSTORE extension (RFC 7162) + ** if advertised by the server. Mutt's current implementation is basic, + ** used only for initial message fetching and flag updates. + */ { "imap_delim_chars", DT_STR, R_NONE, UL &ImapDelimChars, UL "/." }, /* ** .pp diff --git a/lib.c b/lib.c index 78e02dfe..6f7113e1 100644 --- a/lib.c +++ b/lib.c @@ -1141,3 +1141,31 @@ int mutt_atoul (const char *str, unsigned long *dst) return 1; return 0; } + +/* NOTE: this function's return value is different from mutt_atol. + * + * returns: 1 - successful conversion, with trailing characters + * 0 - successful conversion + * -1 - invalid input + */ +int mutt_atoull (const char *str, unsigned long long *dst) +{ + unsigned long long r; + unsigned long long *res = dst ? dst : &r; + char *e = NULL; + + /* no input: 0 */ + if (!str || !*str) + { + *res = 0; + return 0; + } + + errno = 0; + *res = strtoull (str, &e, 10); + if (*res == ULLONG_MAX && errno == ERANGE) + return -1; + if (e && *e != '\0') + return 1; + return 0; +} diff --git a/lib.h b/lib.h index 61ac700d..11638dfe 100644 --- a/lib.h +++ b/lib.h @@ -193,6 +193,7 @@ int mutt_atoi (const char *, int *); int mutt_atol (const char *, long *); int mutt_atoui (const char *, unsigned int *); int mutt_atoul (const char *, unsigned long *); +int mutt_atoull (const char *, unsigned long long *); const char *mutt_stristr (const char *, const char *); const char *mutt_basename (const char *); diff --git a/mutt.h b/mutt.h index 87df6a4a..a646519c 100644 --- a/mutt.h +++ b/mutt.h @@ -416,6 +416,7 @@ enum OPTIGNORELISTREPLYTO, #ifdef USE_IMAP OPTIMAPCHECKSUBSCRIBED, + OPTIMAPCONDSTORE, OPTIMAPIDLE, OPTIMAPLSUB, OPTIMAPPASSIVE,