Add basic CONDSTORE support when fetching initial messages.

Store MODSEQ in the header cache, and use that to perform a "FETCH
CHANGEDSINCE" for header updates when initially downloading messages.

Further improvements could be made to add support when syncing.
Handling MODSEQ for FLAG updates while the mailbox is open would be
complicated by the fact that Mutt supports locally modified headers,
so we couldn't accept the new (or subsequent) MODSEQ.

However, this initial step may at least provide some benefit when
opening the mailbox, which is generally the most time and data
intensive.

diff --git a/imap/command.c b/imap/command.c
index 9b27f5dd9bd42356db9dbce101e6a3940087826a..3a2a4933ff39f70f54956f7d94580e8c97e1fc67 100644 (file)
--- a/imap/command.c
+++ b/imap/command.c
@@ -70,6 +70,7 @@ static const char * const Capabilities[] = {
   "IDLE",
   "SASL-IR",
   "ENABLE",
+  "CONDSTORE",
 
   NULL
 };
@@ -732,6 +733,28 @@ static void cmd_parse_fetch (IMAP_DATA* idata, char* s)
       }
       s = imap_next_word (s);
     }
+    else if (ascii_strncasecmp ("MODSEQ", s, 6) == 0)
+    {
+      s += 6;
+      SKIPWS(s);
+      if (*s != '(')
+      {
+        dprint (1, (debugfile, "cmd_parse_fetch: bogus MODSEQ response: %s\n",
+                    s));
+        return;
+      }
+      s++;
+      while (*s && *s != ')')
+        s++;
+      if (*s == ')')
+        s++;
+      else
+      {
+        dprint (1, (debugfile,
+                    "cmd_parse_fetch: Unterminated MODSEQ response: %s\n", s));
+        return;
+      }
+    }
     else if (*s == ')')
       s++; /* end of request */
     else if (*s)

diff --git a/imap/imap.c b/imap/imap.c
index be766e04a13eee6ec9c1aab172cbcc196f62ec75..47a836ac61b3ca5d8dfe1b3eb1685428fca01052 100644 (file)
--- a/imap/imap.c
+++ b/imap/imap.c
@@ -657,8 +657,15 @@ static int imap_open_mailbox (CONTEXT* ctx)
     imap_status (Postponed, 1);
   FREE (&pmx.mbox);
 
-  snprintf (bufout, sizeof (bufout), "%s %s",
-    ctx->readonly ? "EXAMINE" : "SELECT", buf);
+  snprintf (bufout, sizeof (bufout), "%s %s%s",
+            ctx->readonly ? "EXAMINE" : "SELECT",
+            buf,
+#if USE_HCACHE
+            mutt_bit_isset (idata->capabilities, CONDSTORE) &&
+            option (OPTIMAPCONDSTORE) ? " (CONDSTORE)" : "");
+#else
+            "");
+#endif
 
   idata->state = IMAP_SELECTED;
 
@@ -717,6 +724,20 @@ static int imap_open_mailbox (CONTEXT* ctx)
         goto fail;
       status->uidnext = idata->uidnext;
     }
+    else if (ascii_strncasecmp ("OK [HIGHESTMODSEQ", pc, 17) == 0)
+    {
+      dprint (3, (debugfile, "Getting mailbox HIGHESTMODSEQ\n"));
+      pc += 3;
+      pc = imap_next_word (pc);
+      if (mutt_atoull (pc, &idata->modseq) < 0)
+        goto fail;
+      status->modseq = idata->modseq;
+    }
+    else if (ascii_strncasecmp ("OK [NOMODSEQ", pc, 12) == 0)
+    {
+      dprint (3, (debugfile, "Mailbox has NOMODSEQ set\n"));
+      status->modseq = idata->modseq = 0;
+    }
     else
     {
       pc = imap_next_word (pc);
@@ -1710,6 +1731,7 @@ IMAP_STATUS* imap_mboxcache_get (IMAP_DATA* idata, const char* mbox, int create)
   header_cache_t *hc = NULL;
   unsigned int *uidvalidity = NULL;
   unsigned int *uidnext = NULL;
+  unsigned long long *modseq = NULL;
 #endif
 
   for (cur = idata->mboxcache; cur; cur = cur->next)
@@ -1738,22 +1760,26 @@ IMAP_STATUS* imap_mboxcache_get (IMAP_DATA* idata, const char* mbox, int create)
   {
     uidvalidity = mutt_hcache_fetch_raw (hc, "/UIDVALIDITY", imap_hcache_keylen);
     uidnext = mutt_hcache_fetch_raw (hc, "/UIDNEXT", imap_hcache_keylen);
+    modseq = mutt_hcache_fetch_raw (hc, "/MODSEQ", imap_hcache_keylen);
     if (uidvalidity)
     {
       if (!status)
       {
         mutt_hcache_free ((void **)&uidvalidity);
         mutt_hcache_free ((void **)&uidnext);
+        mutt_hcache_free ((void **)&modseq);
         mutt_hcache_close (hc);
         return imap_mboxcache_get (idata, mbox, 1);
       }
       status->uidvalidity = *uidvalidity;
       status->uidnext = uidnext ? *uidnext: 0;
-      dprint (3, (debugfile, "mboxcache: hcache uidvalidity %u, uidnext %u\n",
-                  status->uidvalidity, status->uidnext));
+      status->modseq = modseq ? *modseq: 0;
+      dprint (3, (debugfile, "mboxcache: hcache uidvalidity %u, uidnext %u, modseq %llu\n",
+                  status->uidvalidity, status->uidnext, status->modseq));
     }
     mutt_hcache_free ((void **)&uidvalidity);
     mutt_hcache_free ((void **)&uidnext);
+    mutt_hcache_free ((void **)&modseq);
     mutt_hcache_close (hc);
   }
 #endif

diff --git a/imap/imap_private.h b/imap/imap_private.h
index 8d9d4fcc250b2585fd3d7a3c2c6b0dd8f5de1c6d..fed8bfe6e3da963a9a68ea6964e70b0273c896f7 100644 (file)
--- a/imap/imap_private.h
+++ b/imap/imap_private.h
@@ -118,6 +118,7 @@ enum
   IDLE,                         /* RFC 2177: IDLE */
   SASL_IR,                      /* SASL initial response draft */
   ENABLE,                       /* RFC 5161 */
+  CONDSTORE,                    /* RFC 7162 */
 
   CAPMAX
 };
@@ -142,6 +143,7 @@ typedef struct
   unsigned int uidnext;
   unsigned int uidvalidity;
   unsigned int unseen;
+  unsigned long long modseq;  /* Used by CONDSTORE. 1 <= modseq < 2^63 */
 } IMAP_STATUS;
 
 typedef struct
@@ -219,6 +221,7 @@ typedef struct
   HASH *uid_hash;
   unsigned int uid_validity;
   unsigned int uidnext;
+  unsigned long long modseq;
   HEADER **msn_index;          /* look up headers by (MSN-1) */
   unsigned int msn_index_size; /* allocation size */
   unsigned int max_msn;        /* the largest MSN fetched so far */

diff --git a/imap/message.c b/imap/message.c
index e6056555510c4fc5760be9b5b0353b12239e74ad..85399929f4533953a0af4f6bbc45040a99801e49 100644 (file)
--- a/imap/message.c
+++ b/imap/message.c
@@ -183,6 +183,11 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms
   unsigned int *uid_validity = NULL;
   unsigned int *puidnext = NULL;
   unsigned int uidnext = 0;
+  int save_modseq = 0;
+  int has_condstore = 0;
+  int eval_condstore = 0;
+  unsigned long long *pmodseq = NULL;
+  unsigned long long hc_modseq = 0;
 #endif /* USE_HCACHE */
 
   ctx = idata->ctx;
@@ -237,8 +242,30 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms
       uidnext = *puidnext;
       mutt_hcache_free ((void **)&puidnext);
     }
+    /* Always save the MODSEQ, even if the server sent NOMODSEQ. */
+    if (mutt_bit_isset (idata->capabilities, CONDSTORE) && option (OPTIMAPCONDSTORE))
+    {
+      save_modseq = 1;
+      if (idata->modseq)
+        has_condstore = 1;
+    }
     if (uid_validity && uidnext && *uid_validity == idata->uid_validity)
+    {
       evalhc = 1;
+      if (has_condstore)
+      {
+        pmodseq = mutt_hcache_fetch_raw (idata->hcache, "/MODSEQ", imap_hcache_keylen);
+        if (pmodseq)
+        {
+          hc_modseq = *pmodseq;
+          mutt_hcache_free ((void **)&pmodseq);
+        }
+        /* The RFC doesn't allow a 0 value for CHANGEDSINCE, so we only
+         * do the CONDSTORE FETCH if we have a modseq to compare against. */
+        if (hc_modseq)
+          eval_condstore = 1;
+      }
+    }
     mutt_hcache_free ((void **)&uid_validity);
   }
   if (evalhc)
@@ -248,8 +275,12 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms
     mutt_progress_init (&progress, _("Evaluating cache..."),
                        MUTT_PROGRESS_MSG, ReadInc, msn_end);
 
+    /* If we are using CONDSTORE's "FETCH CHANGEDSINCE", then we keep
+     * the flags in the header cache, and update them further below.
+     * Otherwise, we fetch the current state of the flags here. */
     snprintf (buf, sizeof (buf),
-      "UID FETCH 1:%u (UID FLAGS)", uidnext - 1);
+              "UID FETCH 1:%u (UID%s)", uidnext - 1,
+              eval_condstore ? "" : " FLAGS");
 
     imap_cmd_start (idata, buf);
 
@@ -300,18 +331,35 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms
          /* messages which have not been expunged are ACTIVE (borrowed from mh
           * folders) */
          ctx->hdrs[idx]->active = 1;
-          ctx->hdrs[idx]->read = h.data->read;
-          ctx->hdrs[idx]->old = h.data->old;
-          ctx->hdrs[idx]->deleted = h.data->deleted;
-          ctx->hdrs[idx]->flagged = h.data->flagged;
-          ctx->hdrs[idx]->replied = h.data->replied;
-          ctx->hdrs[idx]->changed = h.data->changed;
+          ctx->hdrs[idx]->changed = 0;
+          if (!eval_condstore)
+          {
+            ctx->hdrs[idx]->read = h.data->read;
+            ctx->hdrs[idx]->old = h.data->old;
+            ctx->hdrs[idx]->deleted = h.data->deleted;
+            ctx->hdrs[idx]->flagged = h.data->flagged;
+            ctx->hdrs[idx]->replied = h.data->replied;
+          }
+          else
+          {
+            h.data->read = ctx->hdrs[idx]->read;
+            h.data->old = ctx->hdrs[idx]->old;
+            h.data->deleted = ctx->hdrs[idx]->deleted;
+            h.data->flagged = ctx->hdrs[idx]->flagged;
+            h.data->replied = ctx->hdrs[idx]->replied;
+          }
+
           /*  ctx->hdrs[msgno]->received is restored from mutt_hcache_restore */
           ctx->hdrs[idx]->data = (void *) (h.data);
 
           ctx->msgcount++;
           ctx->size += ctx->hdrs[idx]->content->length;
 
+          /* If this is the first time we are fetching, we need to
+           * store the current state of flags back into the header cache */
+          if (has_condstore && !eval_condstore)
+            imap_hcache_put (idata, ctx->hdrs[idx]);
+
           h.data = NULL;
           idx++;
         }
@@ -327,6 +375,59 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms
       }
     }
 
+    if (eval_condstore && (hc_modseq != idata->modseq))
+    {
+      unsigned int header_msn;
+      char *fetch_buf;
+
+      /* L10N:
+         Fetching IMAP flag changes, using the CONDSTORE extension */
+      mutt_progress_init (&progress, _("Fetching flag updates..."),
+                          MUTT_PROGRESS_MSG, ReadInc, msn_end);
+
+      snprintf (buf, sizeof (buf),
+                "UID FETCH 1:%u (FLAGS) (CHANGEDSINCE %llu)",
+                uidnext - 1, hc_modseq);
+
+      imap_cmd_start (idata, buf);
+
+      rc = IMAP_CMD_CONTINUE;
+      for (msgno = 1; rc == IMAP_CMD_CONTINUE; msgno++)
+      {
+        mutt_progress_update (&progress, msgno, -1);
+
+        /* cmd_parse_fetch will update the flags */
+        rc = imap_cmd_step (idata);
+        if (rc != IMAP_CMD_CONTINUE)
+          break;
+
+        /* so we just need to grab the header and persist it back into
+         * the header cache */
+        fetch_buf = idata->buf;
+        if (fetch_buf[0] != '*')
+          continue;
+
+        fetch_buf = imap_next_word (fetch_buf);
+        if (mutt_atoui (fetch_buf, &header_msn) < 0)
+          continue;
+
+        if (header_msn < 1 || header_msn > msn_end ||
+            !idata->msn_index[header_msn - 1])
+        {
+          dprint (1, (debugfile, "imap_read_headers: skipping CONDSTORE flag "
+                      "update for unknown message number %u\n", header_msn));
+          continue;
+        }
+
+        imap_hcache_put (idata, idata->msn_index[header_msn - 1]);
+      }
+
+      /* The IMAP flag setting as part of cmd_parse_fetch() ends up
+       * flipping these on. */
+      idata->check_status &= ~IMAP_FLAGS_PENDING;
+      ctx->changed = 0;
+    }
+
     /* Look for the first empty MSN and start there */
     while (msn_begin <= msn_end)
     {
@@ -417,12 +518,12 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms
         /* messages which have not been expunged are ACTIVE (borrowed from mh
          * folders) */
         ctx->hdrs[idx]->active = 1;
+        ctx->hdrs[idx]->changed = 0;
         ctx->hdrs[idx]->read = h.data->read;
         ctx->hdrs[idx]->old = h.data->old;
         ctx->hdrs[idx]->deleted = h.data->deleted;
         ctx->hdrs[idx]->flagged = h.data->flagged;
         ctx->hdrs[idx]->replied = h.data->replied;
-        ctx->hdrs[idx]->changed = h.data->changed;
         ctx->hdrs[idx]->received = h.received;
         ctx->hdrs[idx]->data = (void *) (h.data);
 
@@ -495,6 +596,9 @@ int imap_read_headers (IMAP_DATA* idata, unsigned int msn_begin, unsigned int ms
   if (idata->uidnext > 1)
     mutt_hcache_store_raw (idata->hcache, "/UIDNEXT", &idata->uidnext,
                           sizeof (idata->uidnext), imap_hcache_keylen);
+  if (save_modseq)
+    mutt_hcache_store_raw (idata->hcache, "/MODSEQ", &idata->modseq,
+                          sizeof (idata->modseq), imap_hcache_keylen);
 
   imap_hcache_close (idata);
 #endif /* USE_HCACHE */
@@ -1412,6 +1516,28 @@ static int msg_parse_fetch (IMAP_HEADER *h, char *s)
       /* handle above, in msg_fetch_header */
       return -2;
     }
+    else if (ascii_strncasecmp ("MODSEQ", s, 6) == 0)
+    {
+      s += 6;
+      SKIPWS(s);
+      if (*s != '(')
+      {
+        dprint (1, (debugfile, "msg_parse_flags: bogus MODSEQ response: %s\n",
+                    s));
+        return -1;
+      }
+      s++;
+      while (*s && *s != ')')
+        s++;
+      if (*s == ')')
+        s++;
+      else
+      {
+        dprint (1, (debugfile,
+                    "msg_parse_flags: Unterminated MODSEQ response: %s\n", s));
+        return -1;
+      }
+    }
     else if (*s == ')')
       s++; /* end of request */
     else if (*s)

diff --git a/imap/message.h b/imap/message.h
index 3c5d2cb28f54654f55e2208aab020940b7e315c2..864b77464737ca7c780054766c844d43e3546439 100644 (file)
--- a/imap/message.h
+++ b/imap/message.h
@@ -32,7 +32,6 @@ typedef struct imap_header_data
   unsigned int deleted : 1;
   unsigned int flagged : 1;
   unsigned int replied : 1;
-  unsigned int changed : 1;
 
   unsigned int parsed : 1;
 

diff --git a/init.h b/init.h
index c1724c13ebb332cb5e22a5c2ad06688c4fa986bf..30e1c766705b4bc21678f79035299722ce5f6e4b 100644 (file)
--- a/init.h
+++ b/init.h
@@ -1265,6 +1265,14 @@ struct option_t MuttVars[] = {
    ** it polls for new mail just as if you had issued individual ``$mailboxes''
    ** commands.
    */
+  { "imap_condstore",  DT_BOOL, R_NONE, OPTIMAPCONDSTORE, 0 },
+  /*
+   ** .pp
+   **
+   ** When \fIset\fP, mutt will use the CONDSTORE extension (RFC 7162)
+   ** if advertised by the server.  Mutt's current implementation is basic,
+   ** used only for initial message fetching and flag updates.
+   */
   { "imap_delim_chars",                DT_STR, R_NONE, UL &ImapDelimChars, UL "/." },
   /*
   ** .pp

diff --git a/lib.c b/lib.c
index 78e02dfe0379ee4b9313b8ef768ebdc1b483607b..6f7113e1a0c5471721a37e946db5ae571562ef02 100644 (file)
--- a/lib.c
+++ b/lib.c
@@ -1141,3 +1141,31 @@ int mutt_atoul (const char *str, unsigned long *dst)
     return 1;
   return 0;
 }
+
+/* NOTE: this function's return value is different from mutt_atol.
+ *
+ * returns: 1 - successful conversion, with trailing characters
+ *          0 - successful conversion
+ *         -1 - invalid input
+ */
+int mutt_atoull (const char *str, unsigned long long *dst)
+{
+  unsigned long long r;
+  unsigned long long *res = dst ? dst : &r;
+  char *e = NULL;
+
+  /* no input: 0 */
+  if (!str || !*str)
+  {
+    *res = 0;
+    return 0;
+  }
+
+  errno = 0;
+  *res = strtoull (str, &e, 10);
+  if (*res == ULLONG_MAX && errno == ERANGE)
+    return -1;
+  if (e && *e != '\0')
+    return 1;
+  return 0;
+}

diff --git a/lib.h b/lib.h
index 61ac700dee1f0002031c5bd76b9bd898975001a7..11638dfe25a5df89beca56a2c0a3d80fd451d4ca 100644 (file)
--- a/lib.h
+++ b/lib.h
@@ -193,6 +193,7 @@ int mutt_atoi (const char *, int *);
 int mutt_atol (const char *, long *);
 int mutt_atoui (const char *, unsigned int *);
 int mutt_atoul (const char *, unsigned long *);
+int mutt_atoull (const char *, unsigned long long *);
 
 const char *mutt_stristr (const char *, const char *);
 const char *mutt_basename (const char *);

diff --git a/mutt.h b/mutt.h
index 87df6a4a4edc75b71bd3a93fbbb3c44718ab0600..a646519ce342bcee73abc9cfe980e90f751015ff 100644 (file)
--- a/mutt.h
+++ b/mutt.h
@@ -416,6 +416,7 @@ enum
   OPTIGNORELISTREPLYTO,
 #ifdef USE_IMAP
   OPTIMAPCHECKSUBSCRIBED,
+  OPTIMAPCONDSTORE,
   OPTIMAPIDLE,
   OPTIMAPLSUB,
   OPTIMAPPASSIVE,