From: Kevin McCarthy <kevin@8t8.us>
Date: Thu, 24 May 2018 20:26:35 +0000 (-0700)
Subject: Add utility functions for QRESYNC support.
X-Git-Tag: 2019-10-25~665^2~6
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=5f6da4d1297f8c2b7263b703aea26ef0c2b3a773;p=neomutt

Add utility functions for QRESYNC support.

* Add function to generate uid_seqset for header cache.
  This will be used by QREFRESH to reconstruct the msn_index state.

* Add seqset data structure and iterator functions.

* Add cmd_parse_vanished, for QRESYNC support.

* Add and enable QRESYNC capability.

* Create helper functions to store and retrieve the uid_seqset.
---

diff --git a/imap/command.c b/imap/command.c
index b7d4a0af1..22bcc2a2e 100644
--- a/imap/command.c
+++ b/imap/command.c
@@ -72,7 +72,8 @@ static const char *const Capabilities[] = {
   "AUTH=GSSAPI", "AUTH=ANONYMOUS", "AUTH=OAUTHBEARER",
   "STARTTLS",    "LOGINDISABLED",  "IDLE",
   "SASL-IR",     "ENABLE",         "CONDSTORE",
-  "X-GM-EXT-1",  "X-GM-EXT1",      NULL,
+  "QRESYNC",     "X-GM-EXT-1",     "X-GM-EXT1",
+  NULL,
 };
 
 /**
@@ -274,6 +275,92 @@ static void cmd_parse_expunge(struct ImapData *idata, const char *s)
   idata->reopen |= IMAP_EXPUNGE_PENDING;
 }
 
+/* cmd_parse_vanished: handles VANISHED (RFC 7162), which is like
+ *   expunge, but passes a seqset of UIDs.  An optional (EARLIER) argument
+ *   specifies not to decrement subsequent MSNs. */
+static void cmd_parse_vanished(struct ImapData *idata, char *s)
+{
+  int earlier = 0, rc;
+  char *end_of_seqset;
+  struct SeqsetIterator *iter;
+  unsigned int uid, exp_msn, cur;
+  struct Header *h;
+
+  mutt_debug(2, "Handling VANISHED\n");
+
+  if (mutt_str_strncasecmp("(EARLIER)", s, 9) == 0)
+  {
+    earlier = 1;
+    s = imap_next_word(s);
+  }
+
+  end_of_seqset = s;
+  while (*end_of_seqset)
+  {
+    if (!strchr("0123456789:,", *end_of_seqset))
+      *end_of_seqset = '\0';
+    else
+      end_of_seqset++;
+  }
+
+  iter = mutt_seqset_iterator_new(s);
+  if (!iter)
+  {
+    mutt_debug(2, "VANISHED: empty seqset [%s]?\n", s);
+    return;
+  }
+
+  while ((rc = mutt_seqset_iterator_next(iter, &uid)) == 0)
+  {
+    h = (struct Header *) mutt_hash_int_find(idata->uid_hash, uid);
+    if (!h)
+      continue;
+
+    exp_msn = HEADER_DATA(h)->msn;
+
+    /* imap_expunge_mailbox() will rewrite h->index.
+     * It needs to resort using SORT_ORDER anyway, so setting to INT_MAX
+     * makes the code simpler and possibly more efficient. */
+    h->index = INT_MAX;
+    HEADER_DATA(h)->msn = 0;
+
+    if (exp_msn < 1 || exp_msn > idata->max_msn)
+    {
+      mutt_debug(1, "VANISHED: msn for UID %u is incorrect.\n", uid);
+      continue;
+    }
+    if (idata->msn_index[exp_msn - 1] != h)
+    {
+      mutt_debug(1, "VANISHED: msn_index for UID %u is incorrect.\n", uid);
+      continue;
+    }
+
+    idata->msn_index[exp_msn - 1] = NULL;
+
+    if (!earlier)
+    {
+      /* decrement seqno of those above. */
+      for (cur = exp_msn; cur < idata->max_msn; cur++)
+      {
+        h = idata->msn_index[cur];
+        if (h)
+          HEADER_DATA(h)->msn--;
+        idata->msn_index[cur - 1] = h;
+      }
+
+      idata->msn_index[idata->max_msn - 1] = NULL;
+      idata->max_msn--;
+    }
+  }
+
+  if (rc < 0)
+    mutt_debug(1, "VANISHED: illegal seqset %s\n", s);
+
+  idata->reopen |= IMAP_EXPUNGE_PENDING;
+
+  mutt_seqset_iterator_free(&iter);
+}
+
 /**
  * cmd_parse_fetch - Load fetch response into ImapData
  * @param idata Server data
@@ -860,6 +947,8 @@ static void cmd_parse_enabled(struct ImapData *idata, const char *s)
     {
       idata->unicode = 1;
     }
+    if (mutt_str_strncasecmp(s, "QRESYNC", 7) == 0)
+      idata->qresync = 1;
   }
 }
 
@@ -920,6 +1009,8 @@ static int cmd_handle_untagged(struct ImapData *idata)
     else if (mutt_str_strncasecmp("FETCH", s, 5) == 0)
       cmd_parse_fetch(idata, pn);
   }
+  else if ((idata->state >= IMAP_SELECTED) && mutt_str_strncasecmp("VANISHED", s, 8) == 0)
+    cmd_parse_vanished(idata, pn);
   else if (mutt_str_strncasecmp("CAPABILITY", s, 10) == 0)
     cmd_parse_capability(idata, s);
   else if (mutt_str_strncasecmp("OK [CAPABILITY", s, 14) == 0)
diff --git a/imap/imap.c b/imap/imap.c
index c3869ea86..27f0373b7 100644
--- a/imap/imap.c
+++ b/imap/imap.c
@@ -985,12 +985,23 @@ struct ImapData *imap_conn_find(const struct Account *account, int flags)
   {
     /* capabilities may have changed */
     imap_exec(idata, "CAPABILITY", IMAP_CMD_QUEUE);
+
     /* enable RFC6855, if the server supports that */
     if (mutt_bit_isset(idata->capabilities, ENABLE))
       imap_exec(idata, "ENABLE UTF8=ACCEPT", IMAP_CMD_QUEUE);
+
+    /* enable QRESYNC.  Advertising QRESYNC also means CONDSTORE
+     * is supported (even if not advertised), so flip that bit. */
+    if (mutt_bit_isset(idata->capabilities, QRESYNC))
+    {
+      mutt_bit_set(idata->capabilities, CONDSTORE);
+      imap_exec(idata, "ENABLE QRESYNC", IMAP_CMD_QUEUE);
+    }
+
     /* get root delimiter, '/' as default */
     idata->delim = '/';
     imap_exec(idata, "LIST \"\" \"\"", IMAP_CMD_QUEUE);
+
     /* we may need the root delimiter before we open a mailbox */
     imap_exec(idata, NULL, IMAP_CMD_FAIL_OK);
   }
diff --git a/imap/imap_private.h b/imap/imap_private.h
index d9880f39f..02c839932 100644
--- a/imap/imap_private.h
+++ b/imap/imap_private.h
@@ -136,6 +136,7 @@ enum ImapCaps
   SASL_IR,               /**< SASL initial response draft */
   ENABLE,                /**< RFC5161 */
   CONDSTORE,             /**< RFC7162 */
+  QRESYNC,               /**< RFC7162 */
   X_GM_EXT1,             /**< https://developers.google.com/gmail/imap/imap-extensions */
   X_GM_ALT1 = X_GM_EXT1, /**< Alternative capability string */
 
@@ -230,6 +231,8 @@ struct ImapData
    * than mUTF7 */
   int unicode;
 
+  int qresync;  /* Set to 1 if QRESYNC is successfully ENABLE'd */
+
   /* if set, the response parser will store results for complicated commands
    * here. */
   enum ImapCommandType cmdtype;
@@ -269,6 +272,16 @@ struct ImapData
 #endif
 };
 
+struct SeqsetIterator
+{
+  char *full_seqset;
+  char *eostr;
+  int in_range;
+  int down;
+  unsigned int range_cur, range_end;
+  char *substr_cur, *substr_end;
+};
+
 /* -- private IMAP functions -- */
 /* imap.c */
 int imap_check(struct ImapData *idata, bool force);
@@ -318,6 +331,8 @@ void imap_hcache_close(struct ImapData *idata);
 struct Header *imap_hcache_get(struct ImapData *idata, unsigned int uid);
 int imap_hcache_put(struct ImapData *idata, struct Header *h);
 int imap_hcache_del(struct ImapData *idata, unsigned int uid);
+int imap_hcache_store_uid_seqset (struct ImapData *idata);
+char *imap_hcache_get_uid_seqset (struct ImapData *idata);
 #endif
 
 int imap_continue(const char *msg, const char *resp);
@@ -335,6 +350,9 @@ void imap_quote_string(char *dest, size_t dlen, const char *src, bool quote_back
 void imap_unquote_string(char *s);
 void imap_munge_mbox_name(struct ImapData *idata, char *dest, size_t dlen, const char *src);
 void imap_unmunge_mbox_name(struct ImapData *idata, char *s);
+struct SeqsetIterator *mutt_seqset_iterator_new (const char *seqset);
+int mutt_seqset_iterator_next (struct SeqsetIterator *iter, unsigned int *next);
+void mutt_seqset_iterator_free (struct SeqsetIterator **p_iter);
 int imap_account_match(const struct Account *a1, const struct Account *a2);
 void imap_get_parent(const char *mbox, char delim, char *buf, size_t buflen);
 
diff --git a/imap/message.c b/imap/message.c
index be3ebf104..9234e912f 100644
--- a/imap/message.c
+++ b/imap/message.c
@@ -530,8 +530,8 @@ static void alloc_msn_index(struct ImapData *idata, size_t msn_count)
  * Ideally, we would generate multiple requests if the number of ranges
  * is too big, but for now just abort to using the whole range.
  */
-static void generate_seqset(struct Buffer *b, struct ImapData *idata,
-                            unsigned int msn_begin, unsigned int msn_end)
+static void imap_fetch_msn_seqset(struct Buffer *b, struct ImapData *idata,
+                                  unsigned int msn_begin, unsigned int msn_end)
 {
   int chunks = 0;
   int state = 0; /* 1: single msn, 2: range of msn */
@@ -880,7 +880,7 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i
         if (header_msn < 1 || header_msn > msn_end || !idata->msn_index[header_msn - 1])
         {
           mutt_debug(1, "imap_read_headers: skipping CONDSTORE flag update for unknown message number %u\n",
-              header_msn);
+                     header_msn);
           continue;
         }
 
@@ -913,7 +913,7 @@ int imap_read_headers(struct ImapData *idata, unsigned int msn_begin, unsigned i
     {
       /* In case there are holes in the header cache. */
       evalhc = false;
-      generate_seqset(b, idata, msn_begin, msn_end);
+      imap_fetch_msn_seqset(b, idata, msn_begin, msn_end);
     }
     else
       mutt_buffer_printf(b, "%u:%u", msn_begin, msn_end);
diff --git a/imap/util.c b/imap/util.c
index f90a92873..4daa38a96 100644
--- a/imap/util.c
+++ b/imap/util.c
@@ -210,6 +210,63 @@ void imap_clean_path(char *path, size_t plen)
 }
 
 #ifdef USE_HCACHE
+/* Generates a seqseq of the UIDs in msn_index to persist in the header cache.
+ *
+ * Empty spots are stored as 0.
+ */
+static void imap_msn_index_to_uid_seqset(struct Buffer *b, struct ImapData *idata)
+{
+  int first = 1, state = 0, match = 0;
+  struct Header *cur_header;
+  unsigned int msn, cur_uid = 0, last_uid = 0;
+  unsigned int range_begin = 0, range_end = 0;
+
+  for (msn = 1; msn <= idata->max_msn + 1; msn++)
+  {
+    match = 0;
+    if (msn <= idata->max_msn)
+    {
+      cur_header = idata->msn_index[msn - 1];
+      cur_uid = cur_header ? HEADER_DATA(cur_header)->uid : 0;
+      if (!state || (cur_uid && (cur_uid - 1 == last_uid)))
+        match = 1;
+      last_uid = cur_uid;
+    }
+
+    if (match)
+    {
+      switch (state)
+      {
+        case 1: /* single: convert to a range */
+          state = 2;
+          /* fall through */
+        case 2: /* extend range ending */
+          range_end = cur_uid;
+          break;
+        default:
+          state = 1;
+          range_begin = cur_uid;
+          break;
+      }
+    }
+    else if (state)
+    {
+      if (first)
+        first = 0;
+      else
+        mutt_buffer_addch(b, ',');
+
+      if (state == 1)
+        mutt_buffer_printf(b, "%u", range_begin);
+      else if (state == 2)
+        mutt_buffer_printf(b, "%u:%u", range_begin, range_end);
+
+      state = 1;
+      range_begin = cur_uid;
+    }
+  }
+}
+
 /**
  * imap_hcache_namer - Generate a filename for the header cache - Implements ::hcache_namer_t
  */
@@ -334,6 +391,46 @@ int imap_hcache_del(struct ImapData *idata, unsigned int uid)
   sprintf(key, "/%u", uid);
   return mutt_hcache_delete(idata->hcache, key, imap_hcache_keylen(key));
 }
+
+int imap_hcache_store_uid_seqset(struct ImapData *idata)
+{
+  struct Buffer *b;
+  size_t seqset_size;
+  int rc;
+
+  if (!idata->hcache)
+    return -1;
+
+  b = mutt_buffer_new();
+  /* The seqset is likely large.  Preallocate to reduce reallocs */
+  mutt_buffer_increase_size(b, HUGE_STRING);
+  imap_msn_index_to_uid_seqset(b, idata);
+
+  seqset_size = b->dptr - b->data;
+  if (seqset_size == 0)
+    b->data[0] = '\0';
+
+  rc = mutt_hcache_store_raw(idata->hcache, "/UIDSEQSET", 10, b->data,
+                             seqset_size + 1);
+  mutt_debug(5, "Stored /UIDSEQSET %s\n", b->data);
+  mutt_buffer_free(&b);
+  return rc;
+}
+
+char *imap_hcache_get_uid_seqset(struct ImapData *idata)
+{
+  char *hc_seqset, *seqset;
+
+  if (!idata->hcache)
+    return NULL;
+
+  hc_seqset = mutt_hcache_fetch_raw(idata->hcache, "/UIDSEQSET", 10);
+  seqset = mutt_str_strdup(hc_seqset);
+  mutt_hcache_free(idata->hcache, (void **) &hc_seqset);
+  mutt_debug(5, "Retrieved /UIDSEQSET %s\n", NONULL(seqset));
+
+  return seqset;
+}
 #endif
 
 /**
@@ -1025,3 +1122,91 @@ int imap_account_match(const struct Account *a1, const struct Account *a2)
 
   return mutt_account_match(a1_canon, a2_canon);
 }
+
+/* Sequence set iteration */
+
+struct SeqsetIterator *mutt_seqset_iterator_new(const char *seqset)
+{
+  struct SeqsetIterator *iter;
+
+  if (!seqset || !*seqset)
+    return NULL;
+
+  iter = mutt_mem_calloc(1, sizeof(struct SeqsetIterator));
+  iter->full_seqset = mutt_str_strdup(seqset);
+  iter->eostr = strchr(iter->full_seqset, '\0');
+  iter->substr_cur = iter->substr_end = iter->full_seqset;
+
+  return iter;
+}
+
+/* Returns: 0 when the next sequence is generated
+ *          1 when the iterator is finished
+ *         -1 on error
+ */
+int mutt_seqset_iterator_next(struct SeqsetIterator *iter, unsigned int *next)
+{
+  char *range_sep;
+
+  if (!iter || !next)
+    return -1;
+
+  if (iter->in_range)
+  {
+    if ((iter->down && iter->range_cur == (iter->range_end - 1)) ||
+        (!iter->down && iter->range_cur == (iter->range_end + 1)))
+      iter->in_range = 0;
+  }
+
+  if (!iter->in_range)
+  {
+    iter->substr_cur = iter->substr_end;
+    if (iter->substr_cur == iter->eostr)
+      return 1;
+
+    while (!*(iter->substr_cur))
+      iter->substr_cur++;
+    iter->substr_end = strchr(iter->substr_cur, ',');
+    if (!iter->substr_end)
+      iter->substr_end = iter->eostr;
+    else
+      *(iter->substr_end) = '\0';
+
+    range_sep = strchr(iter->substr_cur, ':');
+    if (range_sep)
+      *range_sep++ = '\0';
+
+    if (mutt_str_atoui(iter->substr_cur, &iter->range_cur))
+      return -1;
+    if (range_sep)
+    {
+      if (mutt_str_atoui(range_sep, &iter->range_end))
+        return -1;
+    }
+    else
+      iter->range_end = iter->range_cur;
+
+    iter->down = (iter->range_end < iter->range_cur);
+    iter->in_range = 1;
+  }
+
+  *next = iter->range_cur;
+  if (iter->down)
+    iter->range_cur--;
+  else
+    iter->range_cur++;
+
+  return 0;
+}
+
+void mutt_seqset_iterator_free(struct SeqsetIterator **p_iter)
+{
+  struct SeqsetIterator *iter;
+
+  if (!p_iter || !*p_iter)
+    return;
+
+  iter = *p_iter;
+  FREE(&iter->full_seqset);
+  FREE(p_iter); /* __FREE_CHECKED__ */
+}