From: Richard Russon <rich@flatcap.org>
Date: Tue, 11 Sep 2018 16:49:47 +0000 (+0100)
Subject: reorg nntp functions
X-Git-Tag: 2019-10-25~648^2~14
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=d25df7da2732314f4cd915b59ae2b6cd81868b74;p=neomutt

reorg nntp functions
---

diff --git a/nntp/nntp.c b/nntp/nntp.c
index f81e55f6a..f725b0509 100644
--- a/nntp/nntp.c
+++ b/nntp/nntp.c
@@ -75,6 +75,42 @@ bool ShowNewNews; ///< Config: (nntp) Check for new newsgroups when entering the
 
 struct NntpServer *CurrentNewsSrv;
 
+const char *OverviewFmt = "Subject:\0"
+                          "From:\0"
+                          "Date:\0"
+                          "Message-ID:\0"
+                          "References:\0"
+                          "Content-Length:\0"
+                          "Lines:\0"
+                          "\0";
+
+/**
+ * struct FetchCtx - Keep track when getting data from a server
+ */
+struct FetchCtx
+{
+  struct Context *ctx;
+  anum_t first;
+  anum_t last;
+  int restore;
+  unsigned char *messages;
+  struct Progress progress;
+#ifdef USE_HCACHE
+  header_cache_t *hc;
+#endif
+};
+
+/**
+ * struct ChildCtx - Keep track of the children of an article
+ */
+struct ChildCtx
+{
+  struct Mailbox *mailbox;
+  unsigned int num;
+  unsigned int max;
+  anum_t *child;
+};
+
 /**
  * nntp_connect_error - Signal a failed connection
  * @param nserv NNTP server
@@ -190,15 +226,6 @@ static int nntp_capabilities(struct NntpServer *nserv)
   return -1;
 }
 
-const char *OverviewFmt = "Subject:\0"
-                          "From:\0"
-                          "Date:\0"
-                          "Message-ID:\0"
-                          "References:\0"
-                          "Content-Length:\0"
-                          "Lines:\0"
-                          "\0";
-
 /**
  * nntp_attempt_features - Detect supported commands
  * @param nserv NNTP server
@@ -629,166 +656,6 @@ static int nntp_auth(struct NntpServer *nserv)
   return -1;
 }
 
-/**
- * nntp_open_connection - Connect to server, authenticate and get capabilities
- * @param nserv NNTP server
- * @retval  0 Success
- * @retval -1 Failure
- */
-int nntp_open_connection(struct NntpServer *nserv)
-{
-  struct Connection *conn = nserv->conn;
-  char buf[STRING];
-  int cap;
-  bool posting = false, auth = true;
-
-  if (nserv->status == NNTP_OK)
-    return 0;
-  if (nserv->status == NNTP_BYE)
-    return -1;
-  nserv->status = NNTP_NONE;
-
-  if (mutt_socket_open(conn) < 0)
-    return -1;
-
-  if (mutt_socket_readln(buf, sizeof(buf), conn) < 0)
-    return nntp_connect_error(nserv);
-
-  if (mutt_str_strncmp("200", buf, 3) == 0)
-    posting = true;
-  else if (mutt_str_strncmp("201", buf, 3) != 0)
-  {
-    mutt_socket_close(conn);
-    mutt_str_remove_trailing_ws(buf);
-    mutt_error("%s", buf);
-    return -1;
-  }
-
-  /* get initial capabilities */
-  cap = nntp_capabilities(nserv);
-  if (cap < 0)
-    return -1;
-
-  /* tell news server to switch to mode reader if it isn't so */
-  if (cap > 0)
-  {
-    if (mutt_socket_send(conn, "MODE READER\r\n") < 0 ||
-        mutt_socket_readln(buf, sizeof(buf), conn) < 0)
-    {
-      return nntp_connect_error(nserv);
-    }
-
-    if (mutt_str_strncmp("200", buf, 3) == 0)
-      posting = true;
-    else if (mutt_str_strncmp("201", buf, 3) == 0)
-      posting = false;
-    /* error if has capabilities, ignore result if no capabilities */
-    else if (nserv->hasCAPABILITIES)
-    {
-      mutt_socket_close(conn);
-      mutt_error(_("Could not switch to reader mode"));
-      return -1;
-    }
-
-    /* recheck capabilities after MODE READER */
-    if (nserv->hasCAPABILITIES)
-    {
-      cap = nntp_capabilities(nserv);
-      if (cap < 0)
-        return -1;
-    }
-  }
-
-  mutt_message(_("Connected to %s. %s"), conn->account.host,
-               posting ? _("Posting is ok") : _("Posting is NOT ok"));
-  mutt_sleep(1);
-
-#ifdef USE_SSL
-  /* Attempt STARTTLS if available and desired. */
-  if (nserv->use_tls != 1 && (nserv->hasSTARTTLS || SslForceTls))
-  {
-    if (nserv->use_tls == 0)
-    {
-      nserv->use_tls =
-          SslForceTls || query_quadoption(SslStarttls,
-                                          _("Secure connection with TLS?")) == MUTT_YES ?
-              2 :
-              1;
-    }
-    if (nserv->use_tls == 2)
-    {
-      if (mutt_socket_send(conn, "STARTTLS\r\n") < 0 ||
-          mutt_socket_readln(buf, sizeof(buf), conn) < 0)
-      {
-        return nntp_connect_error(nserv);
-      }
-      if (mutt_str_strncmp("382", buf, 3) != 0)
-      {
-        nserv->use_tls = 0;
-        mutt_error("STARTTLS: %s", buf);
-      }
-      else if (mutt_ssl_starttls(conn))
-      {
-        nserv->use_tls = 0;
-        nserv->status = NNTP_NONE;
-        mutt_socket_close(nserv->conn);
-        mutt_error(_("Could not negotiate TLS connection"));
-        return -1;
-      }
-      else
-      {
-        /* recheck capabilities after STARTTLS */
-        cap = nntp_capabilities(nserv);
-        if (cap < 0)
-          return -1;
-      }
-    }
-  }
-#endif
-
-  /* authentication required? */
-  if (conn->account.flags & MUTT_ACCT_USER)
-  {
-    if (!conn->account.user[0])
-      auth = false;
-  }
-  else
-  {
-    if (mutt_socket_send(conn, "STAT\r\n") < 0 ||
-        mutt_socket_readln(buf, sizeof(buf), conn) < 0)
-    {
-      return nntp_connect_error(nserv);
-    }
-    if (mutt_str_strncmp("480", buf, 3) != 0)
-      auth = false;
-  }
-
-  /* authenticate */
-  if (auth && nntp_auth(nserv) < 0)
-    return -1;
-
-  /* get final capabilities after authentication */
-  if (nserv->hasCAPABILITIES && (auth || cap > 0))
-  {
-    cap = nntp_capabilities(nserv);
-    if (cap < 0)
-      return -1;
-    if (cap > 0)
-    {
-      mutt_socket_close(conn);
-      mutt_error(_("Could not switch to reader mode"));
-      return -1;
-    }
-  }
-
-  /* attempt features */
-  if (nntp_attempt_features(nserv) < 0)
-    return -1;
-
-  nserv->status = NNTP_OK;
-  return 0;
-}
-
 /**
  * nntp_query - Send data from buffer and receive answer to same buffer
  * @param nntp_data NNTP server data
@@ -1075,22 +942,6 @@ static int fetch_tempfile(char *line, void *data)
   return 0;
 }
 
-/**
- * struct FetchCtx - Keep track when getting data from a server
- */
-struct FetchCtx
-{
-  struct Context *ctx;
-  anum_t first;
-  anum_t last;
-  int restore;
-  unsigned char *messages;
-  struct Progress progress;
-#ifdef USE_HCACHE
-  header_cache_t *hc;
-#endif
-};
-
 /**
  * fetch_numbers - Parse article number
  * @param line Article number
@@ -1505,274 +1356,495 @@ static int nntp_fetch_headers(struct Context *ctx, void *hc, anum_t first,
 }
 
 /**
- * nntp_mbox_open - Implements MxOps::mbox_open()
+ * nntp_group_poll - Check newsgroup for new articles
+ * @param nntp_data   NNTP server data
+ * @param update_stat Update the stats?
+ * @retval  1 New articles found
+ * @retval  0 No change
+ * @retval -1 Lost connection
  */
-static int nntp_mbox_open(struct Context *ctx)
+static int nntp_group_poll(struct NntpData *nntp_data, int update_stat)
 {
-  struct NntpServer *nserv = NULL;
-  struct NntpData *nntp_data = NULL;
-  char buf[HUGE_STRING];
-  char server[LONG_STRING];
-  char *group = NULL;
-  int rc;
-  void *hc = NULL;
-  anum_t first, last, count = 0;
-  struct Url url;
+  char buf[LONG_STRING] = "";
+  anum_t count, first, last;
 
-  mutt_str_strfcpy(buf, ctx->mailbox->path, sizeof(buf));
-  if (url_parse(&url, buf) < 0 || !url.host || !url.path ||
-      !(url.scheme == U_NNTP || url.scheme == U_NNTPS))
-  {
-    url_free(&url);
-    mutt_error(_("%s is an invalid newsgroup specification"), ctx->mailbox->path);
+  /* use GROUP command to poll newsgroup */
+  if (nntp_query(nntp_data, buf, sizeof(buf)) < 0)
     return -1;
-  }
+  if (sscanf(buf, "211 " ANUM " " ANUM " " ANUM, &count, &first, &last) != 3)
+    return 0;
+  if (first == nntp_data->first_message && last == nntp_data->last_message)
+    return 0;
 
-  group = url.path;
-  url.path = strchr(url.path, '\0');
-  url_tostring(&url, server, sizeof(server), 0);
-  nserv = nntp_select_server(ctx->mailbox, server, true);
-  url_free(&url);
-  if (!nserv)
-    return -1;
-  CurrentNewsSrv = nserv;
-
-  /* find news group data structure */
-  nntp_data = mutt_hash_find(nserv->groups_hash, group);
-  if (!nntp_data)
+  /* articles have been renumbered */
+  if (last < nntp_data->last_message)
   {
-    nntp_newsrc_close(nserv);
-    mutt_error(_("Newsgroup %s not found on the server"), group);
-    return -1;
+    nntp_data->last_cached = 0;
+    if (nntp_data->newsrc_len)
+    {
+      mutt_mem_realloc(&nntp_data->newsrc_ent, sizeof(struct NewsrcEntry));
+      nntp_data->newsrc_len = 1;
+      nntp_data->newsrc_ent[0].first = 1;
+      nntp_data->newsrc_ent[0].last = 0;
+    }
   }
+  nntp_data->first_message = first;
+  nntp_data->last_message = last;
+  if (!update_stat)
+    return 1;
 
-  mutt_bit_unset(ctx->mailbox->rights, MUTT_ACL_INSERT);
-  if (!nntp_data->newsrc_ent && !nntp_data->subscribed && !SaveUnsubscribed)
-    ctx->mailbox->readonly = true;
+  /* update counters */
+  else if (!last || (!nntp_data->newsrc_ent && !nntp_data->last_cached))
+    nntp_data->unread = count;
+  else
+    nntp_group_unread_stat(nntp_data);
+  return 1;
+}
 
-  /* select newsgroup */
-  mutt_message(_("Selecting %s..."), group);
-  buf[0] = '\0';
-  if (nntp_query(nntp_data, buf, sizeof(buf)) < 0)
+/**
+ * check_mailbox - Check current newsgroup for new articles
+ * @param ctx Mailbox
+ * @retval #MUTT_REOPENED Articles have been renumbered or removed from server
+ * @retval #MUTT_NEW_MAIL New articles found
+ * @retval  0             No change
+ * @retval -1             Lost connection
+ *
+ * Leave newsrc locked
+ */
+static int check_mailbox(struct Context *ctx)
+{
+  struct NntpData *nntp_data = ctx->mailbox->data;
+  struct NntpServer *nserv = nntp_data->nserv;
+  time_t now = time(NULL);
+  int rc, ret = 0;
+  void *hc = NULL;
+
+  if (nserv->check_time + NntpPoll > now)
+    return 0;
+
+  mutt_message(_("Checking for new messages..."));
+  if (nntp_newsrc_parse(nserv) < 0)
+    return -1;
+
+  nserv->check_time = now;
+  rc = nntp_group_poll(nntp_data, 0);
+  if (rc < 0)
   {
     nntp_newsrc_close(nserv);
     return -1;
   }
+  if (rc)
+    nntp_active_save_cache(nserv);
 
-  /* newsgroup not found, remove it */
-  if (mutt_str_strncmp("411", buf, 3) == 0)
+  /* articles have been renumbered, remove all headers */
+  if (nntp_data->last_message < nntp_data->last_loaded)
   {
-    mutt_error(_("Newsgroup %s has been removed from the server"), nntp_data->group);
-    if (!nntp_data->deleted)
-    {
-      nntp_data->deleted = true;
-      nntp_active_save_cache(nserv);
-    }
-    if (nntp_data->newsrc_ent && !nntp_data->subscribed && !SaveUnsubscribed)
+    for (int i = 0; i < ctx->mailbox->msg_count; i++)
+      mutt_header_free(&ctx->mailbox->hdrs[i]);
+    ctx->mailbox->msg_count = 0;
+    ctx->tagged = 0;
+
+    if (nntp_data->last_message < nntp_data->last_loaded)
     {
-      FREE(&nntp_data->newsrc_ent);
-      nntp_data->newsrc_len = 0;
-      nntp_delete_group_cache(nntp_data);
-      nntp_newsrc_update(nserv);
+      nntp_data->last_loaded = nntp_data->first_message - 1;
+      if (NntpContext && nntp_data->last_message - nntp_data->last_loaded > NntpContext)
+        nntp_data->last_loaded = nntp_data->last_message - NntpContext;
     }
+    ret = MUTT_REOPENED;
   }
 
-  /* parse newsgroup info */
-  else
+  /* .newsrc has been externally modified */
+  if (nserv->newsrc_modified)
   {
-    if (sscanf(buf, "211 " ANUM " " ANUM " " ANUM, &count, &first, &last) != 3)
+#ifdef USE_HCACHE
+    unsigned char *messages = NULL;
+    char buf[16];
+    void *hdata = NULL;
+    struct Header *hdr = NULL;
+    anum_t first = nntp_data->first_message;
+
+    if (NntpContext && nntp_data->last_message - first + 1 > NntpContext)
+      first = nntp_data->last_message - NntpContext + 1;
+    messages = mutt_mem_calloc(nntp_data->last_loaded - first + 1, sizeof(unsigned char));
+    hc = nntp_hcache_open(nntp_data);
+    nntp_hcache_update(nntp_data, hc);
+#endif
+
+    /* update flags according to .newsrc */
+    int j = 0;
+    anum_t anum;
+    for (int i = 0; i < ctx->mailbox->msg_count; i++)
     {
-      nntp_newsrc_close(nserv);
-      mutt_error("GROUP: %s", buf);
-      return -1;
+      bool flagged = false;
+      anum = NHDR(ctx->mailbox->hdrs[i])->article_num;
+
+#ifdef USE_HCACHE
+      /* check hcache for flagged and deleted flags */
+      if (hc)
+      {
+        if (anum >= first && anum <= nntp_data->last_loaded)
+          messages[anum - first] = 1;
+
+        snprintf(buf, sizeof(buf), "%u", anum);
+        hdata = mutt_hcache_fetch(hc, buf, strlen(buf));
+        if (hdata)
+        {
+          bool deleted;
+
+          mutt_debug(2, "#1 mutt_hcache_fetch %s\n", buf);
+          hdr = mutt_hcache_restore(hdata);
+          mutt_hcache_free(hc, &hdata);
+          hdr->data = NULL;
+          deleted = hdr->deleted;
+          flagged = hdr->flagged;
+          mutt_header_free(&hdr);
+
+          /* header marked as deleted, removing from context */
+          if (deleted)
+          {
+            mutt_set_flag(ctx, ctx->mailbox->hdrs[i], MUTT_TAG, 0);
+            mutt_header_free(&ctx->mailbox->hdrs[i]);
+            continue;
+          }
+        }
+      }
+#endif
+
+      if (!ctx->mailbox->hdrs[i]->changed)
+      {
+        ctx->mailbox->hdrs[i]->flagged = flagged;
+        ctx->mailbox->hdrs[i]->read = false;
+        ctx->mailbox->hdrs[i]->old = false;
+        nntp_article_status(ctx->mailbox, ctx->mailbox->hdrs[i], NULL, anum);
+        if (!ctx->mailbox->hdrs[i]->read)
+          nntp_parse_xref(ctx->mailbox, ctx->mailbox->hdrs[i]);
+      }
+      ctx->mailbox->hdrs[j++] = ctx->mailbox->hdrs[i];
     }
-    nntp_data->first_message = first;
-    nntp_data->last_message = last;
-    nntp_data->deleted = false;
 
-    /* get description if empty */
-    if (NntpLoadDescription && !nntp_data->desc)
+#ifdef USE_HCACHE
+    ctx->mailbox->msg_count = j;
+
+    /* restore headers without "deleted" flag */
+    for (anum = first; anum <= nntp_data->last_loaded; anum++)
     {
-      if (get_description(nntp_data, NULL, NULL) < 0)
+      if (messages[anum - first])
+        continue;
+
+      snprintf(buf, sizeof(buf), "%u", anum);
+      hdata = mutt_hcache_fetch(hc, buf, strlen(buf));
+      if (hdata)
       {
-        nntp_newsrc_close(nserv);
-        return -1;
+        mutt_debug(2, "#2 mutt_hcache_fetch %s\n", buf);
+        if (ctx->mailbox->msg_count >= ctx->mailbox->hdrmax)
+          mx_alloc_memory(ctx->mailbox);
+
+        hdr = mutt_hcache_restore(hdata);
+        ctx->mailbox->hdrs[ctx->mailbox->msg_count] = hdr;
+        mutt_hcache_free(hc, &hdata);
+        hdr->data = NULL;
+        if (hdr->deleted)
+        {
+          mutt_header_free(&hdr);
+          if (nntp_data->bcache)
+          {
+            mutt_debug(2, "mutt_bcache_del %s\n", buf);
+            mutt_bcache_del(nntp_data->bcache, buf);
+          }
+          continue;
+        }
+
+        ctx->mailbox->msg_count++;
+        hdr->read = false;
+        hdr->old = false;
+        hdr->data = mutt_mem_calloc(1, sizeof(struct NntpHeaderData));
+        NHDR(hdr)->article_num = anum;
+        nntp_article_status(ctx->mailbox, hdr, NULL, anum);
+        if (!hdr->read)
+          nntp_parse_xref(ctx->mailbox, hdr);
       }
-      if (nntp_data->desc)
-        nntp_active_save_cache(nserv);
     }
+    FREE(&messages);
+#endif
+
+    nserv->newsrc_modified = false;
+    ret = MUTT_REOPENED;
   }
 
-  time(&nserv->check_time);
-  ctx->mailbox->data = nntp_data;
-  if (!nntp_data->bcache && (nntp_data->newsrc_ent || nntp_data->subscribed || SaveUnsubscribed))
-    nntp_data->bcache = mutt_bcache_open(&nserv->conn->account, nntp_data->group);
+  /* some headers were removed, context must be updated */
+  if (ret == MUTT_REOPENED)
+  {
+    if (ctx->mailbox->subj_hash)
+      mutt_hash_destroy(&ctx->mailbox->subj_hash);
+    if (ctx->mailbox->id_hash)
+      mutt_hash_destroy(&ctx->mailbox->id_hash);
+    mutt_clear_threads(ctx);
 
-  /* strip off extra articles if adding context is greater than $nntp_context */
-  first = nntp_data->first_message;
-  if (NntpContext && nntp_data->last_message - first + 1 > NntpContext)
-    first = nntp_data->last_message - NntpContext + 1;
-  nntp_data->last_loaded = first ? first - 1 : 0;
-  count = nntp_data->first_message;
-  nntp_data->first_message = first;
-  nntp_bcache_update(nntp_data);
-  nntp_data->first_message = count;
+    ctx->mailbox->vcount = 0;
+    ctx->deleted = 0;
+    ctx->new = 0;
+    ctx->mailbox->msg_unread = 0;
+    ctx->mailbox->msg_flagged = 0;
+    ctx->mailbox->changed = false;
+    ctx->mailbox->id_hash = NULL;
+    ctx->mailbox->subj_hash = NULL;
+    mx_update_context(ctx, ctx->mailbox->msg_count);
+  }
+
+  /* fetch headers of new articles */
+  if (nntp_data->last_message > nntp_data->last_loaded)
+  {
+    int oldmsgcount = ctx->mailbox->msg_count;
+    bool quiet = ctx->mailbox->quiet;
+    ctx->mailbox->quiet = true;
 #ifdef USE_HCACHE
-  hc = nntp_hcache_open(nntp_data);
-  nntp_hcache_update(nntp_data, hc);
+    if (!hc)
+    {
+      hc = nntp_hcache_open(nntp_data);
+      nntp_hcache_update(nntp_data, hc);
+    }
 #endif
-  if (!hc)
-  {
-    mutt_bit_unset(ctx->mailbox->rights, MUTT_ACL_WRITE);
-    mutt_bit_unset(ctx->mailbox->rights, MUTT_ACL_DELETE);
+    rc = nntp_fetch_headers(ctx, hc, nntp_data->last_loaded + 1, nntp_data->last_message, 0);
+    ctx->mailbox->quiet = quiet;
+    if (rc >= 0)
+      nntp_data->last_loaded = nntp_data->last_message;
+    if (ret == 0 && ctx->mailbox->msg_count > oldmsgcount)
+      ret = MUTT_NEW_MAIL;
   }
-  nntp_newsrc_close(nserv);
-  rc = nntp_fetch_headers(ctx, hc, first, nntp_data->last_message, 0);
+
 #ifdef USE_HCACHE
   mutt_hcache_close(hc);
 #endif
-  if (rc < 0)
-    return -1;
-  nntp_data->last_loaded = nntp_data->last_message;
-  nserv->newsrc_modified = false;
-  return 0;
+  if (ret)
+    nntp_newsrc_close(nserv);
+  mutt_clear_error();
+  return ret;
 }
 
 /**
- * nntp_msg_open - Implements MxOps::msg_open()
+ * nntp_date - Get date and time from server
+ * @param nserv NNTP server
+ * @param now   Server time
+ * @retval  0 Success
+ * @retval -1 Failure
  */
-static int nntp_msg_open(struct Context *ctx, struct Message *msg, int msgno)
+static int nntp_date(struct NntpServer *nserv, time_t *now)
 {
-  struct NntpData *nntp_data = ctx->mailbox->data;
-  struct Header *hdr = ctx->mailbox->hdrs[msgno];
-  char article[16];
-
-  /* try to get article from cache */
-  struct NntpAcache *acache = &nntp_data->acache[hdr->index % NNTP_ACACHE_LEN];
-  if (acache->path)
+  if (nserv->hasDATE)
   {
-    if (acache->index == hdr->index)
+    struct NntpData nntp_data;
+    char buf[LONG_STRING];
+    struct tm tm;
+    memset(&tm, 0, sizeof(tm));
+
+    nntp_data.nserv = nserv;
+    nntp_data.group = NULL;
+    mutt_str_strfcpy(buf, "DATE\r\n", sizeof(buf));
+    if (nntp_query(&nntp_data, buf, sizeof(buf)) < 0)
+      return -1;
+
+    if (sscanf(buf, "111 %4d%2d%2d%2d%2d%2d%*s", &tm.tm_year, &tm.tm_mon,
+               &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec) == 6)
     {
-      msg->fp = mutt_file_fopen(acache->path, "r");
-      if (msg->fp)
+      tm.tm_year -= 1900;
+      tm.tm_mon--;
+      *now = timegm(&tm);
+      if (*now >= 0)
+      {
+        mutt_debug(1, "server time is %lu\n", *now);
         return 0;
-    }
-    /* clear previous entry */
-    else
-    {
-      unlink(acache->path);
-      FREE(&acache->path);
+      }
     }
   }
-  snprintf(article, sizeof(article), "%d", NHDR(hdr)->article_num);
-  msg->fp = mutt_bcache_get(nntp_data->bcache, article);
-  if (msg->fp)
-  {
-    if (NHDR(hdr)->parsed)
+  time(now);
+  return 0;
+}
+
+/**
+ * fetch_children - Parse XPAT line
+ * @param line String to parse
+ * @param data ChildCtx
+ * @retval 0 Always
+ */
+static int fetch_children(char *line, void *data)
+{
+  struct ChildCtx *cc = data;
+  anum_t anum;
+
+  if (!line || sscanf(line, ANUM, &anum) != 1)
+    return 0;
+  for (unsigned int i = 0; i < cc->mailbox->msg_count; i++)
+    if (NHDR(cc->mailbox->hdrs[i])->article_num == anum)
       return 0;
+  if (cc->num >= cc->max)
+  {
+    cc->max *= 2;
+    mutt_mem_realloc(&cc->child, sizeof(anum_t) * cc->max);
   }
-  else
+  cc->child[cc->num++] = anum;
+  return 0;
+}
+
+/**
+ * nntp_open_connection - Connect to server, authenticate and get capabilities
+ * @param nserv NNTP server
+ * @retval  0 Success
+ * @retval -1 Failure
+ */
+int nntp_open_connection(struct NntpServer *nserv)
+{
+  struct Connection *conn = nserv->conn;
+  char buf[STRING];
+  int cap;
+  bool posting = false, auth = true;
+
+  if (nserv->status == NNTP_OK)
+    return 0;
+  if (nserv->status == NNTP_BYE)
+    return -1;
+  nserv->status = NNTP_NONE;
+
+  if (mutt_socket_open(conn) < 0)
+    return -1;
+
+  if (mutt_socket_readln(buf, sizeof(buf), conn) < 0)
+    return nntp_connect_error(nserv);
+
+  if (mutt_str_strncmp("200", buf, 3) == 0)
+    posting = true;
+  else if (mutt_str_strncmp("201", buf, 3) != 0)
   {
-    char buf[PATH_MAX];
-    /* don't try to fetch article from removed newsgroup */
-    if (nntp_data->deleted)
+    mutt_socket_close(conn);
+    mutt_str_remove_trailing_ws(buf);
+    mutt_error("%s", buf);
+    return -1;
+  }
+
+  /* get initial capabilities */
+  cap = nntp_capabilities(nserv);
+  if (cap < 0)
+    return -1;
+
+  /* tell news server to switch to mode reader if it isn't so */
+  if (cap > 0)
+  {
+    if (mutt_socket_send(conn, "MODE READER\r\n") < 0 ||
+        mutt_socket_readln(buf, sizeof(buf), conn) < 0)
+    {
+      return nntp_connect_error(nserv);
+    }
+
+    if (mutt_str_strncmp("200", buf, 3) == 0)
+      posting = true;
+    else if (mutt_str_strncmp("201", buf, 3) == 0)
+      posting = false;
+    /* error if has capabilities, ignore result if no capabilities */
+    else if (nserv->hasCAPABILITIES)
+    {
+      mutt_socket_close(conn);
+      mutt_error(_("Could not switch to reader mode"));
       return -1;
+    }
 
-    /* create new cache file */
-    const char *fetch_msg = _("Fetching message...");
-    mutt_message(fetch_msg);
-    msg->fp = mutt_bcache_put(nntp_data->bcache, article);
-    if (!msg->fp)
+    /* recheck capabilities after MODE READER */
+    if (nserv->hasCAPABILITIES)
     {
-      mutt_mktemp(buf, sizeof(buf));
-      acache->path = mutt_str_strdup(buf);
-      acache->index = hdr->index;
-      msg->fp = mutt_file_fopen(acache->path, "w+");
-      if (!msg->fp)
-      {
-        mutt_perror(acache->path);
-        unlink(acache->path);
-        FREE(&acache->path);
+      cap = nntp_capabilities(nserv);
+      if (cap < 0)
         return -1;
+    }
+  }
+
+  mutt_message(_("Connected to %s. %s"), conn->account.host,
+               posting ? _("Posting is ok") : _("Posting is NOT ok"));
+  mutt_sleep(1);
+
+#ifdef USE_SSL
+  /* Attempt STARTTLS if available and desired. */
+  if (nserv->use_tls != 1 && (nserv->hasSTARTTLS || SslForceTls))
+  {
+    if (nserv->use_tls == 0)
+    {
+      nserv->use_tls =
+          SslForceTls || query_quadoption(SslStarttls,
+                                          _("Secure connection with TLS?")) == MUTT_YES ?
+              2 :
+              1;
+    }
+    if (nserv->use_tls == 2)
+    {
+      if (mutt_socket_send(conn, "STARTTLS\r\n") < 0 ||
+          mutt_socket_readln(buf, sizeof(buf), conn) < 0)
+      {
+        return nntp_connect_error(nserv);
+      }
+      if (mutt_str_strncmp("382", buf, 3) != 0)
+      {
+        nserv->use_tls = 0;
+        mutt_error("STARTTLS: %s", buf);
       }
-    }
-
-    /* fetch message to cache file */
-    snprintf(buf, sizeof(buf), "ARTICLE %s\r\n",
-             NHDR(hdr)->article_num ? article : hdr->env->message_id);
-    const int rc = nntp_fetch_lines(nntp_data, buf, sizeof(buf), fetch_msg,
-                                    fetch_tempfile, msg->fp);
-    if (rc)
-    {
-      mutt_file_fclose(&msg->fp);
-      if (acache->path)
+      else if (mutt_ssl_starttls(conn))
       {
-        unlink(acache->path);
-        FREE(&acache->path);
+        nserv->use_tls = 0;
+        nserv->status = NNTP_NONE;
+        mutt_socket_close(nserv->conn);
+        mutt_error(_("Could not negotiate TLS connection"));
+        return -1;
       }
-      if (rc > 0)
+      else
       {
-        if (mutt_str_strncmp(NHDR(hdr)->article_num ? "423" : "430", buf, 3) == 0)
-        {
-          mutt_error(_("Article %d not found on the server"),
-                     NHDR(hdr)->article_num ? article : hdr->env->message_id);
-        }
-        else
-          mutt_error("ARTICLE: %s", buf);
+        /* recheck capabilities after STARTTLS */
+        cap = nntp_capabilities(nserv);
+        if (cap < 0)
+          return -1;
       }
-      return -1;
     }
-
-    if (!acache->path)
-      mutt_bcache_commit(nntp_data->bcache, article);
   }
+#endif
 
-  /* replace envelope with new one
-   * hash elements must be updated because pointers will be changed */
-  if (ctx->mailbox->id_hash && hdr->env->message_id)
-    mutt_hash_delete(ctx->mailbox->id_hash, hdr->env->message_id, hdr);
-  if (ctx->mailbox->subj_hash && hdr->env->real_subj)
-    mutt_hash_delete(ctx->mailbox->subj_hash, hdr->env->real_subj, hdr);
-
-  mutt_env_free(&hdr->env);
-  hdr->env = mutt_rfc822_read_header(msg->fp, hdr, false, false);
-
-  if (ctx->mailbox->id_hash && hdr->env->message_id)
-    mutt_hash_insert(ctx->mailbox->id_hash, hdr->env->message_id, hdr);
-  if (ctx->mailbox->subj_hash && hdr->env->real_subj)
-    mutt_hash_insert(ctx->mailbox->subj_hash, hdr->env->real_subj, hdr);
+  /* authentication required? */
+  if (conn->account.flags & MUTT_ACCT_USER)
+  {
+    if (!conn->account.user[0])
+      auth = false;
+  }
+  else
+  {
+    if (mutt_socket_send(conn, "STAT\r\n") < 0 ||
+        mutt_socket_readln(buf, sizeof(buf), conn) < 0)
+    {
+      return nntp_connect_error(nserv);
+    }
+    if (mutt_str_strncmp("480", buf, 3) != 0)
+      auth = false;
+  }
 
-  /* fix content length */
-  fseek(msg->fp, 0, SEEK_END);
-  hdr->content->length = ftell(msg->fp) - hdr->content->offset;
+  /* authenticate */
+  if (auth && nntp_auth(nserv) < 0)
+    return -1;
 
-  /* this is called in neomutt before the open which fetches the message,
-   * which is probably wrong, but we just call it again here to handle
-   * the problem instead of fixing it */
-  NHDR(hdr)->parsed = true;
-  mutt_parse_mime_message(ctx, hdr);
+  /* get final capabilities after authentication */
+  if (nserv->hasCAPABILITIES && (auth || cap > 0))
+  {
+    cap = nntp_capabilities(nserv);
+    if (cap < 0)
+      return -1;
+    if (cap > 0)
+    {
+      mutt_socket_close(conn);
+      mutt_error(_("Could not switch to reader mode"));
+      return -1;
+    }
+  }
 
-  /* these would normally be updated in mx_update_context(), but the
-   * full headers aren't parsed with overview, so the information wasn't
-   * available then */
-  if (WithCrypto)
-    hdr->security = crypt_query(hdr->content);
+  /* attempt features */
+  if (nntp_attempt_features(nserv) < 0)
+    return -1;
 
-  rewind(msg->fp);
-  mutt_clear_error();
+  nserv->status = NNTP_OK;
   return 0;
 }
 
-/**
- * nntp_msg_close - Implements MxOps::msg_close()
- *
- * @note May also return EOF Failure, see errno
- */
-static int nntp_msg_close(struct Context *ctx, struct Message *msg)
-{
-  return mutt_file_fclose(&msg->fp);
-}
-
 /**
  * nntp_post - Post article
  * @param mailbox Mailbox
@@ -1856,777 +1928,705 @@ int nntp_post(struct Mailbox *mailbox, const char *msg)
 }
 
 /**
- * nntp_group_poll - Check newsgroup for new articles
- * @param nntp_data   NNTP server data
- * @param update_stat Update the stats?
- * @retval  1 New articles found
- * @retval  0 No change
- * @retval -1 Lost connection
+ * nntp_active_fetch - Fetch list of all newsgroups from server
+ * @param nserv NNTP server
+ * @param new   Mark the groups as new
+ * @retval  0 Success
+ * @retval -1 Failure
  */
-static int nntp_group_poll(struct NntpData *nntp_data, int update_stat)
+int nntp_active_fetch(struct NntpServer *nserv, bool new)
 {
-  char buf[LONG_STRING] = "";
-  anum_t count, first, last;
+  struct NntpData nntp_data;
+  char msg[STRING];
+  char buf[LONG_STRING];
+  unsigned int i;
+  int rc;
 
-  /* use GROUP command to poll newsgroup */
-  if (nntp_query(nntp_data, buf, sizeof(buf)) < 0)
+  snprintf(msg, sizeof(msg), _("Loading list of groups from server %s..."),
+           nserv->conn->account.host);
+  mutt_message(msg);
+  if (nntp_date(nserv, &nserv->newgroups_time) < 0)
     return -1;
-  if (sscanf(buf, "211 " ANUM " " ANUM " " ANUM, &count, &first, &last) != 3)
-    return 0;
-  if (first == nntp_data->first_message && last == nntp_data->last_message)
-    return 0;
 
-  /* articles have been renumbered */
-  if (last < nntp_data->last_message)
+  nntp_data.nserv = nserv;
+  nntp_data.group = NULL;
+  i = nserv->groups_num;
+  mutt_str_strfcpy(buf, "LIST\r\n", sizeof(buf));
+  rc = nntp_fetch_lines(&nntp_data, buf, sizeof(buf), msg, nntp_add_group, nserv);
+  if (rc)
   {
-    nntp_data->last_cached = 0;
-    if (nntp_data->newsrc_len)
+    if (rc > 0)
     {
-      mutt_mem_realloc(&nntp_data->newsrc_ent, sizeof(struct NewsrcEntry));
-      nntp_data->newsrc_len = 1;
-      nntp_data->newsrc_ent[0].first = 1;
-      nntp_data->newsrc_ent[0].last = 0;
+      mutt_error("LIST: %s", buf);
     }
+    return -1;
   }
-  nntp_data->first_message = first;
-  nntp_data->last_message = last;
-  if (!update_stat)
-    return 1;
 
-  /* update counters */
-  else if (!last || (!nntp_data->newsrc_ent && !nntp_data->last_cached))
-    nntp_data->unread = count;
-  else
-    nntp_group_unread_stat(nntp_data);
-  return 1;
+  if (new)
+  {
+    for (; i < nserv->groups_num; i++)
+    {
+      struct NntpData *data = nserv->groups_list[i];
+      data->new = true;
+    }
+  }
+
+  for (i = 0; i < nserv->groups_num; i++)
+  {
+    struct NntpData *data = nserv->groups_list[i];
+
+    if (data && data->deleted && !data->newsrc_ent)
+    {
+      nntp_delete_group_cache(data);
+      mutt_hash_delete(nserv->groups_hash, data->group, NULL);
+      nserv->groups_list[i] = NULL;
+    }
+  }
+
+  if (NntpLoadDescription)
+    rc = get_description(&nntp_data, "*", _("Loading descriptions..."));
+
+  nntp_active_save_cache(nserv);
+  if (rc < 0)
+    return -1;
+  mutt_clear_error();
+  return 0;
 }
 
 /**
- * check_mailbox - Check current newsgroup for new articles
- * @param ctx Mailbox
- * @retval #MUTT_REOPENED Articles have been renumbered or removed from server
- * @retval #MUTT_NEW_MAIL New articles found
- * @retval  0             No change
- * @retval -1             Lost connection
- *
- * Leave newsrc locked
+ * nntp_check_new_groups - Check for new groups/articles in subscribed groups
+ * @param mailbox Mailbox
+ * @param nserv   NNTP server
+ * @retval  1 New groups found
+ * @retval  0 No new groups
+ * @retval -1 Error
  */
-static int check_mailbox(struct Context *ctx)
+int nntp_check_new_groups(struct Mailbox *mailbox, struct NntpServer *nserv)
 {
-  struct NntpData *nntp_data = ctx->mailbox->data;
-  struct NntpServer *nserv = nntp_data->nserv;
-  time_t now = time(NULL);
-  int rc, ret = 0;
-  void *hc = NULL;
+  struct NntpData nntp_data;
+  time_t now;
+  struct tm *tm = NULL;
+  char buf[LONG_STRING];
+  char *msg = _("Checking for new newsgroups...");
+  unsigned int i;
+  int rc, update_active = false;
 
-  if (nserv->check_time + NntpPoll > now)
+  if (!nserv || !nserv->newgroups_time)
+    return -1;
+
+  /* check subscribed newsgroups for new articles */
+  if (ShowNewNews)
+  {
+    mutt_message(_("Checking for new messages..."));
+    for (i = 0; i < nserv->groups_num; i++)
+    {
+      struct NntpData *data = nserv->groups_list[i];
+
+      if (data && data->subscribed)
+      {
+        rc = nntp_group_poll(data, 1);
+        if (rc < 0)
+          return -1;
+        if (rc > 0)
+          update_active = true;
+      }
+    }
+    /* select current newsgroup */
+    if (mailbox && (mailbox->magic == MUTT_NNTP))
+    {
+      buf[0] = '\0';
+      if (nntp_query(mailbox->data, buf, sizeof(buf)) < 0)
+        return -1;
+    }
+  }
+  else if (nserv->newgroups_time)
     return 0;
 
-  mutt_message(_("Checking for new messages..."));
-  if (nntp_newsrc_parse(nserv) < 0)
-    return -1;
-
-  nserv->check_time = now;
-  rc = nntp_group_poll(nntp_data, 0);
-  if (rc < 0)
-  {
-    nntp_newsrc_close(nserv);
+  /* get list of new groups */
+  mutt_message(msg);
+  if (nntp_date(nserv, &now) < 0)
     return -1;
-  }
+  nntp_data.nserv = nserv;
+  if (mailbox && (mailbox->magic == MUTT_NNTP))
+    nntp_data.group = ((struct NntpData *) mailbox->data)->group;
+  else
+    nntp_data.group = NULL;
+  i = nserv->groups_num;
+  tm = gmtime(&nserv->newgroups_time);
+  snprintf(buf, sizeof(buf), "NEWGROUPS %02d%02d%02d %02d%02d%02d GMT\r\n",
+           tm->tm_year % 100, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour,
+           tm->tm_min, tm->tm_sec);
+  rc = nntp_fetch_lines(&nntp_data, buf, sizeof(buf), msg, nntp_add_group, nserv);
   if (rc)
-    nntp_active_save_cache(nserv);
-
-  /* articles have been renumbered, remove all headers */
-  if (nntp_data->last_message < nntp_data->last_loaded)
   {
-    for (int i = 0; i < ctx->mailbox->msg_count; i++)
-      mutt_header_free(&ctx->mailbox->hdrs[i]);
-    ctx->mailbox->msg_count = 0;
-    ctx->tagged = 0;
-
-    if (nntp_data->last_message < nntp_data->last_loaded)
+    if (rc > 0)
     {
-      nntp_data->last_loaded = nntp_data->first_message - 1;
-      if (NntpContext && nntp_data->last_message - nntp_data->last_loaded > NntpContext)
-        nntp_data->last_loaded = nntp_data->last_message - NntpContext;
+      mutt_error("NEWGROUPS: %s", buf);
     }
-    ret = MUTT_REOPENED;
+    return -1;
   }
 
-  /* .newsrc has been externally modified */
-  if (nserv->newsrc_modified)
+  /* new groups found */
+  rc = 0;
+  if (nserv->groups_num != i)
   {
-#ifdef USE_HCACHE
-    unsigned char *messages = NULL;
-    char buf[16];
-    void *hdata = NULL;
-    struct Header *hdr = NULL;
-    anum_t first = nntp_data->first_message;
-
-    if (NntpContext && nntp_data->last_message - first + 1 > NntpContext)
-      first = nntp_data->last_message - NntpContext + 1;
-    messages = mutt_mem_calloc(nntp_data->last_loaded - first + 1, sizeof(unsigned char));
-    hc = nntp_hcache_open(nntp_data);
-    nntp_hcache_update(nntp_data, hc);
-#endif
+    int groups_num = i;
 
-    /* update flags according to .newsrc */
-    int j = 0;
-    anum_t anum;
-    for (int i = 0; i < ctx->mailbox->msg_count; i++)
+    nserv->newgroups_time = now;
+    for (; i < nserv->groups_num; i++)
     {
-      bool flagged = false;
-      anum = NHDR(ctx->mailbox->hdrs[i])->article_num;
-
-#ifdef USE_HCACHE
-      /* check hcache for flagged and deleted flags */
-      if (hc)
-      {
-        if (anum >= first && anum <= nntp_data->last_loaded)
-          messages[anum - first] = 1;
-
-        snprintf(buf, sizeof(buf), "%u", anum);
-        hdata = mutt_hcache_fetch(hc, buf, strlen(buf));
-        if (hdata)
-        {
-          bool deleted;
-
-          mutt_debug(2, "#1 mutt_hcache_fetch %s\n", buf);
-          hdr = mutt_hcache_restore(hdata);
-          mutt_hcache_free(hc, &hdata);
-          hdr->data = NULL;
-          deleted = hdr->deleted;
-          flagged = hdr->flagged;
-          mutt_header_free(&hdr);
-
-          /* header marked as deleted, removing from context */
-          if (deleted)
-          {
-            mutt_set_flag(ctx, ctx->mailbox->hdrs[i], MUTT_TAG, 0);
-            mutt_header_free(&ctx->mailbox->hdrs[i]);
-            continue;
-          }
-        }
-      }
-#endif
-
-      if (!ctx->mailbox->hdrs[i]->changed)
-      {
-        ctx->mailbox->hdrs[i]->flagged = flagged;
-        ctx->mailbox->hdrs[i]->read = false;
-        ctx->mailbox->hdrs[i]->old = false;
-        nntp_article_status(ctx->mailbox, ctx->mailbox->hdrs[i], NULL, anum);
-        if (!ctx->mailbox->hdrs[i]->read)
-          nntp_parse_xref(ctx->mailbox, ctx->mailbox->hdrs[i]);
-      }
-      ctx->mailbox->hdrs[j++] = ctx->mailbox->hdrs[i];
+      struct NntpData *data = nserv->groups_list[i];
+      data->new = true;
     }
 
-#ifdef USE_HCACHE
-    ctx->mailbox->msg_count = j;
-
-    /* restore headers without "deleted" flag */
-    for (anum = first; anum <= nntp_data->last_loaded; anum++)
+    /* loading descriptions */
+    if (NntpLoadDescription)
     {
-      if (messages[anum - first])
-        continue;
+      unsigned int count = 0;
+      struct Progress progress;
 
-      snprintf(buf, sizeof(buf), "%u", anum);
-      hdata = mutt_hcache_fetch(hc, buf, strlen(buf));
-      if (hdata)
+      mutt_progress_init(&progress, _("Loading descriptions..."),
+                         MUTT_PROGRESS_MSG, ReadInc, nserv->groups_num - i);
+      for (i = groups_num; i < nserv->groups_num; i++)
       {
-        mutt_debug(2, "#2 mutt_hcache_fetch %s\n", buf);
-        if (ctx->mailbox->msg_count >= ctx->mailbox->hdrmax)
-          mx_alloc_memory(ctx->mailbox);
-
-        hdr = mutt_hcache_restore(hdata);
-        ctx->mailbox->hdrs[ctx->mailbox->msg_count] = hdr;
-        mutt_hcache_free(hc, &hdata);
-        hdr->data = NULL;
-        if (hdr->deleted)
-        {
-          mutt_header_free(&hdr);
-          if (nntp_data->bcache)
-          {
-            mutt_debug(2, "mutt_bcache_del %s\n", buf);
-            mutt_bcache_del(nntp_data->bcache, buf);
-          }
-          continue;
-        }
+        struct NntpData *data = nserv->groups_list[i];
 
-        ctx->mailbox->msg_count++;
-        hdr->read = false;
-        hdr->old = false;
-        hdr->data = mutt_mem_calloc(1, sizeof(struct NntpHeaderData));
-        NHDR(hdr)->article_num = anum;
-        nntp_article_status(ctx->mailbox, hdr, NULL, anum);
-        if (!hdr->read)
-          nntp_parse_xref(ctx->mailbox, hdr);
+        if (get_description(data, NULL, NULL) < 0)
+          return -1;
+        mutt_progress_update(&progress, ++count, -1);
       }
     }
-    FREE(&messages);
-#endif
-
-    nserv->newsrc_modified = false;
-    ret = MUTT_REOPENED;
-  }
-
-  /* some headers were removed, context must be updated */
-  if (ret == MUTT_REOPENED)
-  {
-    if (ctx->mailbox->subj_hash)
-      mutt_hash_destroy(&ctx->mailbox->subj_hash);
-    if (ctx->mailbox->id_hash)
-      mutt_hash_destroy(&ctx->mailbox->id_hash);
-    mutt_clear_threads(ctx);
-
-    ctx->mailbox->vcount = 0;
-    ctx->deleted = 0;
-    ctx->new = 0;
-    ctx->mailbox->msg_unread = 0;
-    ctx->mailbox->msg_flagged = 0;
-    ctx->mailbox->changed = false;
-    ctx->mailbox->id_hash = NULL;
-    ctx->mailbox->subj_hash = NULL;
-    mx_update_context(ctx, ctx->mailbox->msg_count);
-  }
-
-  /* fetch headers of new articles */
-  if (nntp_data->last_message > nntp_data->last_loaded)
-  {
-    int oldmsgcount = ctx->mailbox->msg_count;
-    bool quiet = ctx->mailbox->quiet;
-    ctx->mailbox->quiet = true;
-#ifdef USE_HCACHE
-    if (!hc)
-    {
-      hc = nntp_hcache_open(nntp_data);
-      nntp_hcache_update(nntp_data, hc);
-    }
-#endif
-    rc = nntp_fetch_headers(ctx, hc, nntp_data->last_loaded + 1, nntp_data->last_message, 0);
-    ctx->mailbox->quiet = quiet;
-    if (rc >= 0)
-      nntp_data->last_loaded = nntp_data->last_message;
-    if (ret == 0 && ctx->mailbox->msg_count > oldmsgcount)
-      ret = MUTT_NEW_MAIL;
+    update_active = true;
+    rc = 1;
   }
-
-#ifdef USE_HCACHE
-  mutt_hcache_close(hc);
-#endif
-  if (ret)
-    nntp_newsrc_close(nserv);
+  if (update_active)
+    nntp_active_save_cache(nserv);
   mutt_clear_error();
-  return ret;
+  return rc;
 }
 
 /**
- * nntp_mbox_check - Implements MxOps::mbox_check()
- * @param ctx        Mailbox
- * @param index_hint Current message (UNUSED)
- * @retval #MUTT_REOPENED Articles have been renumbered or removed from server
- * @retval #MUTT_NEW_MAIL New articles found
- * @retval  0             No change
- * @retval -1             Lost connection
+ * nntp_check_msgid - Fetch article by Message-ID
+ * @param ctx   Mailbox
+ * @param msgid Message ID
+ * @retval  0 Success
+ * @retval  1 No such article
+ * @retval -1 Error
  */
-static int nntp_mbox_check(struct Context *ctx, int *index_hint)
+int nntp_check_msgid(struct Context *ctx, const char *msgid)
 {
-  int ret = check_mailbox(ctx);
-  if (ret == 0)
+  struct NntpData *nntp_data = ctx->mailbox->data;
+  char buf[LONG_STRING];
+
+  FILE *fp = mutt_file_mkstemp();
+  if (!fp)
+  {
+    mutt_perror(_("Can't create temporary file"));
+    return -1;
+  }
+
+  snprintf(buf, sizeof(buf), "HEAD %s\r\n", msgid);
+  int rc = nntp_fetch_lines(nntp_data, buf, sizeof(buf), NULL, fetch_tempfile, fp);
+  if (rc)
+  {
+    mutt_file_fclose(&fp);
+    if (rc < 0)
+      return -1;
+    if (mutt_str_strncmp("430", buf, 3) == 0)
+      return 1;
+    mutt_error("HEAD: %s", buf);
+    return -1;
+  }
+
+  /* parse header */
+  if (ctx->mailbox->msg_count == ctx->mailbox->hdrmax)
+    mx_alloc_memory(ctx->mailbox);
+  ctx->mailbox->hdrs[ctx->mailbox->msg_count] = mutt_header_new();
+  struct Header *hdr = ctx->mailbox->hdrs[ctx->mailbox->msg_count];
+  hdr->data = mutt_mem_calloc(1, sizeof(struct NntpHeaderData));
+  hdr->env = mutt_rfc822_read_header(fp, hdr, false, false);
+  mutt_file_fclose(&fp);
+
+  /* get article number */
+  if (hdr->env->xref)
+    nntp_parse_xref(ctx->mailbox, hdr);
+  else
   {
-    struct NntpData *nntp_data = ctx->mailbox->data;
-    struct NntpServer *nserv = nntp_data->nserv;
-    nntp_newsrc_close(nserv);
+    snprintf(buf, sizeof(buf), "STAT %s\r\n", msgid);
+    if (nntp_query(nntp_data, buf, sizeof(buf)) < 0)
+    {
+      mutt_header_free(&hdr);
+      return -1;
+    }
+    sscanf(buf + 4, ANUM, &NHDR(hdr)->article_num);
   }
-  return ret;
+
+  /* reset flags */
+  hdr->read = false;
+  hdr->old = false;
+  hdr->deleted = false;
+  hdr->changed = true;
+  hdr->received = hdr->date_sent;
+  hdr->index = ctx->mailbox->msg_count++;
+  mx_update_context(ctx, 1);
+  return 0;
 }
 
 /**
- * nntp_mbox_sync - Implements MxOps::mbox_sync()
- *
- * @note May also return values from check_mailbox()
+ * nntp_check_children - Fetch children of article with the Message-ID
+ * @param ctx   Mailbox
+ * @param msgid Message ID to find
+ * @retval  0 Success
+ * @retval -1 Failure
  */
-static int nntp_mbox_sync(struct Context *ctx, int *index_hint)
+int nntp_check_children(struct Context *ctx, const char *msgid)
 {
   struct NntpData *nntp_data = ctx->mailbox->data;
+  struct ChildCtx cc;
+  char buf[STRING];
   int rc;
-#ifdef USE_HCACHE
-  header_cache_t *hc = NULL;
-#endif
+  bool quiet;
+  void *hc = NULL;
 
-  /* check for new articles */
-  nntp_data->nserv->check_time = 0;
-  rc = check_mailbox(ctx);
-  if (rc)
-    return rc;
+  if (!nntp_data || !nntp_data->nserv)
+    return -1;
+  if (nntp_data->first_message > nntp_data->last_loaded)
+    return 0;
 
-#ifdef USE_HCACHE
-  nntp_data->last_cached = 0;
-  hc = nntp_hcache_open(nntp_data);
-#endif
+  /* init context */
+  cc.mailbox = ctx->mailbox;
+  cc.num = 0;
+  cc.max = 10;
+  cc.child = mutt_mem_malloc(sizeof(anum_t) * cc.max);
 
-  for (int i = 0; i < ctx->mailbox->msg_count; i++)
+  /* fetch numbers of child messages */
+  snprintf(buf, sizeof(buf), "XPAT References %u-%u *%s*\r\n",
+           nntp_data->first_message, nntp_data->last_loaded, msgid);
+  rc = nntp_fetch_lines(nntp_data, buf, sizeof(buf), NULL, fetch_children, &cc);
+  if (rc)
   {
-    struct Header *hdr = ctx->mailbox->hdrs[i];
-    char buf[16];
-
-    snprintf(buf, sizeof(buf), "%d", NHDR(hdr)->article_num);
-    if (nntp_data->bcache && hdr->deleted)
-    {
-      mutt_debug(2, "mutt_bcache_del %s\n", buf);
-      mutt_bcache_del(nntp_data->bcache, buf);
-    }
-
-#ifdef USE_HCACHE
-    if (hc && (hdr->changed || hdr->deleted))
+    FREE(&cc.child);
+    if (rc > 0)
     {
-      if (hdr->deleted && !hdr->read)
-        nntp_data->unread--;
-      mutt_debug(2, "mutt_hcache_store %s\n", buf);
-      mutt_hcache_store(hc, buf, strlen(buf), hdr, 0);
+      if (mutt_str_strncmp("500", buf, 3) != 0)
+        mutt_error("XPAT: %s", buf);
+      else
+      {
+        mutt_error(_("Unable to find child articles because server does not "
+                     "support XPAT command"));
+      }
     }
-#endif
+    return -1;
   }
 
+  /* fetch all found messages */
+  quiet = ctx->mailbox->quiet;
+  ctx->mailbox->quiet = true;
 #ifdef USE_HCACHE
-  if (hc)
+  hc = nntp_hcache_open(nntp_data);
+#endif
+  for (int i = 0; i < cc.num; i++)
   {
-    mutt_hcache_close(hc);
-    nntp_data->last_cached = nntp_data->last_loaded;
+    rc = nntp_fetch_headers(ctx, hc, cc.child[i], cc.child[i], 1);
+    if (rc < 0)
+      break;
   }
+#ifdef USE_HCACHE
+  mutt_hcache_close(hc);
 #endif
-
-  /* save .newsrc entries */
-  nntp_newsrc_gen_entries(ctx);
-  nntp_newsrc_update(nntp_data->nserv);
-  nntp_newsrc_close(nntp_data->nserv);
-  return 0;
+  ctx->mailbox->quiet = quiet;
+  FREE(&cc.child);
+  return (rc < 0) ? -1 : 0;
 }
 
 /**
- * nntp_mbox_close - Implements MxOps::mbox_close()
- * @retval 0 Always
+ * nntp_compare_order - Sort to mailbox order - Implements ::sort_t
  */
-static int nntp_mbox_close(struct Context *ctx)
+int nntp_compare_order(const void *a, const void *b)
 {
-  struct NntpData *nntp_data = ctx->mailbox->data, *nntp_tmp = NULL;
-
-  if (!nntp_data)
-    return 0;
-
-  nntp_data->unread = ctx->mailbox->msg_unread;
-
-  nntp_acache_free(nntp_data);
-  if (!nntp_data->nserv || !nntp_data->nserv->groups_hash || !nntp_data->group)
-    return 0;
+  struct Header **ha = (struct Header **) a;
+  struct Header **hb = (struct Header **) b;
 
-  nntp_tmp = mutt_hash_find(nntp_data->nserv->groups_hash, nntp_data->group);
-  if (!nntp_tmp || nntp_tmp != nntp_data)
-    nntp_data_free(nntp_data);
-  return 0;
+  anum_t na = NHDR(*ha)->article_num;
+  anum_t nb = NHDR(*hb)->article_num;
+  int result = (na == nb) ? 0 : (na > nb) ? 1 : -1;
+  result = perform_auxsort(result, a, b);
+  return SORTCODE(result);
 }
 
 /**
- * nntp_date - Get date and time from server
- * @param nserv NNTP server
- * @param now   Server time
- * @retval  0 Success
- * @retval -1 Failure
+ * nntp_mbox_open - Implements MxOps::mbox_open()
  */
-static int nntp_date(struct NntpServer *nserv, time_t *now)
+static int nntp_mbox_open(struct Context *ctx)
 {
-  if (nserv->hasDATE)
-  {
-    struct NntpData nntp_data;
-    char buf[LONG_STRING];
-    struct tm tm;
-    memset(&tm, 0, sizeof(tm));
-
-    nntp_data.nserv = nserv;
-    nntp_data.group = NULL;
-    mutt_str_strfcpy(buf, "DATE\r\n", sizeof(buf));
-    if (nntp_query(&nntp_data, buf, sizeof(buf)) < 0)
-      return -1;
+  struct NntpServer *nserv = NULL;
+  struct NntpData *nntp_data = NULL;
+  char buf[HUGE_STRING];
+  char server[LONG_STRING];
+  char *group = NULL;
+  int rc;
+  void *hc = NULL;
+  anum_t first, last, count = 0;
+  struct Url url;
 
-    if (sscanf(buf, "111 %4d%2d%2d%2d%2d%2d%*s", &tm.tm_year, &tm.tm_mon,
-               &tm.tm_mday, &tm.tm_hour, &tm.tm_min, &tm.tm_sec) == 6)
-    {
-      tm.tm_year -= 1900;
-      tm.tm_mon--;
-      *now = timegm(&tm);
-      if (*now >= 0)
-      {
-        mutt_debug(1, "server time is %lu\n", *now);
-        return 0;
-      }
-    }
+  mutt_str_strfcpy(buf, ctx->mailbox->path, sizeof(buf));
+  if (url_parse(&url, buf) < 0 || !url.host || !url.path ||
+      !(url.scheme == U_NNTP || url.scheme == U_NNTPS))
+  {
+    url_free(&url);
+    mutt_error(_("%s is an invalid newsgroup specification"), ctx->mailbox->path);
+    return -1;
   }
-  time(now);
-  return 0;
-}
-
-/**
- * nntp_active_fetch - Fetch list of all newsgroups from server
- * @param nserv NNTP server
- * @param new   Mark the groups as new
- * @retval  0 Success
- * @retval -1 Failure
- */
-int nntp_active_fetch(struct NntpServer *nserv, bool new)
-{
-  struct NntpData nntp_data;
-  char msg[STRING];
-  char buf[LONG_STRING];
-  unsigned int i;
-  int rc;
 
-  snprintf(msg, sizeof(msg), _("Loading list of groups from server %s..."),
-           nserv->conn->account.host);
-  mutt_message(msg);
-  if (nntp_date(nserv, &nserv->newgroups_time) < 0)
+  group = url.path;
+  url.path = strchr(url.path, '\0');
+  url_tostring(&url, server, sizeof(server), 0);
+  nserv = nntp_select_server(ctx->mailbox, server, true);
+  url_free(&url);
+  if (!nserv)
     return -1;
+  CurrentNewsSrv = nserv;
 
-  nntp_data.nserv = nserv;
-  nntp_data.group = NULL;
-  i = nserv->groups_num;
-  mutt_str_strfcpy(buf, "LIST\r\n", sizeof(buf));
-  rc = nntp_fetch_lines(&nntp_data, buf, sizeof(buf), msg, nntp_add_group, nserv);
-  if (rc)
+  /* find news group data structure */
+  nntp_data = mutt_hash_find(nserv->groups_hash, group);
+  if (!nntp_data)
   {
-    if (rc > 0)
-    {
-      mutt_error("LIST: %s", buf);
-    }
+    nntp_newsrc_close(nserv);
+    mutt_error(_("Newsgroup %s not found on the server"), group);
+    return -1;
+  }
+
+  mutt_bit_unset(ctx->mailbox->rights, MUTT_ACL_INSERT);
+  if (!nntp_data->newsrc_ent && !nntp_data->subscribed && !SaveUnsubscribed)
+    ctx->mailbox->readonly = true;
+
+  /* select newsgroup */
+  mutt_message(_("Selecting %s..."), group);
+  buf[0] = '\0';
+  if (nntp_query(nntp_data, buf, sizeof(buf)) < 0)
+  {
+    nntp_newsrc_close(nserv);
     return -1;
   }
 
-  if (new)
+  /* newsgroup not found, remove it */
+  if (mutt_str_strncmp("411", buf, 3) == 0)
   {
-    for (; i < nserv->groups_num; i++)
+    mutt_error(_("Newsgroup %s has been removed from the server"), nntp_data->group);
+    if (!nntp_data->deleted)
     {
-      struct NntpData *data = nserv->groups_list[i];
-      data->new = true;
+      nntp_data->deleted = true;
+      nntp_active_save_cache(nserv);
+    }
+    if (nntp_data->newsrc_ent && !nntp_data->subscribed && !SaveUnsubscribed)
+    {
+      FREE(&nntp_data->newsrc_ent);
+      nntp_data->newsrc_len = 0;
+      nntp_delete_group_cache(nntp_data);
+      nntp_newsrc_update(nserv);
     }
   }
 
-  for (i = 0; i < nserv->groups_num; i++)
+  /* parse newsgroup info */
+  else
   {
-    struct NntpData *data = nserv->groups_list[i];
+    if (sscanf(buf, "211 " ANUM " " ANUM " " ANUM, &count, &first, &last) != 3)
+    {
+      nntp_newsrc_close(nserv);
+      mutt_error("GROUP: %s", buf);
+      return -1;
+    }
+    nntp_data->first_message = first;
+    nntp_data->last_message = last;
+    nntp_data->deleted = false;
 
-    if (data && data->deleted && !data->newsrc_ent)
+    /* get description if empty */
+    if (NntpLoadDescription && !nntp_data->desc)
     {
-      nntp_delete_group_cache(data);
-      mutt_hash_delete(nserv->groups_hash, data->group, NULL);
-      nserv->groups_list[i] = NULL;
+      if (get_description(nntp_data, NULL, NULL) < 0)
+      {
+        nntp_newsrc_close(nserv);
+        return -1;
+      }
+      if (nntp_data->desc)
+        nntp_active_save_cache(nserv);
     }
   }
 
-  if (NntpLoadDescription)
-    rc = get_description(&nntp_data, "*", _("Loading descriptions..."));
+  time(&nserv->check_time);
+  ctx->mailbox->data = nntp_data;
+  if (!nntp_data->bcache && (nntp_data->newsrc_ent || nntp_data->subscribed || SaveUnsubscribed))
+    nntp_data->bcache = mutt_bcache_open(&nserv->conn->account, nntp_data->group);
 
-  nntp_active_save_cache(nserv);
+  /* strip off extra articles if adding context is greater than $nntp_context */
+  first = nntp_data->first_message;
+  if (NntpContext && nntp_data->last_message - first + 1 > NntpContext)
+    first = nntp_data->last_message - NntpContext + 1;
+  nntp_data->last_loaded = first ? first - 1 : 0;
+  count = nntp_data->first_message;
+  nntp_data->first_message = first;
+  nntp_bcache_update(nntp_data);
+  nntp_data->first_message = count;
+#ifdef USE_HCACHE
+  hc = nntp_hcache_open(nntp_data);
+  nntp_hcache_update(nntp_data, hc);
+#endif
+  if (!hc)
+  {
+    mutt_bit_unset(ctx->mailbox->rights, MUTT_ACL_WRITE);
+    mutt_bit_unset(ctx->mailbox->rights, MUTT_ACL_DELETE);
+  }
+  nntp_newsrc_close(nserv);
+  rc = nntp_fetch_headers(ctx, hc, first, nntp_data->last_message, 0);
+#ifdef USE_HCACHE
+  mutt_hcache_close(hc);
+#endif
   if (rc < 0)
     return -1;
-  mutt_clear_error();
+  nntp_data->last_loaded = nntp_data->last_message;
+  nserv->newsrc_modified = false;
   return 0;
 }
 
 /**
- * nntp_check_new_groups - Check for new groups/articles in subscribed groups
- * @param mailbox Mailbox
- * @param nserv   NNTP server
- * @retval  1 New groups found
- * @retval  0 No new groups
- * @retval -1 Error
+ * nntp_mbox_check - Implements MxOps::mbox_check()
+ * @param ctx        Mailbox
+ * @param index_hint Current message (UNUSED)
+ * @retval #MUTT_REOPENED Articles have been renumbered or removed from server
+ * @retval #MUTT_NEW_MAIL New articles found
+ * @retval  0             No change
+ * @retval -1             Lost connection
  */
-int nntp_check_new_groups(struct Mailbox *mailbox, struct NntpServer *nserv)
+static int nntp_mbox_check(struct Context *ctx, int *index_hint)
 {
-  struct NntpData nntp_data;
-  time_t now;
-  struct tm *tm = NULL;
-  char buf[LONG_STRING];
-  char *msg = _("Checking for new newsgroups...");
-  unsigned int i;
-  int rc, update_active = false;
-
-  if (!nserv || !nserv->newgroups_time)
-    return -1;
-
-  /* check subscribed newsgroups for new articles */
-  if (ShowNewNews)
-  {
-    mutt_message(_("Checking for new messages..."));
-    for (i = 0; i < nserv->groups_num; i++)
-    {
-      struct NntpData *data = nserv->groups_list[i];
-
-      if (data && data->subscribed)
-      {
-        rc = nntp_group_poll(data, 1);
-        if (rc < 0)
-          return -1;
-        if (rc > 0)
-          update_active = true;
-      }
-    }
-    /* select current newsgroup */
-    if (mailbox && (mailbox->magic == MUTT_NNTP))
-    {
-      buf[0] = '\0';
-      if (nntp_query(mailbox->data, buf, sizeof(buf)) < 0)
-        return -1;
-    }
-  }
-  else if (nserv->newgroups_time)
-    return 0;
-
-  /* get list of new groups */
-  mutt_message(msg);
-  if (nntp_date(nserv, &now) < 0)
-    return -1;
-  nntp_data.nserv = nserv;
-  if (mailbox && (mailbox->magic == MUTT_NNTP))
-    nntp_data.group = ((struct NntpData *) mailbox->data)->group;
-  else
-    nntp_data.group = NULL;
-  i = nserv->groups_num;
-  tm = gmtime(&nserv->newgroups_time);
-  snprintf(buf, sizeof(buf), "NEWGROUPS %02d%02d%02d %02d%02d%02d GMT\r\n",
-           tm->tm_year % 100, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour,
-           tm->tm_min, tm->tm_sec);
-  rc = nntp_fetch_lines(&nntp_data, buf, sizeof(buf), msg, nntp_add_group, nserv);
-  if (rc)
-  {
-    if (rc > 0)
-    {
-      mutt_error("NEWGROUPS: %s", buf);
-    }
-    return -1;
-  }
-
-  /* new groups found */
-  rc = 0;
-  if (nserv->groups_num != i)
+  int ret = check_mailbox(ctx);
+  if (ret == 0)
   {
-    int groups_num = i;
-
-    nserv->newgroups_time = now;
-    for (; i < nserv->groups_num; i++)
-    {
-      struct NntpData *data = nserv->groups_list[i];
-      data->new = true;
-    }
-
-    /* loading descriptions */
-    if (NntpLoadDescription)
-    {
-      unsigned int count = 0;
-      struct Progress progress;
-
-      mutt_progress_init(&progress, _("Loading descriptions..."),
-                         MUTT_PROGRESS_MSG, ReadInc, nserv->groups_num - i);
-      for (i = groups_num; i < nserv->groups_num; i++)
-      {
-        struct NntpData *data = nserv->groups_list[i];
-
-        if (get_description(data, NULL, NULL) < 0)
-          return -1;
-        mutt_progress_update(&progress, ++count, -1);
-      }
-    }
-    update_active = true;
-    rc = 1;
+    struct NntpData *nntp_data = ctx->mailbox->data;
+    struct NntpServer *nserv = nntp_data->nserv;
+    nntp_newsrc_close(nserv);
   }
-  if (update_active)
-    nntp_active_save_cache(nserv);
-  mutt_clear_error();
-  return rc;
+  return ret;
 }
 
 /**
- * nntp_check_msgid - Fetch article by Message-ID
- * @param ctx   Mailbox
- * @param msgid Message ID
- * @retval  0 Success
- * @retval  1 No such article
- * @retval -1 Error
+ * nntp_mbox_sync - Implements MxOps::mbox_sync()
+ *
+ * @note May also return values from check_mailbox()
  */
-int nntp_check_msgid(struct Context *ctx, const char *msgid)
+static int nntp_mbox_sync(struct Context *ctx, int *index_hint)
 {
   struct NntpData *nntp_data = ctx->mailbox->data;
-  char buf[LONG_STRING];
-
-  FILE *fp = mutt_file_mkstemp();
-  if (!fp)
-  {
-    mutt_perror(_("Can't create temporary file"));
-    return -1;
-  }
+  int rc;
+#ifdef USE_HCACHE
+  header_cache_t *hc = NULL;
+#endif
 
-  snprintf(buf, sizeof(buf), "HEAD %s\r\n", msgid);
-  int rc = nntp_fetch_lines(nntp_data, buf, sizeof(buf), NULL, fetch_tempfile, fp);
+  /* check for new articles */
+  nntp_data->nserv->check_time = 0;
+  rc = check_mailbox(ctx);
   if (rc)
+    return rc;
+
+#ifdef USE_HCACHE
+  nntp_data->last_cached = 0;
+  hc = nntp_hcache_open(nntp_data);
+#endif
+
+  for (int i = 0; i < ctx->mailbox->msg_count; i++)
   {
-    mutt_file_fclose(&fp);
-    if (rc < 0)
-      return -1;
-    if (mutt_str_strncmp("430", buf, 3) == 0)
-      return 1;
-    mutt_error("HEAD: %s", buf);
-    return -1;
-  }
+    struct Header *hdr = ctx->mailbox->hdrs[i];
+    char buf[16];
 
-  /* parse header */
-  if (ctx->mailbox->msg_count == ctx->mailbox->hdrmax)
-    mx_alloc_memory(ctx->mailbox);
-  ctx->mailbox->hdrs[ctx->mailbox->msg_count] = mutt_header_new();
-  struct Header *hdr = ctx->mailbox->hdrs[ctx->mailbox->msg_count];
-  hdr->data = mutt_mem_calloc(1, sizeof(struct NntpHeaderData));
-  hdr->env = mutt_rfc822_read_header(fp, hdr, false, false);
-  mutt_file_fclose(&fp);
+    snprintf(buf, sizeof(buf), "%d", NHDR(hdr)->article_num);
+    if (nntp_data->bcache && hdr->deleted)
+    {
+      mutt_debug(2, "mutt_bcache_del %s\n", buf);
+      mutt_bcache_del(nntp_data->bcache, buf);
+    }
 
-  /* get article number */
-  if (hdr->env->xref)
-    nntp_parse_xref(ctx->mailbox, hdr);
-  else
-  {
-    snprintf(buf, sizeof(buf), "STAT %s\r\n", msgid);
-    if (nntp_query(nntp_data, buf, sizeof(buf)) < 0)
+#ifdef USE_HCACHE
+    if (hc && (hdr->changed || hdr->deleted))
     {
-      mutt_header_free(&hdr);
-      return -1;
+      if (hdr->deleted && !hdr->read)
+        nntp_data->unread--;
+      mutt_debug(2, "mutt_hcache_store %s\n", buf);
+      mutt_hcache_store(hc, buf, strlen(buf), hdr, 0);
     }
-    sscanf(buf + 4, ANUM, &NHDR(hdr)->article_num);
+#endif
   }
 
-  /* reset flags */
-  hdr->read = false;
-  hdr->old = false;
-  hdr->deleted = false;
-  hdr->changed = true;
-  hdr->received = hdr->date_sent;
-  hdr->index = ctx->mailbox->msg_count++;
-  mx_update_context(ctx, 1);
+#ifdef USE_HCACHE
+  if (hc)
+  {
+    mutt_hcache_close(hc);
+    nntp_data->last_cached = nntp_data->last_loaded;
+  }
+#endif
+
+  /* save .newsrc entries */
+  nntp_newsrc_gen_entries(ctx);
+  nntp_newsrc_update(nntp_data->nserv);
+  nntp_newsrc_close(nntp_data->nserv);
   return 0;
 }
 
 /**
- * struct ChildCtx - Keep track of the children of an article
- */
-struct ChildCtx
-{
-  struct Mailbox *mailbox;
-  unsigned int num;
-  unsigned int max;
-  anum_t *child;
-};
-
-/**
- * fetch_children - Parse XPAT line
- * @param line String to parse
- * @param data ChildCtx
+ * nntp_mbox_close - Implements MxOps::mbox_close()
  * @retval 0 Always
  */
-static int fetch_children(char *line, void *data)
+static int nntp_mbox_close(struct Context *ctx)
 {
-  struct ChildCtx *cc = data;
-  anum_t anum;
+  struct NntpData *nntp_data = ctx->mailbox->data, *nntp_tmp = NULL;
 
-  if (!line || sscanf(line, ANUM, &anum) != 1)
+  if (!nntp_data)
     return 0;
-  for (unsigned int i = 0; i < cc->mailbox->msg_count; i++)
-    if (NHDR(cc->mailbox->hdrs[i])->article_num == anum)
-      return 0;
-  if (cc->num >= cc->max)
-  {
-    cc->max *= 2;
-    mutt_mem_realloc(&cc->child, sizeof(anum_t) * cc->max);
-  }
-  cc->child[cc->num++] = anum;
+
+  nntp_data->unread = ctx->mailbox->msg_unread;
+
+  nntp_acache_free(nntp_data);
+  if (!nntp_data->nserv || !nntp_data->nserv->groups_hash || !nntp_data->group)
+    return 0;
+
+  nntp_tmp = mutt_hash_find(nntp_data->nserv->groups_hash, nntp_data->group);
+  if (!nntp_tmp || nntp_tmp != nntp_data)
+    nntp_data_free(nntp_data);
   return 0;
 }
 
 /**
- * nntp_check_children - Fetch children of article with the Message-ID
- * @param ctx   Mailbox
- * @param msgid Message ID to find
- * @retval  0 Success
- * @retval -1 Failure
+ * nntp_msg_open - Implements MxOps::msg_open()
  */
-int nntp_check_children(struct Context *ctx, const char *msgid)
+static int nntp_msg_open(struct Context *ctx, struct Message *msg, int msgno)
 {
   struct NntpData *nntp_data = ctx->mailbox->data;
-  struct ChildCtx cc;
-  char buf[STRING];
-  int rc;
-  bool quiet;
-  void *hc = NULL;
+  struct Header *hdr = ctx->mailbox->hdrs[msgno];
+  char article[16];
 
-  if (!nntp_data || !nntp_data->nserv)
-    return -1;
-  if (nntp_data->first_message > nntp_data->last_loaded)
-    return 0;
+  /* try to get article from cache */
+  struct NntpAcache *acache = &nntp_data->acache[hdr->index % NNTP_ACACHE_LEN];
+  if (acache->path)
+  {
+    if (acache->index == hdr->index)
+    {
+      msg->fp = mutt_file_fopen(acache->path, "r");
+      if (msg->fp)
+        return 0;
+    }
+    /* clear previous entry */
+    else
+    {
+      unlink(acache->path);
+      FREE(&acache->path);
+    }
+  }
+  snprintf(article, sizeof(article), "%d", NHDR(hdr)->article_num);
+  msg->fp = mutt_bcache_get(nntp_data->bcache, article);
+  if (msg->fp)
+  {
+    if (NHDR(hdr)->parsed)
+      return 0;
+  }
+  else
+  {
+    char buf[PATH_MAX];
+    /* don't try to fetch article from removed newsgroup */
+    if (nntp_data->deleted)
+      return -1;
 
-  /* init context */
-  cc.mailbox = ctx->mailbox;
-  cc.num = 0;
-  cc.max = 10;
-  cc.child = mutt_mem_malloc(sizeof(anum_t) * cc.max);
+    /* create new cache file */
+    const char *fetch_msg = _("Fetching message...");
+    mutt_message(fetch_msg);
+    msg->fp = mutt_bcache_put(nntp_data->bcache, article);
+    if (!msg->fp)
+    {
+      mutt_mktemp(buf, sizeof(buf));
+      acache->path = mutt_str_strdup(buf);
+      acache->index = hdr->index;
+      msg->fp = mutt_file_fopen(acache->path, "w+");
+      if (!msg->fp)
+      {
+        mutt_perror(acache->path);
+        unlink(acache->path);
+        FREE(&acache->path);
+        return -1;
+      }
+    }
 
-  /* fetch numbers of child messages */
-  snprintf(buf, sizeof(buf), "XPAT References %u-%u *%s*\r\n",
-           nntp_data->first_message, nntp_data->last_loaded, msgid);
-  rc = nntp_fetch_lines(nntp_data, buf, sizeof(buf), NULL, fetch_children, &cc);
-  if (rc)
-  {
-    FREE(&cc.child);
-    if (rc > 0)
+    /* fetch message to cache file */
+    snprintf(buf, sizeof(buf), "ARTICLE %s\r\n",
+             NHDR(hdr)->article_num ? article : hdr->env->message_id);
+    const int rc = nntp_fetch_lines(nntp_data, buf, sizeof(buf), fetch_msg,
+                                    fetch_tempfile, msg->fp);
+    if (rc)
     {
-      if (mutt_str_strncmp("500", buf, 3) != 0)
-        mutt_error("XPAT: %s", buf);
-      else
+      mutt_file_fclose(&msg->fp);
+      if (acache->path)
       {
-        mutt_error(_("Unable to find child articles because server does not "
-                     "support XPAT command"));
+        unlink(acache->path);
+        FREE(&acache->path);
+      }
+      if (rc > 0)
+      {
+        if (mutt_str_strncmp(NHDR(hdr)->article_num ? "423" : "430", buf, 3) == 0)
+        {
+          mutt_error(_("Article %d not found on the server"),
+                     NHDR(hdr)->article_num ? article : hdr->env->message_id);
+        }
+        else
+          mutt_error("ARTICLE: %s", buf);
       }
+      return -1;
     }
-    return -1;
-  }
 
-  /* fetch all found messages */
-  quiet = ctx->mailbox->quiet;
-  ctx->mailbox->quiet = true;
-#ifdef USE_HCACHE
-  hc = nntp_hcache_open(nntp_data);
-#endif
-  for (int i = 0; i < cc.num; i++)
-  {
-    rc = nntp_fetch_headers(ctx, hc, cc.child[i], cc.child[i], 1);
-    if (rc < 0)
-      break;
+    if (!acache->path)
+      mutt_bcache_commit(nntp_data->bcache, article);
   }
-#ifdef USE_HCACHE
-  mutt_hcache_close(hc);
-#endif
-  ctx->mailbox->quiet = quiet;
-  FREE(&cc.child);
-  return (rc < 0) ? -1 : 0;
+
+  /* replace envelope with new one
+   * hash elements must be updated because pointers will be changed */
+  if (ctx->mailbox->id_hash && hdr->env->message_id)
+    mutt_hash_delete(ctx->mailbox->id_hash, hdr->env->message_id, hdr);
+  if (ctx->mailbox->subj_hash && hdr->env->real_subj)
+    mutt_hash_delete(ctx->mailbox->subj_hash, hdr->env->real_subj, hdr);
+
+  mutt_env_free(&hdr->env);
+  hdr->env = mutt_rfc822_read_header(msg->fp, hdr, false, false);
+
+  if (ctx->mailbox->id_hash && hdr->env->message_id)
+    mutt_hash_insert(ctx->mailbox->id_hash, hdr->env->message_id, hdr);
+  if (ctx->mailbox->subj_hash && hdr->env->real_subj)
+    mutt_hash_insert(ctx->mailbox->subj_hash, hdr->env->real_subj, hdr);
+
+  /* fix content length */
+  fseek(msg->fp, 0, SEEK_END);
+  hdr->content->length = ftell(msg->fp) - hdr->content->offset;
+
+  /* this is called in neomutt before the open which fetches the message,
+   * which is probably wrong, but we just call it again here to handle
+   * the problem instead of fixing it */
+  NHDR(hdr)->parsed = true;
+  mutt_parse_mime_message(ctx, hdr);
+
+  /* these would normally be updated in mx_update_context(), but the
+   * full headers aren't parsed with overview, so the information wasn't
+   * available then */
+  if (WithCrypto)
+    hdr->security = crypt_query(hdr->content);
+
+  rewind(msg->fp);
+  mutt_clear_error();
+  return 0;
 }
 
 /**
- * nntp_compare_order - Sort to mailbox order - Implements ::sort_t
+ * nntp_msg_close - Implements MxOps::msg_close()
+ *
+ * @note May also return EOF Failure, see errno
  */
-int nntp_compare_order(const void *a, const void *b)
+static int nntp_msg_close(struct Context *ctx, struct Message *msg)
 {
-  struct Header **ha = (struct Header **) a;
-  struct Header **hb = (struct Header **) b;
-
-  anum_t na = NHDR(*ha)->article_num;
-  anum_t nb = NHDR(*hb)->article_num;
-  int result = (na == nb) ? 0 : (na > nb) ? 1 : -1;
-  result = perform_auxsort(result, a, b);
-  return SORTCODE(result);
+  return mutt_file_fclose(&msg->fp);
 }
 
 /**