Add LMDB backend support for header cache. (see #3691)

Based on the original from JP Mens:
https://gist.github.com/jpmens/15969d9d678a3d450e4e

The following performance patch was manually applied on top of the
original patch:
https://github.com/neomutt/neomutt/commit/7e5380cd4c40d119ff83b2cf5f51f2cdb8a95ab3

A variant of this patch was added to handle larger mailboxes:
https://github.com/neomutt/neomutt/commit/6d337642e701b1dde4b5d0812e01c85f41ba65ca

Thanks to all the developers and contributors to this patch, and to
Fabian Groffen for bundling and posting this to mutt-dev.

diff --git a/configure.ac b/configure.ac
index 57e408e4bed2c0e441343c900ea16f23db064738..c7fc41567071a9ea55a1b41ae153251096478a88 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -881,6 +881,7 @@ AC_ARG_WITH(tokyocabinet, AS_HELP_STRING([--without-tokyocabinet],[Don't use tok
 AC_ARG_WITH(qdbm, AS_HELP_STRING([--without-qdbm],[Don't use qdbm even if it is available]))
 AC_ARG_WITH(gdbm, AS_HELP_STRING([--without-gdbm],[Don't use gdbm even if it is available]))
 AC_ARG_WITH(bdb, AS_HELP_STRING([--with-bdb@<:@=DIR@:>@],[Use BerkeleyDB4 if gdbm is not available]))
+AC_ARG_WITH(lmdb, AS_HELP_STRING([--with-lmdb@<:@=DIR@:>@],[Use LMDB if gdbm is not available]))
 
 db_found=no
 if test x$enable_hcache = xyes
@@ -925,6 +926,15 @@ then
         db_requested=bdb
       fi
     fi
+    if test -n "$with_lmdb" && test "$with_lmdb" != "no"
+    then
+      if test "$db_requested" != "auto"
+      then
+        AC_MSG_ERROR([more than one header cache engine requested.])
+      else
+        db_requested=lmdb
+      fi
+    fi
     
     dnl -- Tokyo Cabinet --
     if test "$with_tokyocabinet" != "no" \
@@ -1012,7 +1022,8 @@ then
 
     dnl -- BDB --
     ac_bdb_prefix="$with_bdb"
-    if test x$ac_bdb_prefix != xno && test $db_found = no
+    if test x$with_bdb != xno && test $db_found = no \
+           && test "$db_requested" = auto -o "$db_requested" = bdb
     then
         if test x$ac_bdb_prefix = xyes || test x$ac_bdb_prefix = x
         then
@@ -1068,6 +1079,34 @@ then
         fi
     fi
 
+    dnl -- LMDB --
+    if test x$with_lmdb != xno && test $db_found = no \
+           && test "$db_requested" = auto -o "$db_requested" = lmdb
+    then
+        if test "$with_lmdb" != "yes"
+        then
+          CPPFLAGS="$CPPFLAGS -I$with_lmdb/include"
+          LDFLAGS="$LDFLAGS -L$with_lmdb/lib"
+        fi
+        saved_LIBS="$LIBS"
+        LIBS="$LIBS -llmdb"
+        AC_CACHE_CHECK(for mdb_env_create, ac_cv_mdbenvcreate,[
+            ac_cv_mdbenvcreate=no
+            AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <lmdb.h>]], [[mdb_env_create(0);]])],[ac_cv_mdbenvcreate=yes],[])
+        ])
+        LIBS="$saved_LIBS"
+        if test "$ac_cv_mdbenvcreate" = yes
+        then
+          AC_DEFINE(HAVE_LMDB, 1, [LMDB Support])
+          MUTTLIBS="$MUTTLIBS -llmdb"
+          db_found=lmdb
+        fi
+        if test "$db_requested" != auto && test "$db_found" != "$db_requested"
+        then
+          AC_MSG_ERROR([LMDB could not be used. Check config.log for details.])
+        fi
+    fi
+
     if test $db_found = no
     then
         AC_MSG_ERROR([You need Tokyo Cabinet, QDBM, GDBM or Berkeley DB4 for hcache])

diff --git a/hcache.c b/hcache.c
index 7eada101dbfbb559189bba77c2ed25543d5535c9..a123016abb62e9c458136813103cd17849a2fa75 100644 (file)
--- a/hcache.c
+++ b/hcache.c
@@ -32,6 +32,12 @@
 #include <gdbm.h>
 #elif HAVE_DB4
 #include <db.h>
+#elif HAVE_LMDB
+/* This is the maximum size of the database file (2GiB) which is
+ * mmap(2)'ed into memory.  This limit should be good for ~800,000
+ * emails. */
+#define LMDB_DB_SIZE    (2 * 1024 * 1024 * 1024)
+#include <lmdb.h>
 #endif
 
 #include <errno.h>
@@ -83,6 +89,61 @@ struct header_cache
 
 static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len);
 static void mutt_hcache_dbt_empty_init(DBT * dbt);
+#elif HAVE_LMDB
+enum mdb_txn_mode
+{
+  txn_uninitialized = 0,
+  txn_read = 1 << 0,
+  txn_write = 1 << 1
+};
+struct header_cache
+{
+  MDB_env *env;
+  MDB_txn *txn;
+  MDB_dbi db;
+  char *folder;
+  unsigned int crc;
+  enum mdb_txn_mode txn_mode;
+};
+
+static int mdb_get_r_txn(header_cache_t *h)
+{
+  int rc;
+
+  if (h->txn && (h->txn_mode & (txn_read | txn_write)) > 0)
+    return MDB_SUCCESS;
+
+  if (h->txn)
+    rc = mdb_txn_renew(h->txn);
+  else
+    rc = mdb_txn_begin(h->env, NULL, MDB_RDONLY, &h->txn);
+
+  if (rc == MDB_SUCCESS)
+    h->txn_mode = txn_read;
+
+  return rc;
+}
+
+static int mdb_get_w_txn(header_cache_t *h)
+{
+  int rc;
+
+  if (h->txn && (h->txn_mode == txn_write))
+    return MDB_SUCCESS;
+
+  if (h->txn)
+  {
+    if (h->txn_mode == txn_read)
+      mdb_txn_reset(h->txn);
+    h->txn = NULL;
+  }
+
+  rc = mdb_txn_begin(h->env, NULL, 0, &h->txn);
+  if (rc == MDB_SUCCESS)
+    h->txn_mode = txn_write;
+
+  return rc;
+}
 #endif
 
 typedef union
@@ -732,6 +793,11 @@ mutt_hcache_fetch_raw (header_cache_t *h, const char *filename,
 #elif HAVE_DB4
   DBT key;
   DBT data;
+#elif HAVE_LMDB
+  MDB_val key;
+  MDB_val data;
+  size_t folderlen;
+  int rc;
 #endif
   
   if (!h)
@@ -748,6 +814,37 @@ mutt_hcache_fetch_raw (header_cache_t *h, const char *filename,
   h->db->get(h->db, NULL, &key, &data, 0);
   
   return data.data;
+#elif HAVE_LMDB
+  strncpy(path, h->folder, sizeof (path));
+  safe_strcat(path, sizeof (path), filename);
+
+  folderlen = strlen(h->folder);
+  ksize = folderlen + keylen(path + folderlen);  
+  key.mv_data = (char *)path;
+  key.mv_size = ksize;
+  data.mv_data = NULL;
+  data.mv_size = 0;
+  rc = mdb_get_r_txn(h);
+  if (rc != MDB_SUCCESS) {
+    h->txn = NULL;
+    fprintf(stderr, "txn_renew: %s\n", mdb_strerror(rc));
+    return NULL;
+  }
+  rc = mdb_get(h->txn, h->db, &key, &data);
+  if (rc == MDB_NOTFOUND) {
+    return NULL;
+  }
+  if (rc != MDB_SUCCESS) {
+    fprintf(stderr, "mdb_get: %s\n", mdb_strerror(rc));
+    return NULL;
+  }
+  /* Caller frees the data we return, so I MUST make a copy of it */
+
+  char *d = safe_malloc(data.mv_size);
+  memcpy(d, data.mv_data, data.mv_size);
+
+  return d;
+
 #else
   strncpy(path, h->folder, sizeof (path));
   safe_strcat(path, sizeof (path), filename);
@@ -813,6 +910,11 @@ mutt_hcache_store_raw (header_cache_t* h, const char* filename, void* data,
 #elif HAVE_DB4
   DBT key;
   DBT databuf;
+#elif HAVE_LMDB
+  MDB_val key;
+  MDB_val databuf;
+  size_t folderlen;
+  int rc;
 #endif
   
   if (!h)
@@ -831,6 +933,30 @@ mutt_hcache_store_raw (header_cache_t* h, const char* filename, void* data,
   databuf.ulen = dlen;
   
   return h->db->put(h->db, NULL, &key, &databuf, 0);
+#elif HAVE_LMDB
+  folderlen = strlen(h->folder);
+  strncpy(path, h->folder, sizeof (path));
+  safe_strcat(path, sizeof (path), filename);
+  ksize = folderlen + keylen(path + folderlen);
+
+  key.mv_data = (char *)path;
+  key.mv_size = ksize;
+  databuf.mv_data = data;
+  databuf.mv_size = dlen;
+  rc = mdb_get_w_txn(h);
+  if (rc != MDB_SUCCESS) {
+       fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc));
+    return rc;
+  }
+  rc = mdb_put(h->txn, h->db, &key, &databuf, 0);
+  if (rc != MDB_SUCCESS) {
+       fprintf(stderr, "mdb_put: %s\n", mdb_strerror(rc));
+    mdb_txn_abort(h->txn);
+    h->txn_mode = txn_uninitialized;
+    h->txn = NULL;
+    return rc;
+  }
+  return rc;
 #else
   strncpy(path, h->folder, sizeof (path));
   safe_strcat(path, sizeof (path), filename);
@@ -1134,6 +1260,106 @@ mutt_hcache_delete(header_cache_t *h, const char *filename,
   mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename));
   return h->db->del(h->db, NULL, &key, 0);
 }
+#elif HAVE_LMDB
+
+static int
+hcache_open_lmdb (struct header_cache* h, const char* path)
+{
+  int rc;
+
+  h->txn = NULL;
+
+  rc = mdb_env_create(&h->env);
+  if (rc != MDB_SUCCESS) {
+         fprintf(stderr, "hcache_open_lmdb: mdb_env_create: %s", mdb_strerror(rc));
+         return -1;
+  }
+
+  mdb_env_set_mapsize(h->env, LMDB_DB_SIZE);
+
+  rc = mdb_env_open(h->env, path, MDB_NOSUBDIR, 0644);
+  if (rc != MDB_SUCCESS) {
+         fprintf(stderr, "hcache_open_lmdb: mdb_env_open: %s", mdb_strerror(rc));
+         goto fail_env;
+  }
+
+  rc = mdb_get_r_txn(h);
+  if (rc != MDB_SUCCESS) {
+      fprintf(stderr, "hcache_open_lmdb: mdb_txn_begin: %s", mdb_strerror(rc));
+      goto fail_env;
+  }
+
+  rc = mdb_dbi_open(h->txn, NULL, MDB_CREATE, &h->db);
+  if (rc != MDB_SUCCESS) {
+         fprintf(stderr, "hcache_open_lmdb: mdb_dbi_open: %s", mdb_strerror(rc));
+         goto fail_dbi;
+  }
+
+  mdb_txn_reset(h->txn);
+  h->txn_mode = txn_uninitialized;
+  return 0;
+
+fail_dbi:
+  mdb_txn_abort(h->txn);
+  h->txn_mode = txn_uninitialized;
+  h->txn = NULL;
+
+fail_env:
+  mdb_env_close(h->env);
+  return -1;
+}
+
+void
+mutt_hcache_close(header_cache_t *h)
+{
+  if (!h)
+    return;
+
+  if (h->txn && h->txn_mode == txn_write)
+  {
+    mdb_txn_commit(h->txn);
+    h->txn_mode = txn_uninitialized;
+    h->txn = NULL;
+  }
+
+  mdb_env_close(h->env);
+  FREE (&h->folder);
+  FREE (&h);
+}
+
+int
+mutt_hcache_delete(header_cache_t *h, const char *filename,
+                  size_t(*keylen) (const char *fn))
+{
+  MDB_val key;
+  int rc;
+
+  if (!h)
+    return -1;
+
+  if (filename[0] == '/')
+    filename++;
+
+  key.mv_data = (char *)filename;
+  key.mv_size = strlen(filename);
+  rc = mdb_get_w_txn(h);
+  if (rc != MDB_SUCCESS) {
+       fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc));
+    return rc;
+  }
+  rc = mdb_del(h->txn, h->db, &key, NULL);
+  if (rc != MDB_SUCCESS) {
+    if (rc != MDB_NOTFOUND) {
+         fprintf(stderr, "mdb_del: %s\n", mdb_strerror(rc));
+      mdb_txn_abort(h->txn);
+      h->txn_mode = txn_uninitialized;
+      h->txn = NULL;
+    }
+    return rc;
+  }
+
+  return rc;
+}
 #endif
 
 header_cache_t *
@@ -1151,6 +1377,8 @@ mutt_hcache_open(const char *path, const char *folder, hcache_namer_t namer)
   hcache_open = hcache_open_gdbm;
 #elif HAVE_DB4
   hcache_open = hcache_open_db4;
+#elif HAVE_LMDB
+  hcache_open = hcache_open_lmdb;
 #endif
 
   /* Calculate the current hcache version from dynamic configuration */
@@ -1188,7 +1416,11 @@ mutt_hcache_open(const char *path, const char *folder, hcache_namer_t namer)
     hcachever = digest.intval;
   }
 
+#if HAVE_LMDB
+  h->db = 0;
+#else
   h->db = NULL;
+#endif
   h->folder = get_foldername(folder);
   h->crc = hcachever;
 
@@ -1223,6 +1455,11 @@ const char *mutt_hcache_backend (void)
 {
   return DB_VERSION_STRING;
 }
+#elif HAVE_LMDB
+const char *mutt_hcache_backend (void)
+{
+  return "lmdb " MDB_VERSION_STRING;
+}
 #elif HAVE_GDBM
 const char *mutt_hcache_backend (void)
 {