From 483948c9de9ad5d841c6eec502eaa82dbb08a699 Mon Sep 17 00:00:00 2001 From: Kevin McCarthy Date: Sat, 4 Feb 2017 12:53:38 -0800 Subject: [PATCH] Add LMDB backend support for header cache. (see #3691) Based on the original from JP Mens: https://gist.github.com/jpmens/15969d9d678a3d450e4e The following performance patch was manually applied on top of the original patch: https://github.com/neomutt/neomutt/commit/7e5380cd4c40d119ff83b2cf5f51f2cdb8a95ab3 A variant of this patch was added to handle larger mailboxes: https://github.com/neomutt/neomutt/commit/6d337642e701b1dde4b5d0812e01c85f41ba65ca Thanks to all the developers and contributors to this patch, and to Fabian Groffen for bundling and posting this to mutt-dev. --- configure.ac | 41 ++++++++- hcache.c | 237 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 57e408e4..c7fc4156 100644 --- a/configure.ac +++ b/configure.ac @@ -881,6 +881,7 @@ AC_ARG_WITH(tokyocabinet, AS_HELP_STRING([--without-tokyocabinet],[Don't use tok AC_ARG_WITH(qdbm, AS_HELP_STRING([--without-qdbm],[Don't use qdbm even if it is available])) AC_ARG_WITH(gdbm, AS_HELP_STRING([--without-gdbm],[Don't use gdbm even if it is available])) AC_ARG_WITH(bdb, AS_HELP_STRING([--with-bdb@<:@=DIR@:>@],[Use BerkeleyDB4 if gdbm is not available])) +AC_ARG_WITH(lmdb, AS_HELP_STRING([--with-lmdb@<:@=DIR@:>@],[Use LMDB if gdbm is not available])) db_found=no if test x$enable_hcache = xyes @@ -925,6 +926,15 @@ then db_requested=bdb fi fi + if test -n "$with_lmdb" && test "$with_lmdb" != "no" + then + if test "$db_requested" != "auto" + then + AC_MSG_ERROR([more than one header cache engine requested.]) + else + db_requested=lmdb + fi + fi dnl -- Tokyo Cabinet -- if test "$with_tokyocabinet" != "no" \ @@ -1012,7 +1022,8 @@ then dnl -- BDB -- ac_bdb_prefix="$with_bdb" - if test x$ac_bdb_prefix != xno && test $db_found = no + if test x$with_bdb != xno && test $db_found = no \ + && test "$db_requested" = auto -o "$db_requested" = bdb then if test x$ac_bdb_prefix = xyes || test x$ac_bdb_prefix = x then @@ -1068,6 +1079,34 @@ then fi fi + dnl -- LMDB -- + if test x$with_lmdb != xno && test $db_found = no \ + && test "$db_requested" = auto -o "$db_requested" = lmdb + then + if test "$with_lmdb" != "yes" + then + CPPFLAGS="$CPPFLAGS -I$with_lmdb/include" + LDFLAGS="$LDFLAGS -L$with_lmdb/lib" + fi + saved_LIBS="$LIBS" + LIBS="$LIBS -llmdb" + AC_CACHE_CHECK(for mdb_env_create, ac_cv_mdbenvcreate,[ + ac_cv_mdbenvcreate=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]], [[mdb_env_create(0);]])],[ac_cv_mdbenvcreate=yes],[]) + ]) + LIBS="$saved_LIBS" + if test "$ac_cv_mdbenvcreate" = yes + then + AC_DEFINE(HAVE_LMDB, 1, [LMDB Support]) + MUTTLIBS="$MUTTLIBS -llmdb" + db_found=lmdb + fi + if test "$db_requested" != auto && test "$db_found" != "$db_requested" + then + AC_MSG_ERROR([LMDB could not be used. Check config.log for details.]) + fi + fi + if test $db_found = no then AC_MSG_ERROR([You need Tokyo Cabinet, QDBM, GDBM or Berkeley DB4 for hcache]) diff --git a/hcache.c b/hcache.c index 7eada101..a123016a 100644 --- a/hcache.c +++ b/hcache.c @@ -32,6 +32,12 @@ #include #elif HAVE_DB4 #include +#elif HAVE_LMDB +/* This is the maximum size of the database file (2GiB) which is + * mmap(2)'ed into memory. This limit should be good for ~800,000 + * emails. */ +#define LMDB_DB_SIZE (2 * 1024 * 1024 * 1024) +#include #endif #include @@ -83,6 +89,61 @@ struct header_cache static void mutt_hcache_dbt_init(DBT * dbt, void *data, size_t len); static void mutt_hcache_dbt_empty_init(DBT * dbt); +#elif HAVE_LMDB +enum mdb_txn_mode +{ + txn_uninitialized = 0, + txn_read = 1 << 0, + txn_write = 1 << 1 +}; +struct header_cache +{ + MDB_env *env; + MDB_txn *txn; + MDB_dbi db; + char *folder; + unsigned int crc; + enum mdb_txn_mode txn_mode; +}; + +static int mdb_get_r_txn(header_cache_t *h) +{ + int rc; + + if (h->txn && (h->txn_mode & (txn_read | txn_write)) > 0) + return MDB_SUCCESS; + + if (h->txn) + rc = mdb_txn_renew(h->txn); + else + rc = mdb_txn_begin(h->env, NULL, MDB_RDONLY, &h->txn); + + if (rc == MDB_SUCCESS) + h->txn_mode = txn_read; + + return rc; +} + +static int mdb_get_w_txn(header_cache_t *h) +{ + int rc; + + if (h->txn && (h->txn_mode == txn_write)) + return MDB_SUCCESS; + + if (h->txn) + { + if (h->txn_mode == txn_read) + mdb_txn_reset(h->txn); + h->txn = NULL; + } + + rc = mdb_txn_begin(h->env, NULL, 0, &h->txn); + if (rc == MDB_SUCCESS) + h->txn_mode = txn_write; + + return rc; +} #endif typedef union @@ -732,6 +793,11 @@ mutt_hcache_fetch_raw (header_cache_t *h, const char *filename, #elif HAVE_DB4 DBT key; DBT data; +#elif HAVE_LMDB + MDB_val key; + MDB_val data; + size_t folderlen; + int rc; #endif if (!h) @@ -748,6 +814,37 @@ mutt_hcache_fetch_raw (header_cache_t *h, const char *filename, h->db->get(h->db, NULL, &key, &data, 0); return data.data; +#elif HAVE_LMDB + strncpy(path, h->folder, sizeof (path)); + safe_strcat(path, sizeof (path), filename); + + folderlen = strlen(h->folder); + ksize = folderlen + keylen(path + folderlen); + key.mv_data = (char *)path; + key.mv_size = ksize; + data.mv_data = NULL; + data.mv_size = 0; + rc = mdb_get_r_txn(h); + if (rc != MDB_SUCCESS) { + h->txn = NULL; + fprintf(stderr, "txn_renew: %s\n", mdb_strerror(rc)); + return NULL; + } + rc = mdb_get(h->txn, h->db, &key, &data); + if (rc == MDB_NOTFOUND) { + return NULL; + } + if (rc != MDB_SUCCESS) { + fprintf(stderr, "mdb_get: %s\n", mdb_strerror(rc)); + return NULL; + } + /* Caller frees the data we return, so I MUST make a copy of it */ + + char *d = safe_malloc(data.mv_size); + memcpy(d, data.mv_data, data.mv_size); + + return d; + #else strncpy(path, h->folder, sizeof (path)); safe_strcat(path, sizeof (path), filename); @@ -813,6 +910,11 @@ mutt_hcache_store_raw (header_cache_t* h, const char* filename, void* data, #elif HAVE_DB4 DBT key; DBT databuf; +#elif HAVE_LMDB + MDB_val key; + MDB_val databuf; + size_t folderlen; + int rc; #endif if (!h) @@ -831,6 +933,30 @@ mutt_hcache_store_raw (header_cache_t* h, const char* filename, void* data, databuf.ulen = dlen; return h->db->put(h->db, NULL, &key, &databuf, 0); +#elif HAVE_LMDB + folderlen = strlen(h->folder); + strncpy(path, h->folder, sizeof (path)); + safe_strcat(path, sizeof (path), filename); + ksize = folderlen + keylen(path + folderlen); + + key.mv_data = (char *)path; + key.mv_size = ksize; + databuf.mv_data = data; + databuf.mv_size = dlen; + rc = mdb_get_w_txn(h); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc)); + return rc; + } + rc = mdb_put(h->txn, h->db, &key, &databuf, 0); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "mdb_put: %s\n", mdb_strerror(rc)); + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + return rc; + } + return rc; #else strncpy(path, h->folder, sizeof (path)); safe_strcat(path, sizeof (path), filename); @@ -1134,6 +1260,106 @@ mutt_hcache_delete(header_cache_t *h, const char *filename, mutt_hcache_dbt_init(&key, (void *) filename, keylen(filename)); return h->db->del(h->db, NULL, &key, 0); } +#elif HAVE_LMDB + +static int +hcache_open_lmdb (struct header_cache* h, const char* path) +{ + int rc; + + h->txn = NULL; + + rc = mdb_env_create(&h->env); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_env_create: %s", mdb_strerror(rc)); + return -1; + } + + mdb_env_set_mapsize(h->env, LMDB_DB_SIZE); + + rc = mdb_env_open(h->env, path, MDB_NOSUBDIR, 0644); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_env_open: %s", mdb_strerror(rc)); + goto fail_env; + } + + rc = mdb_get_r_txn(h); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_txn_begin: %s", mdb_strerror(rc)); + goto fail_env; + } + + rc = mdb_dbi_open(h->txn, NULL, MDB_CREATE, &h->db); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "hcache_open_lmdb: mdb_dbi_open: %s", mdb_strerror(rc)); + goto fail_dbi; + } + + mdb_txn_reset(h->txn); + h->txn_mode = txn_uninitialized; + return 0; + +fail_dbi: + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + +fail_env: + mdb_env_close(h->env); + return -1; +} + +void +mutt_hcache_close(header_cache_t *h) +{ + if (!h) + return; + + if (h->txn && h->txn_mode == txn_write) + { + mdb_txn_commit(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + } + + mdb_env_close(h->env); + FREE (&h->folder); + FREE (&h); +} + +int +mutt_hcache_delete(header_cache_t *h, const char *filename, + size_t(*keylen) (const char *fn)) +{ + MDB_val key; + int rc; + + if (!h) + return -1; + + if (filename[0] == '/') + filename++; + + key.mv_data = (char *)filename; + key.mv_size = strlen(filename); + rc = mdb_get_w_txn(h); + if (rc != MDB_SUCCESS) { + fprintf(stderr, "txn_begin: %s\n", mdb_strerror(rc)); + return rc; + } + rc = mdb_del(h->txn, h->db, &key, NULL); + if (rc != MDB_SUCCESS) { + if (rc != MDB_NOTFOUND) { + fprintf(stderr, "mdb_del: %s\n", mdb_strerror(rc)); + mdb_txn_abort(h->txn); + h->txn_mode = txn_uninitialized; + h->txn = NULL; + } + return rc; + } + + return rc; +} #endif header_cache_t * @@ -1151,6 +1377,8 @@ mutt_hcache_open(const char *path, const char *folder, hcache_namer_t namer) hcache_open = hcache_open_gdbm; #elif HAVE_DB4 hcache_open = hcache_open_db4; +#elif HAVE_LMDB + hcache_open = hcache_open_lmdb; #endif /* Calculate the current hcache version from dynamic configuration */ @@ -1188,7 +1416,11 @@ mutt_hcache_open(const char *path, const char *folder, hcache_namer_t namer) hcachever = digest.intval; } +#if HAVE_LMDB + h->db = 0; +#else h->db = NULL; +#endif h->folder = get_foldername(folder); h->crc = hcachever; @@ -1223,6 +1455,11 @@ const char *mutt_hcache_backend (void) { return DB_VERSION_STRING; } +#elif HAVE_LMDB +const char *mutt_hcache_backend (void) +{ + return "lmdb " MDB_VERSION_STRING; +} #elif HAVE_GDBM const char *mutt_hcache_backend (void) { -- 2.50.1