Add spl_kmem_cache_expire module option

Cache aging was implemented because it was part of the default Solaris
kmem_cache behavior.  The idea is that per-cpu objects which haven't been
accessed in several seconds should be returned to the cache.  On the other
hand Linux slabs never move objects back to the slabs unless there is
memory pressure on the system.

This behavior is now configurable through the 'spl_kmem_cache_expire'
module option.  The value is a bit mask with the following meaning.

  0x1 - Solaris style cache aging eviction is enabled.
  0x2 - Linux style low memory eviction is enabled.

Both methods may be safely enabled simultaneously, but by default
both are disabled.  It has never been clear if the kmem cache aging
(which has been around from day one) actually does any good.  It has
however been the source of numerous bugs so I wouldn't mind retiring
it entirely.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes zfsonlinux/zfs#1227
Closes #210

diff --git a/include/sys/kmem.h b/include/sys/kmem.h
index 6904bec3f18a3da7e232585ca7d803e19157eed7..d5d3061a59c4158bd33930370d17378afbdbc45b 100644 (file)
--- a/include/sys/kmem.h
+++ b/include/sys/kmem.h
@@ -377,9 +377,13 @@ typedef enum kmem_cbrc {
 #define KMC_ALLOC              (1 << KMC_BIT_ALLOC)
 #define KMC_MAX                        (1 << KMC_BIT_MAX)
 
-#define KMC_REAP_CHUNK                 INT_MAX
-#define KMC_DEFAULT_SEEKS              1
+#define KMC_REAP_CHUNK         INT_MAX
+#define KMC_DEFAULT_SEEKS      1
 
+#define KMC_EXPIRE_AGE         0x1     /* Due to age */
+#define KMC_EXPIRE_MEM         0x2     /* Due to low memory */
+
+extern unsigned int spl_kmem_cache_expire;
 extern struct list_head spl_kmem_cache_list;
 extern struct rw_semaphore spl_kmem_cache_sem;
 

diff --git a/module/spl/spl-kmem.c b/module/spl/spl-kmem.c
index 6cd3acb720365748040df8fdf3d8b9549cf94c62..413aa1245132fa795c49ac32a91cbbc4ea44d16f 100644 (file)
--- a/module/spl/spl-kmem.c
+++ b/module/spl/spl-kmem.c
@@ -33,6 +33,19 @@
 
 #define SS_DEBUG_SUBSYS SS_KMEM
 
+/*
+ * Cache expiration was implemented because it was part of the default Solaris
+ * kmem_cache behavior.  The idea is that per-cpu objects which haven't been
+ * accessed in several seconds should be returned to the cache.  On the other
+ * hand Linux slabs never move objects back to the slabs unless there is
+ * memory pressure on the system.  By default both methods are disabled, but
+ * may be enabled by setting KMC_EXPIRE_AGE or KMC_EXPIRE_MEM.
+ */
+unsigned int spl_kmem_cache_expire = 0;
+EXPORT_SYMBOL(spl_kmem_cache_expire);
+module_param(spl_kmem_cache_expire, uint, 0644);
+MODULE_PARM_DESC(spl_kmem_cache_expire, "By age (0x1) or low memory (0x2)");
+
 /*
  * The minimum amount of memory measured in pages to be free at all
  * times on the system.  This is similar to Linux's zone->pages_min
@@ -1317,6 +1330,10 @@ spl_cache_age(void *data)
 
        ASSERT(skc->skc_magic == SKC_MAGIC);
 
+       /* Dynamically disabled at run time */
+       if (!(spl_kmem_cache_expire & KMC_EXPIRE_AGE))
+               return;
+
        atomic_inc(&skc->skc_ref);
        spl_on_each_cpu(spl_magazine_age, skc, 1);
        spl_slab_reclaim(skc, skc->skc_reap, 0);
@@ -1609,9 +1626,10 @@ spl_kmem_cache_create(char *name, size_t size, size_t align,
        if (rc)
                SGOTO(out, rc);
 
-       skc->skc_taskqid = taskq_dispatch_delay(spl_kmem_cache_taskq,
-           spl_cache_age, skc, TQ_SLEEP,
-           ddi_get_lbolt() + skc->skc_delay / 3 * HZ);
+       if (spl_kmem_cache_expire & KMC_EXPIRE_AGE)
+               skc->skc_taskqid = taskq_dispatch_delay(spl_kmem_cache_taskq,
+                   spl_cache_age, skc, TQ_SLEEP,
+                   ddi_get_lbolt() + skc->skc_delay / 3 * HZ);
 
        down_write(&spl_kmem_cache_sem);
        list_add_tail(&skc->skc_list, &spl_kmem_cache_list);
@@ -2168,7 +2186,17 @@ spl_kmem_cache_reap_now(spl_kmem_cache_t *skc, int count)
                } while (do_reclaim);
        }
 
-       /* Reclaim from the cache, ignoring it's age and delay. */
+       /* Reclaim from the magazine then the slabs ignoring age and delay. */
+       if (spl_kmem_cache_expire & KMC_EXPIRE_MEM) {
+               spl_kmem_magazine_t *skm;
+               int i;
+
+               for_each_online_cpu(i) {
+                       skm = skc->skc_mag[i];
+                       spl_cache_flush(skc, skm, skm->skm_avail);
+               }
+       }
+
        spl_slab_reclaim(skc, count, 1);
        clear_bit(KMC_BIT_REAPING, &skc->skc_flags);
        smp_mb__after_clear_bit();

diff --git a/module/splat/splat-kmem.c b/module/splat/splat-kmem.c
index 30750aa0111979ae75ae1bbc98c766953ecf1f0b..789e75e5613bc756e119dc3ff477c6d39824bbd2 100644 (file)
--- a/module/splat/splat-kmem.c
+++ b/module/splat/splat-kmem.c
@@ -810,8 +810,13 @@ splat_kmem_test8(struct file *file, void *arg)
 {
        kmem_cache_priv_t *kcp;
        kmem_cache_thread_t *kct;
+       unsigned int spl_kmem_cache_expire_old;
        int i, rc = 0;
 
+       /* Enable cache aging just for this test if it is disabled */
+       spl_kmem_cache_expire_old = spl_kmem_cache_expire;
+       spl_kmem_cache_expire = KMC_EXPIRE_AGE;
+
        kcp = splat_kmem_cache_test_kcp_alloc(file, SPLAT_KMEM_TEST8_NAME,
                                              256, 0, 0);
        if (!kcp) {
@@ -882,6 +887,8 @@ out_cache:
 out_kcp:
        splat_kmem_cache_test_kcp_free(kcp);
 out:
+       spl_kmem_cache_expire = spl_kmem_cache_expire_old;
+
        return rc;
 }
 
@@ -898,8 +905,13 @@ splat_kmem_test9(struct file *file, void *arg)
 {
        kmem_cache_priv_t *kcp;
        kmem_cache_thread_t *kct;
+       unsigned int spl_kmem_cache_expire_old;
        int i, rc = 0, count = SPLAT_KMEM_OBJ_COUNT * 128;
 
+       /* Enable cache aging just for this test if it is disabled */
+       spl_kmem_cache_expire_old = spl_kmem_cache_expire;
+       spl_kmem_cache_expire = KMC_EXPIRE_AGE;
+
        kcp = splat_kmem_cache_test_kcp_alloc(file, SPLAT_KMEM_TEST9_NAME,
                                              256, 0, 0);
        if (!kcp) {
@@ -968,6 +980,8 @@ out_cache:
 out_kcp:
        splat_kmem_cache_test_kcp_free(kcp);
 out:
+       spl_kmem_cache_expire = spl_kmem_cache_expire_old;
+
        return rc;
 }