Improve spl slab cache alloc

The policy is to try to allocate with KM_NOSLEEP, which will lead to
memory allocation with GFP_ATOMIC, and if it fails, it will launch
an taskq to expand slab space.

This way it should be able to get better NUMA memory locality and
reduce the overhead of context switch.

Signed-off-by: Jinshan Xiong <jinshan.xiong@gmail.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #551

diff --git a/module/spl/spl-kmem-cache.c b/module/spl/spl-kmem-cache.c
index e3edca5a0f78ab7010c4a44f29ac5300b74d7ae1..99967b14fdb20ad3926e9c7c7a1fd0a4181b35df 100644 (file)
--- a/module/spl/spl-kmem-cache.c
+++ b/module/spl/spl-kmem-cache.c
@@ -1149,15 +1149,13 @@ spl_cache_obj(spl_kmem_cache_t *skc, spl_kmem_slab_t *sks)
  * It is responsible for allocating a new slab, linking it in to the list
  * of partial slabs, and then waking any waiters.
  */
-static void
-spl_cache_grow_work(void *data)
+static int
+__spl_cache_grow(spl_kmem_cache_t *skc, int flags)
 {
-       spl_kmem_alloc_t *ska = (spl_kmem_alloc_t *)data;
-       spl_kmem_cache_t *skc = ska->ska_cache;
        spl_kmem_slab_t *sks;
 
        fstrans_cookie_t cookie = spl_fstrans_mark();
-       sks = spl_slab_alloc(skc, ska->ska_flags);
+       sks = spl_slab_alloc(skc, flags);
        spl_fstrans_unmark(cookie);
 
        spin_lock(&skc->skc_lock);
@@ -1165,15 +1163,29 @@ spl_cache_grow_work(void *data)
                skc->skc_slab_total++;
                skc->skc_obj_total += sks->sks_objs;
                list_add_tail(&sks->sks_list, &skc->skc_partial_list);
+
+               smp_mb__before_atomic();
+               clear_bit(KMC_BIT_DEADLOCKED, &skc->skc_flags);
+               smp_mb__after_atomic();
+               wake_up_all(&skc->skc_waitq);
        }
+       spin_unlock(&skc->skc_lock);
+
+       return (sks == NULL ? -ENOMEM : 0);
+}
+
+static void
+spl_cache_grow_work(void *data)
+{
+       spl_kmem_alloc_t *ska = (spl_kmem_alloc_t *)data;
+       spl_kmem_cache_t *skc = ska->ska_cache;
+
+       (void)__spl_cache_grow(skc, ska->ska_flags);
 
        atomic_dec(&skc->skc_ref);
        smp_mb__before_atomic();
        clear_bit(KMC_BIT_GROWING, &skc->skc_flags);
-       clear_bit(KMC_BIT_DEADLOCKED, &skc->skc_flags);
        smp_mb__after_atomic();
-       wake_up_all(&skc->skc_waitq);
-       spin_unlock(&skc->skc_lock);
 
        kfree(ska);
 }
@@ -1213,6 +1225,21 @@ spl_cache_grow(spl_kmem_cache_t *skc, int flags, void **obj)
                return (rc ? rc : -EAGAIN);
        }
 
+       /*
+        * To reduce the overhead of context switch and improve NUMA locality,
+        * it tries to allocate a new slab in the current process context with
+        * KM_NOSLEEP flag. If it fails, it will launch a new taskq to do the
+        * allocation.
+        *
+        * However, this can't be applied to KVM_VMEM due to a bug that
+        * __vmalloc() doesn't honor gfp flags in page table allocation.
+        */
+       if (!(skc->skc_flags & KMC_VMEM)) {
+               rc = __spl_cache_grow(skc, flags | KM_NOSLEEP);
+               if (rc == 0)
+                       return (0);
+       }
+
        /*
         * This is handled by dispatching a work request to the global work
         * queue.  This allows us to asynchronously allocate a new slab while