Add hooks for disabling direct reclaim

The port of XFS to Linux introduced a thread-specific PF_FSTRANS bit
that is used to mark contexts which are processing transactions.  When
set, allocations in this context can dip into kernel memory reserves
to avoid deadlocks during writeback.  Linux 3.9 provided the additional
PF_MEMALLOC_NOIO for disabling __GFP_IO in page allocations, which XFS
began using in 3.15.

This patch implements hooks for marking transactions via PF_FSTRANS.
When an allocation is performed in the context of PF_FSTRANS, any
KM_SLEEP allocation is transparently converted to a GFP_NOIO allocation.

Additionally, when using a Linux 3.9 or newer kernel, it will set
PF_MEMALLOC_NOIO to prevent direct reclaim from entering pageout() on
on any KM_PUSHPAGE or KM_NOSLEEP allocation.  This effectively allows
the spl_vmalloc() helper function to be used safely in a thread which
is responsible for IO.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>

diff --git a/include/sys/kmem.h b/include/sys/kmem.h
index 045d07c2c706f9681bd47c01885a77c82234d752..8d5e729373fa265a14f393280df89d2aecbed9b3 100644 (file)
--- a/include/sys/kmem.h
+++ b/include/sys/kmem.h
@@ -25,6 +25,7 @@
 #ifndef _SPL_KMEM_H
 #define        _SPL_KMEM_H
 
+#include <sys/debug.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
 
@@ -72,6 +73,39 @@ kmem_flags_convert(int flags)
        return (lflags);
 }
 
+typedef struct {
+       struct task_struct *fstrans_thread;
+       unsigned int saved_flags;
+} fstrans_cookie_t;
+
+static inline fstrans_cookie_t
+spl_fstrans_mark(void)
+{
+       fstrans_cookie_t cookie;
+
+       cookie.fstrans_thread = current;
+       cookie.saved_flags = current->flags & PF_FSTRANS;
+       current->flags |= PF_FSTRANS;
+
+       return (cookie);
+}
+
+static inline void
+spl_fstrans_unmark(fstrans_cookie_t cookie)
+{
+       ASSERT3P(cookie.fstrans_thread, ==, current);
+       ASSERT(current->flags & PF_FSTRANS);
+
+       current->flags &= ~(PF_FSTRANS);
+       current->flags |= cookie.saved_flags;
+}
+
+static inline int
+spl_fstrans_check(void)
+{
+       return (current->flags & PF_FSTRANS);
+}
+
 #ifdef HAVE_ATOMIC64_T
 #define        kmem_alloc_used_add(size)       atomic64_add(size, &kmem_alloc_used)
 #define        kmem_alloc_used_sub(size)       atomic64_sub(size, &kmem_alloc_used)

diff --git a/include/sys/vmem.h b/include/sys/vmem.h
index 6eb2c67699ff48327b059664da7512d9319db7e9..8aadc9d03b300e8f611261ab8f798b8c276af986 100644 (file)
--- a/include/sys/vmem.h
+++ b/include/sys/vmem.h
@@ -36,6 +36,7 @@ extern vmem_t *zio_alloc_arena;
 extern vmem_t *zio_arena;
 
 extern size_t vmem_size(vmem_t *vmp, int typemask);
+extern void *spl_vmalloc(unsigned long size, gfp_t lflags, pgprot_t prot);
 
 /*
  * Memory allocation interfaces

diff --git a/module/spl/spl-kmem-cache.c b/module/spl/spl-kmem-cache.c
index 9a8ccfe4212bfd785ddcadd8eba30128052a8798..f8edb44a9e04cec8491a0c5264f0630809d1a015 100644 (file)
--- a/module/spl/spl-kmem-cache.c
+++ b/module/spl/spl-kmem-cache.c
@@ -153,7 +153,7 @@ kv_alloc(spl_kmem_cache_t *skc, int size, int flags)
        if (skc->skc_flags & KMC_KMEM)
                ptr = (void *)__get_free_pages(lflags, get_order(size));
        else
-               ptr = __vmalloc(size, lflags | __GFP_HIGHMEM, PAGE_KERNEL);
+               ptr = spl_vmalloc(size, lflags | __GFP_HIGHMEM, PAGE_KERNEL);
 
        /* Resulting allocated memory will be page aligned */
        ASSERT(IS_P2ALIGNED(ptr, PAGE_SIZE));
@@ -1098,7 +1098,9 @@ spl_cache_grow_work(void *data)
        sks = spl_slab_alloc(skc, ska->ska_flags);
        memalloc_noio_restore(noio_flag);
 #else
+       fstrans_cookie_t cookie = spl_fstrans_mark();
        sks = spl_slab_alloc(skc, ska->ska_flags);
+       spl_fstrans_unmark(cookie);
 #endif
        spin_lock(&skc->skc_lock);
        if (sks) {

diff --git a/module/spl/spl-kmem.c b/module/spl/spl-kmem.c
index 4cd7cdbeee612e9637886ce93b7226b5db756710..914f0fbf7d2fd88ffad1882b44fdbb5ed360ed15 100644 (file)
--- a/module/spl/spl-kmem.c
+++ b/module/spl/spl-kmem.c
@@ -184,7 +184,7 @@ spl_kmem_alloc_impl(size_t size, int flags, int node)
                 */
                if (unlikely(size > spl_kmem_alloc_max)) {
                        if (flags & KM_VMEM) {
-                               ptr = __vmalloc(size, lflags, PAGE_KERNEL);
+                               ptr = spl_vmalloc(size, lflags, PAGE_KERNEL);
                        } else {
                                return (NULL);
                        }

diff --git a/module/spl/spl-vmem.c b/module/spl/spl-vmem.c
index e177988a7e2d06a509c1815058fc701e89914307..bca27f263d91cd2d82b5bfc074cddfe6084e3a4e 100644 (file)
--- a/module/spl/spl-vmem.c
+++ b/module/spl/spl-vmem.c
@@ -97,6 +97,31 @@ spl_vmem_free(const void *buf, size_t size)
 }
 EXPORT_SYMBOL(spl_vmem_free);
 
+/*
+ * Public vmalloc() interface designed to be safe to be called during I/O.
+ */
+void *
+spl_vmalloc(unsigned long size, gfp_t lflags, pgprot_t prot)
+{
+#if defined(PF_MEMALLOC_NOIO)
+       void *ptr;
+       unsigned noio_flag = 0;
+
+       if (spl_fstrans_check())
+               noio_flag = memalloc_noio_save();
+
+       ptr =  __vmalloc(size, lflags, prot);
+
+       if (spl_fstrans_check())
+               memalloc_noio_restore(noio_flag);
+
+       return (ptr);
+#else
+       return (__vmalloc(size, lflags, prot));
+#endif
+}
+EXPORT_SYMBOL(spl_vmalloc);
+
 int
 spl_vmem_init(void)
 {