Use a dedicated taskq for vdev_file

Originally, vdev_file used system_taskq. This would cause a deadlock,
especially on system with few CPUs. The reason is that the prefetcher
threads, which are on system_taskq, will sometimes be blocked waiting
for I/O to finish. If the prefetcher threads consume all the tasks in
system_taskq, the I/O cannot be served and thus results in a deadlock.

We fix this by creating a dedicated vdev_file_taskq for vdev_file I/O.

Signed-off-by: Chunwei Chen <tuxoko@gmail.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #2270

diff --git a/include/sys/vdev_file.h b/include/sys/vdev_file.h
index b4d40f9df29fe2e969699b15cb5d90b938da49b4..cf38a86afeabae1ec5c3587ed5ac247c41a88f4c 100644 (file)
--- a/include/sys/vdev_file.h
+++ b/include/sys/vdev_file.h
@@ -39,6 +39,9 @@ typedef struct vdev_file {
        vnode_t         *vf_vnode;
 } vdev_file_t;
 
+extern void vdev_file_init(void);
+extern void vdev_file_fini(void);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c
index cdbe5a522a194cbec816abf294e8f1a6dd07b61b..935a617965061a6d76f9c1746a1dad25993dad94 100644 (file)
--- a/module/zfs/spa_misc.c
+++ b/module/zfs/spa_misc.c
@@ -34,6 +34,7 @@
 #include <sys/zap.h>
 #include <sys/zil.h>
 #include <sys/vdev_impl.h>
+#include <sys/vdev_file.h>
 #include <sys/metaslab.h>
 #include <sys/uberblock_impl.h>
 #include <sys/txg.h>
@@ -1660,6 +1661,7 @@ spa_init(int mode)
        dmu_init();
        zil_init();
        vdev_cache_stat_init();
+       vdev_file_init();
        zfs_prop_init();
        zpool_prop_init();
        zpool_feature_init();
@@ -1674,6 +1676,7 @@ spa_fini(void)
 
        spa_evict_all();
 
+       vdev_file_fini();
        vdev_cache_stat_fini();
        zil_fini();
        dmu_fini();

diff --git a/module/zfs/vdev_file.c b/module/zfs/vdev_file.c
index 858582acad74b97db7bbabbb43cf312e389d4812..3305802241d7c63ebba0baba792551009643c244 100644 (file)
--- a/module/zfs/vdev_file.c
+++ b/module/zfs/vdev_file.c
@@ -36,6 +36,8 @@
  * Virtual device vector for files.
  */
 
+static taskq_t *vdev_file_taskq;
+
 static void
 vdev_file_hold(vdev_t *vd)
 {
@@ -184,7 +186,7 @@ vdev_file_io_start(zio_t *zio)
                return (ZIO_PIPELINE_CONTINUE);
        }
 
-       VERIFY3U(taskq_dispatch(system_taskq, vdev_file_io_strategy, zio,
+       VERIFY3U(taskq_dispatch(vdev_file_taskq, vdev_file_io_strategy, zio,
            TQ_PUSHPAGE), !=, 0);
 
        return (ZIO_PIPELINE_STOP);
@@ -209,6 +211,21 @@ vdev_ops_t vdev_file_ops = {
        B_TRUE                  /* leaf vdev */
 };
 
+void
+vdev_file_init(void)
+{
+       vdev_file_taskq = taskq_create("vdev_file_taskq", 100, minclsyspri,
+           max_ncpus, INT_MAX, TASKQ_PREPOPULATE | TASKQ_THREADS_CPU_PCT);
+
+       VERIFY(vdev_file_taskq);
+}
+
+void
+vdev_file_fini(void)
+{
+       taskq_destroy(vdev_file_taskq);
+}
+
 /*
  * From userland we access disks just like files.
  */