Add rw_tryupgrade()

This implementation of rw_tryupgrade() behaves slightly differently
from its counterparts on other platforms.  It drops the RW_READER lock
and then acquires the RW_WRITER lock leaving a small window where no
lock is held.  On other platforms the lock is never released during
the upgrade process.  This is necessary under Linux because the kernel
does not provide an upgrade function.

There are currently no callers in the ZFS code where this change in
behavior is a problem.  In fact, in most cases the code is already
written such that if the upgrade fails the RW_READER lock is dropped
and the caller blocks waiting to acquire the lock as RW_WRITER.

Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Matthew Thode <prometheanfire@gentoo.org>
Closes zfsonlinux/zfs#4388
Closes #534

diff --git a/include/sys/rwlock.h b/include/sys/rwlock.h
index c82764ce9eb1b0889931a72eabc97f95e2229de6..14d097b01ac161b906268c7938d464a485190423 100644 (file)
--- a/include/sys/rwlock.h
+++ b/include/sys/rwlock.h
@@ -208,49 +208,31 @@ RW_LOCK_HELD(krwlock_t *rwp)
        spl_rw_lockdep_on_maybe(rwp);                                   \
 })
 
-#if defined(CONFIG_RWSEM_GENERIC_SPINLOCK)
-#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
-#error spinlock rwsem should not have spin on owner
-#endif
 /*
- * For the generic implementations of rw-semaphores the following is
- * true.  If your semaphore implementation internally represents the
- * semaphore state differently then special case handling is required.
- * - if activity/count is 0 then there are no active readers or writers
- * - if activity/count is +ve then that is the number of active readers
- * - if activity/count is -1 then there is one active writer
+ * This implementation of rw_tryupgrade() behaves slightly differently
+ * from its counterparts on other platforms.  It drops the RW_READER lock
+ * and then acquires the RW_WRITER lock leaving a small window where no
+ * lock is held.  On other platforms the lock is never released during
+ * the upgrade process.  This is necessary under Linux because the kernel
+ * does not provide an upgrade function.
  */
-
-extern void __up_read_locked(struct rw_semaphore *);
-extern int __down_write_trylock_locked(struct rw_semaphore *);
-
 #define rw_tryupgrade(rwp)                                             \
 ({                                                                     \
-       unsigned long _flags_;                                          \
        int _rc_ = 0;                                                   \
                                                                        \
-       spl_rw_lockdep_off_maybe(rwp);                                  \
-       spl_rwsem_lock_irqsave(&SEM(rwp)->wait_lock, _flags_);          \
-       if ((list_empty(&SEM(rwp)->wait_list)) &&                       \
-           (SEM(rwp)->activity == 1)) {                                \
-               __up_read_locked(SEM(rwp));                             \
-               VERIFY(_rc_ = __down_write_trylock_locked(SEM(rwp)));   \
-               (rwp)->rw_owner = current;                              \
+       if (RW_WRITE_HELD(rwp)) {                                       \
+               _rc_ = 1;                                               \
+       } else {                                                        \
+               rw_exit(rwp);                                           \
+               if (rw_tryenter(rwp, RW_WRITER)) {                      \
+                       _rc_ = 1;                                       \
+               } else {                                                \
+                       rw_enter(rwp, RW_READER);                       \
+                       _rc_ = 0;                                       \
+               }                                                       \
        }                                                               \
-       spl_rwsem_unlock_irqrestore(&SEM(rwp)->wait_lock, _flags_);     \
-       spl_rw_lockdep_on_maybe(rwp);                                   \
        _rc_;                                                           \
 })
-#else
-/*
- * rw_tryupgrade() can be implemented correctly but for each supported
- * arch we will need a custom implementation.  For the x86 implementation
- * it looks like a custom cmpxchg() to atomically check and promote the
- * rwsem would be safe.  For now that's not worth the trouble so in this
- * case rw_tryupgrade() has just been disabled.
- */
-#define rw_tryupgrade(rwp)     ({ 0; })
-#endif
 
 int spl_rw_init(void);
 void spl_rw_fini(void);

diff --git a/module/spl/spl-rwlock.c b/module/spl/spl-rwlock.c
index 462a6f0de7761f3ccfeb3f082070c753b326c928..98251c01154d6312b4d30ee8726e9e77ee67baec 100644 (file)
--- a/module/spl/spl-rwlock.c
+++ b/module/spl/spl-rwlock.c
@@ -32,65 +32,5 @@
 
 #define DEBUG_SUBSYSTEM S_RWLOCK
 
-#ifdef CONFIG_RWSEM_GENERIC_SPINLOCK
-
-/*
- * From lib/rwsem-spinlock.c but modified such that the caller is
- * responsible for acquiring and dropping the sem->wait_lock.
- */
-struct rwsem_waiter {
-        struct list_head list;
-        struct task_struct *task;
-        unsigned int flags;
-#define RWSEM_WAITING_FOR_READ  0x00000001
-#define RWSEM_WAITING_FOR_WRITE 0x00000002
-};
-
-/* wake a single writer */
-static struct rw_semaphore *
-__rwsem_wake_one_writer_locked(struct rw_semaphore *sem)
-{
-        struct rwsem_waiter *waiter;
-        struct task_struct *tsk;
-
-        sem->activity = -1;
-
-        waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
-        list_del(&waiter->list);
-
-        tsk = waiter->task;
-        smp_mb();
-        waiter->task = NULL;
-        wake_up_process(tsk);
-        put_task_struct(tsk);
-        return sem;
-}
-
-/* release a read lock on the semaphore */
-void
-__up_read_locked(struct rw_semaphore *sem)
-{
-        if (--sem->activity == 0 && !list_empty(&sem->wait_list))
-                (void)__rwsem_wake_one_writer_locked(sem);
-}
-EXPORT_SYMBOL(__up_read_locked);
-
-/* trylock for writing -- returns 1 if successful, 0 if contention */
-int
-__down_write_trylock_locked(struct rw_semaphore *sem)
-{
-        int ret = 0;
-
-        if (sem->activity == 0 && list_empty(&sem->wait_list)) {
-                sem->activity = -1;
-                ret = 1;
-        }
-
-        return ret;
-}
-EXPORT_SYMBOL(__down_write_trylock_locked);
-
-#endif
-
 int spl_rw_init(void) { return 0; }
 void spl_rw_fini(void) { }

diff --git a/module/splat/splat-rwlock.c b/module/splat/splat-rwlock.c
index 284f77370d37dc245dd2982ac947e362bce12347..abd6a0e6018f12574b0d03d34e2533b1a16b3bc3 100644 (file)
--- a/module/splat/splat-rwlock.c
+++ b/module/splat/splat-rwlock.c
@@ -588,7 +588,6 @@ splat_rwlock_test6(struct file *file, void *arg)
                goto out;
        }
 
-#if defined(CONFIG_RWSEM_GENERIC_SPINLOCK)
        /* With one reader upgrade should never fail. */
        rc = rw_tryupgrade(&rwp->rw_rwlock);
        if (!rc) {
@@ -610,11 +609,6 @@ splat_rwlock_test6(struct file *file, void *arg)
        rc = 0;
        splat_vprint(file, SPLAT_RWLOCK_TEST6_NAME, "%s",
                     "rwlock properly upgraded\n");
-#else
-       rc = 0;
-       splat_vprint(file, SPLAT_RWLOCK_TEST6_NAME, "%s",
-                    "rw_tryupgrade() is disabled for this arch\n");
-#endif
 out:
        rw_exit(&rwp->rw_rwlock);
        rw_destroy(&rwp->rw_rwlock);