Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 1 | /* |
Waiman Long | c7114b4 | 2015-05-11 13:57:11 -0400 | [diff] [blame] | 2 | * Queued read/write locks |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License as published by |
| 6 | * the Free Software Foundation; either version 2 of the License, or |
| 7 | * (at your option) any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU General Public License for more details. |
| 13 | * |
| 14 | * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P. |
| 15 | * |
| 16 | * Authors: Waiman Long <waiman.long@hp.com> |
| 17 | */ |
| 18 | #include <linux/smp.h> |
| 19 | #include <linux/bug.h> |
| 20 | #include <linux/cpumask.h> |
| 21 | #include <linux/percpu.h> |
| 22 | #include <linux/hardirq.h> |
Babu Moger | 9ab6055 | 2017-05-24 17:55:10 -0600 | [diff] [blame] | 23 | #include <linux/spinlock.h> |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 24 | #include <asm/qrwlock.h> |
| 25 | |
| 26 | /** |
Waiman Long | f7d71f2 | 2015-06-19 11:50:00 -0400 | [diff] [blame] | 27 | * queued_read_lock_slowpath - acquire read lock of a queue rwlock |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 28 | * @lock: Pointer to queue rwlock structure |
| 29 | */ |
Will Deacon | b519b56 | 2017-10-12 13:20:49 +0100 | [diff] [blame] | 30 | void queued_read_lock_slowpath(struct qrwlock *lock) |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 31 | { |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 32 | /* |
| 33 | * Readers come here when they cannot get the lock without waiting |
| 34 | */ |
| 35 | if (unlikely(in_interrupt())) { |
| 36 | /* |
Waiman Long | 0e06e5be | 2015-06-19 11:50:01 -0400 | [diff] [blame] | 37 | * Readers in interrupt context will get the lock immediately |
Will Deacon | b519b56 | 2017-10-12 13:20:49 +0100 | [diff] [blame] | 38 | * if the writer is just waiting (not holding the lock yet), |
| 39 | * so spin with ACQUIRE semantics until the lock is available |
| 40 | * without waiting in the queue. |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 41 | */ |
Will Deacon | d133166 | 2017-10-12 13:20:51 +0100 | [diff] [blame] | 42 | atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 43 | return; |
| 44 | } |
| 45 | atomic_sub(_QR_BIAS, &lock->cnts); |
| 46 | |
| 47 | /* |
| 48 | * Put the reader into the wait queue |
| 49 | */ |
Davidlohr Bueso | 6e1e519 | 2015-09-14 00:37:22 -0700 | [diff] [blame] | 50 | arch_spin_lock(&lock->wait_lock); |
Will Deacon | b519b56 | 2017-10-12 13:20:49 +0100 | [diff] [blame] | 51 | atomic_add(_QR_BIAS, &lock->cnts); |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 52 | |
| 53 | /* |
Will Deacon | 77e430e | 2015-08-06 17:54:42 +0100 | [diff] [blame] | 54 | * The ACQUIRE semantics of the following spinning code ensure |
| 55 | * that accesses can't leak upwards out of our subsequent critical |
| 56 | * section in the case that the lock is currently held for write. |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 57 | */ |
Will Deacon | d133166 | 2017-10-12 13:20:51 +0100 | [diff] [blame] | 58 | atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED)); |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 59 | |
| 60 | /* |
| 61 | * Signal the next one in queue to become queue head |
| 62 | */ |
Davidlohr Bueso | 6e1e519 | 2015-09-14 00:37:22 -0700 | [diff] [blame] | 63 | arch_spin_unlock(&lock->wait_lock); |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 64 | } |
Waiman Long | f7d71f2 | 2015-06-19 11:50:00 -0400 | [diff] [blame] | 65 | EXPORT_SYMBOL(queued_read_lock_slowpath); |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 66 | |
| 67 | /** |
Waiman Long | f7d71f2 | 2015-06-19 11:50:00 -0400 | [diff] [blame] | 68 | * queued_write_lock_slowpath - acquire write lock of a queue rwlock |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 69 | * @lock : Pointer to queue rwlock structure |
| 70 | */ |
Waiman Long | f7d71f2 | 2015-06-19 11:50:00 -0400 | [diff] [blame] | 71 | void queued_write_lock_slowpath(struct qrwlock *lock) |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 72 | { |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 73 | /* Put the writer into the wait queue */ |
Davidlohr Bueso | 6e1e519 | 2015-09-14 00:37:22 -0700 | [diff] [blame] | 74 | arch_spin_lock(&lock->wait_lock); |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 75 | |
| 76 | /* Try to acquire the lock directly if no reader is present */ |
| 77 | if (!atomic_read(&lock->cnts) && |
Will Deacon | 77e430e | 2015-08-06 17:54:42 +0100 | [diff] [blame] | 78 | (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0)) |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 79 | goto unlock; |
| 80 | |
Will Deacon | d133166 | 2017-10-12 13:20:51 +0100 | [diff] [blame] | 81 | /* Set the waiting flag to notify readers that a writer is pending */ |
| 82 | atomic_add(_QW_WAITING, &lock->cnts); |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 83 | |
Will Deacon | d133166 | 2017-10-12 13:20:51 +0100 | [diff] [blame] | 84 | /* When no more readers or writers, set the locked flag */ |
Will Deacon | b519b56 | 2017-10-12 13:20:49 +0100 | [diff] [blame] | 85 | do { |
| 86 | atomic_cond_read_acquire(&lock->cnts, VAL == _QW_WAITING); |
| 87 | } while (atomic_cmpxchg_relaxed(&lock->cnts, _QW_WAITING, |
| 88 | _QW_LOCKED) != _QW_WAITING); |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 89 | unlock: |
Davidlohr Bueso | 6e1e519 | 2015-09-14 00:37:22 -0700 | [diff] [blame] | 90 | arch_spin_unlock(&lock->wait_lock); |
Waiman Long | 70af2f8 | 2014-02-03 13:18:49 +0100 | [diff] [blame] | 91 | } |
Waiman Long | f7d71f2 | 2015-06-19 11:50:00 -0400 | [diff] [blame] | 92 | EXPORT_SYMBOL(queued_write_lock_slowpath); |