blob: 47b0a88c12a23a6d983eff4c3ff0e4bdf1347d2e [file] [log] [blame]
Chris Mason925baed2008-06-25 16:01:30 -04001/*
2 * Copyright (C) 2008 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18#include <linux/sched.h>
19#include <linux/gfp.h>
20#include <linux/pagemap.h>
21#include <linux/spinlock.h>
22#include <linux/page-flags.h>
Chris Mason4881ee52008-07-24 09:51:08 -040023#include <asm/bug.h>
Chris Mason925baed2008-06-25 16:01:30 -040024#include "ctree.h"
25#include "extent_io.h"
26#include "locking.h"
27
Chris Masonb4ce94d2009-02-04 09:25:08 -050028static inline void spin_nested(struct extent_buffer *eb)
29{
30 spin_lock(&eb->lock);
31}
Chris Masond3977122009-01-05 21:25:51 -050032
Chris Masonb4ce94d2009-02-04 09:25:08 -050033/*
34 * Setting a lock to blocking will drop the spinlock and set the
35 * flag that forces other procs who want the lock to wait. After
36 * this you can safely schedule with the lock held.
37 */
38void btrfs_set_lock_blocking(struct extent_buffer *eb)
39{
40 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
41 set_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags);
42 spin_unlock(&eb->lock);
43 }
44 /* exit with the spin lock released and the bit set */
45}
46
47/*
48 * clearing the blocking flag will take the spinlock again.
49 * After this you can't safely schedule
50 */
51void btrfs_clear_lock_blocking(struct extent_buffer *eb)
52{
53 if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
54 spin_nested(eb);
55 clear_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags);
56 smp_mb__after_clear_bit();
57 }
58 /* exit with the spin lock held */
59}
60
61/*
62 * unfortunately, many of the places that currently set a lock to blocking
63 * don't end up blocking for every long, and often they don't block
64 * at all. For a dbench 50 run, if we don't spin one the blocking bit
65 * at all, the context switch rate can jump up to 400,000/sec or more.
66 *
67 * So, we're still stuck with this crummy spin on the blocking bit,
68 * at least until the most common causes of the short blocks
69 * can be dealt with.
70 */
71static int btrfs_spin_on_block(struct extent_buffer *eb)
Chris Mason925baed2008-06-25 16:01:30 -040072{
Chris Masonf9efa9c2008-06-25 16:14:04 -040073 int i;
Chris Masonf9efa9c2008-06-25 16:14:04 -040074 for (i = 0; i < 512; i++) {
75 cpu_relax();
Chris Masonb4ce94d2009-02-04 09:25:08 -050076 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
77 return 1;
78 if (need_resched())
79 break;
Chris Masonf9efa9c2008-06-25 16:14:04 -040080 }
Chris Mason925baed2008-06-25 16:01:30 -040081 return 0;
82}
83
Chris Masonb4ce94d2009-02-04 09:25:08 -050084/*
85 * This is somewhat different from trylock. It will take the
86 * spinlock but if it finds the lock is set to blocking, it will
87 * return without the lock held.
88 *
89 * returns 1 if it was able to take the lock and zero otherwise
90 *
91 * After this call, scheduling is not safe without first calling
92 * btrfs_set_lock_blocking()
93 */
94int btrfs_try_spin_lock(struct extent_buffer *eb)
95{
96 int i;
97
98 spin_nested(eb);
99 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
100 return 1;
101 spin_unlock(&eb->lock);
102
103 /* spin for a bit on the BLOCKING flag */
104 for (i = 0; i < 2; i++) {
105 if (!btrfs_spin_on_block(eb))
106 break;
107
108 spin_nested(eb);
109 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
110 return 1;
111 spin_unlock(&eb->lock);
112 }
113 return 0;
114}
115
116/*
117 * the autoremove wake function will return 0 if it tried to wake up
118 * a process that was already awake, which means that process won't
119 * count as an exclusive wakeup. The waitq code will continue waking
120 * procs until it finds one that was actually sleeping.
121 *
122 * For btrfs, this isn't quite what we want. We want a single proc
123 * to be notified that the lock is ready for taking. If that proc
124 * already happen to be awake, great, it will loop around and try for
125 * the lock.
126 *
127 * So, btrfs_wake_function always returns 1, even when the proc that we
128 * tried to wake up was already awake.
129 */
130static int btrfs_wake_function(wait_queue_t *wait, unsigned mode,
131 int sync, void *key)
132{
133 autoremove_wake_function(wait, mode, sync, key);
134 return 1;
135}
136
137/*
138 * returns with the extent buffer spinlocked.
139 *
140 * This will spin and/or wait as required to take the lock, and then
141 * return with the spinlock held.
142 *
143 * After this call, scheduling is not safe without first calling
144 * btrfs_set_lock_blocking()
145 */
146int btrfs_tree_lock(struct extent_buffer *eb)
147{
148 DEFINE_WAIT(wait);
149 wait.func = btrfs_wake_function;
150
151 while(1) {
152 spin_nested(eb);
153
154 /* nobody is blocking, exit with the spinlock held */
155 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
156 return 0;
157
158 /*
159 * we have the spinlock, but the real owner is blocking.
160 * wait for them
161 */
162 spin_unlock(&eb->lock);
163
164 /*
165 * spin for a bit, and if the blocking flag goes away,
166 * loop around
167 */
168 if (btrfs_spin_on_block(eb))
169 continue;
170
171 prepare_to_wait_exclusive(&eb->lock_wq, &wait,
172 TASK_UNINTERRUPTIBLE);
173
174 if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
175 schedule();
176
177 finish_wait(&eb->lock_wq, &wait);
178 }
179 return 0;
180}
181
182/*
183 * Very quick trylock, this does not spin or schedule. It returns
184 * 1 with the spinlock held if it was able to take the lock, or it
185 * returns zero if it was unable to take the lock.
186 *
187 * After this call, scheduling is not safe without first calling
188 * btrfs_set_lock_blocking()
189 */
Chris Mason925baed2008-06-25 16:01:30 -0400190int btrfs_try_tree_lock(struct extent_buffer *eb)
191{
Chris Masonb4ce94d2009-02-04 09:25:08 -0500192 if (spin_trylock(&eb->lock)) {
193 if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
194 /*
195 * we've got the spinlock, but the real owner is
196 * blocking. Drop the spinlock and return failure
197 */
198 spin_unlock(&eb->lock);
199 return 0;
200 }
201 return 1;
202 }
203 /* someone else has the spinlock giveup */
204 return 0;
Chris Mason925baed2008-06-25 16:01:30 -0400205}
206
207int btrfs_tree_unlock(struct extent_buffer *eb)
208{
Chris Masonb4ce94d2009-02-04 09:25:08 -0500209 /*
210 * if we were a blocking owner, we don't have the spinlock held
211 * just clear the bit and look for waiters
212 */
213 if (test_and_clear_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
214 smp_mb__after_clear_bit();
215 else
216 spin_unlock(&eb->lock);
217
218 if (waitqueue_active(&eb->lock_wq))
219 wake_up(&eb->lock_wq);
Chris Mason925baed2008-06-25 16:01:30 -0400220 return 0;
221}
222
Chris Masonb9447ef82009-03-09 11:45:38 -0400223void btrfs_assert_tree_locked(struct extent_buffer *eb)
Chris Mason925baed2008-06-25 16:01:30 -0400224{
Chris Masonb9447ef82009-03-09 11:45:38 -0400225 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
226 assert_spin_locked(&eb->lock);
Chris Mason925baed2008-06-25 16:01:30 -0400227}