Maarten Lankhorst | 786d725 | 2013-06-27 13:48:16 +0200 | [diff] [blame] | 1 | /* |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 2 | * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst) |
Maarten Lankhorst | 786d725 | 2013-06-27 13:48:16 +0200 | [diff] [blame] | 3 | * |
| 4 | * Based on bo.c which bears the following copyright notice, |
| 5 | * but is dual licensed: |
| 6 | * |
| 7 | * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA |
| 8 | * All Rights Reserved. |
| 9 | * |
| 10 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 11 | * copy of this software and associated documentation files (the |
| 12 | * "Software"), to deal in the Software without restriction, including |
| 13 | * without limitation the rights to use, copy, modify, merge, publish, |
| 14 | * distribute, sub license, and/or sell copies of the Software, and to |
| 15 | * permit persons to whom the Software is furnished to do so, subject to |
| 16 | * the following conditions: |
| 17 | * |
| 18 | * The above copyright notice and this permission notice (including the |
| 19 | * next paragraph) shall be included in all copies or substantial portions |
| 20 | * of the Software. |
| 21 | * |
| 22 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 23 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 24 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL |
| 25 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, |
| 26 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR |
| 27 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE |
| 28 | * USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 29 | * |
| 30 | **************************************************************************/ |
| 31 | /* |
| 32 | * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> |
| 33 | */ |
| 34 | |
| 35 | #include <linux/reservation.h> |
| 36 | #include <linux/export.h> |
| 37 | |
Rob Clark | dad6c39 | 2016-03-31 16:26:51 -0400 | [diff] [blame] | 38 | /** |
| 39 | * DOC: Reservation Object Overview |
| 40 | * |
| 41 | * The reservation object provides a mechanism to manage shared and |
| 42 | * exclusive fences associated with a buffer. A reservation object |
| 43 | * can have attached one exclusive fence (normally associated with |
| 44 | * write operations) or N shared fences (read operations). The RCU |
| 45 | * mechanism is used to protect read access to fences from locked |
| 46 | * write-side updates. |
| 47 | */ |
| 48 | |
Maarten Lankhorst | 786d725 | 2013-06-27 13:48:16 +0200 | [diff] [blame] | 49 | DEFINE_WW_CLASS(reservation_ww_class); |
| 50 | EXPORT_SYMBOL(reservation_ww_class); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 51 | |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 52 | struct lock_class_key reservation_seqcount_class; |
| 53 | EXPORT_SYMBOL(reservation_seqcount_class); |
| 54 | |
| 55 | const char reservation_seqcount_string[] = "reservation_seqcount"; |
| 56 | EXPORT_SYMBOL(reservation_seqcount_string); |
Rob Clark | dad6c39 | 2016-03-31 16:26:51 -0400 | [diff] [blame] | 57 | |
| 58 | /** |
| 59 | * reservation_object_reserve_shared - Reserve space to add a shared |
| 60 | * fence to a reservation_object. |
| 61 | * @obj: reservation object |
| 62 | * |
| 63 | * Should be called before reservation_object_add_shared_fence(). Must |
| 64 | * be called with obj->lock held. |
| 65 | * |
| 66 | * RETURNS |
| 67 | * Zero for success, or -errno |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 68 | */ |
| 69 | int reservation_object_reserve_shared(struct reservation_object *obj) |
| 70 | { |
| 71 | struct reservation_object_list *fobj, *old; |
| 72 | u32 max; |
| 73 | |
| 74 | old = reservation_object_get_list(obj); |
| 75 | |
| 76 | if (old && old->shared_max) { |
| 77 | if (old->shared_count < old->shared_max) { |
| 78 | /* perform an in-place update */ |
| 79 | kfree(obj->staged); |
| 80 | obj->staged = NULL; |
| 81 | return 0; |
| 82 | } else |
| 83 | max = old->shared_max * 2; |
| 84 | } else |
| 85 | max = 4; |
| 86 | |
| 87 | /* |
| 88 | * resize obj->staged or allocate if it doesn't exist, |
| 89 | * noop if already correct size |
| 90 | */ |
| 91 | fobj = krealloc(obj->staged, offsetof(typeof(*fobj), shared[max]), |
| 92 | GFP_KERNEL); |
| 93 | if (!fobj) |
| 94 | return -ENOMEM; |
| 95 | |
| 96 | obj->staged = fobj; |
| 97 | fobj->shared_max = max; |
| 98 | return 0; |
| 99 | } |
| 100 | EXPORT_SYMBOL(reservation_object_reserve_shared); |
| 101 | |
| 102 | static void |
| 103 | reservation_object_add_shared_inplace(struct reservation_object *obj, |
| 104 | struct reservation_object_list *fobj, |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 105 | struct dma_fence *fence) |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 106 | { |
Christian König | ca25fe5 | 2017-11-14 15:24:36 +0100 | [diff] [blame] | 107 | struct dma_fence *signaled = NULL; |
| 108 | u32 i, signaled_idx; |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 109 | |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 110 | dma_fence_get(fence); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 111 | |
| 112 | preempt_disable(); |
| 113 | write_seqcount_begin(&obj->seq); |
| 114 | |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 115 | for (i = 0; i < fobj->shared_count; ++i) { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 116 | struct dma_fence *old_fence; |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 117 | |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 118 | old_fence = rcu_dereference_protected(fobj->shared[i], |
| 119 | reservation_object_held(obj)); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 120 | |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 121 | if (old_fence->context == fence->context) { |
| 122 | /* memory barrier is added by write_seqcount_begin */ |
| 123 | RCU_INIT_POINTER(fobj->shared[i], fence); |
| 124 | write_seqcount_end(&obj->seq); |
| 125 | preempt_enable(); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 126 | |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 127 | dma_fence_put(old_fence); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 128 | return; |
| 129 | } |
Christian König | ca25fe5 | 2017-11-14 15:24:36 +0100 | [diff] [blame] | 130 | |
| 131 | if (!signaled && dma_fence_is_signaled(old_fence)) { |
| 132 | signaled = old_fence; |
| 133 | signaled_idx = i; |
| 134 | } |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 135 | } |
| 136 | |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 137 | /* |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 138 | * memory barrier is added by write_seqcount_begin, |
| 139 | * fobj->shared_count is protected by this lock too |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 140 | */ |
Christian König | ca25fe5 | 2017-11-14 15:24:36 +0100 | [diff] [blame] | 141 | if (signaled) { |
| 142 | RCU_INIT_POINTER(fobj->shared[signaled_idx], fence); |
| 143 | } else { |
| 144 | RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); |
| 145 | fobj->shared_count++; |
| 146 | } |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 147 | |
| 148 | write_seqcount_end(&obj->seq); |
| 149 | preempt_enable(); |
Christian König | ca25fe5 | 2017-11-14 15:24:36 +0100 | [diff] [blame] | 150 | |
| 151 | dma_fence_put(signaled); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 152 | } |
| 153 | |
| 154 | static void |
| 155 | reservation_object_add_shared_replace(struct reservation_object *obj, |
| 156 | struct reservation_object_list *old, |
| 157 | struct reservation_object_list *fobj, |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 158 | struct dma_fence *fence) |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 159 | { |
Christian König | 4d9c62e | 2017-11-14 15:24:35 +0100 | [diff] [blame] | 160 | unsigned i, j, k; |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 161 | |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 162 | dma_fence_get(fence); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 163 | |
| 164 | if (!old) { |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 165 | RCU_INIT_POINTER(fobj->shared[0], fence); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 166 | fobj->shared_count = 1; |
| 167 | goto done; |
| 168 | } |
| 169 | |
| 170 | /* |
| 171 | * no need to bump fence refcounts, rcu_read access |
| 172 | * requires the use of kref_get_unless_zero, and the |
| 173 | * references from the old struct are carried over to |
| 174 | * the new. |
| 175 | */ |
Christian König | 4d9c62e | 2017-11-14 15:24:35 +0100 | [diff] [blame] | 176 | for (i = 0, j = 0, k = fobj->shared_max; i < old->shared_count; ++i) { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 177 | struct dma_fence *check; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 178 | |
| 179 | check = rcu_dereference_protected(old->shared[i], |
| 180 | reservation_object_held(obj)); |
| 181 | |
Christian König | 4d9c62e | 2017-11-14 15:24:35 +0100 | [diff] [blame] | 182 | if (check->context == fence->context || |
| 183 | dma_fence_is_signaled(check)) |
| 184 | RCU_INIT_POINTER(fobj->shared[--k], check); |
| 185 | else |
| 186 | RCU_INIT_POINTER(fobj->shared[j++], check); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 187 | } |
Christian König | 4d9c62e | 2017-11-14 15:24:35 +0100 | [diff] [blame] | 188 | fobj->shared_count = j; |
| 189 | RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); |
| 190 | fobj->shared_count++; |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 191 | |
| 192 | done: |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 193 | preempt_disable(); |
| 194 | write_seqcount_begin(&obj->seq); |
| 195 | /* |
| 196 | * RCU_INIT_POINTER can be used here, |
| 197 | * seqcount provides the necessary barriers |
| 198 | */ |
| 199 | RCU_INIT_POINTER(obj->fence, fobj); |
| 200 | write_seqcount_end(&obj->seq); |
| 201 | preempt_enable(); |
| 202 | |
Christian König | 4d9c62e | 2017-11-14 15:24:35 +0100 | [diff] [blame] | 203 | if (!old) |
| 204 | return; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 205 | |
Christian König | 4d9c62e | 2017-11-14 15:24:35 +0100 | [diff] [blame] | 206 | /* Drop the references to the signaled fences */ |
| 207 | for (i = k; i < fobj->shared_max; ++i) { |
| 208 | struct dma_fence *f; |
| 209 | |
| 210 | f = rcu_dereference_protected(fobj->shared[i], |
| 211 | reservation_object_held(obj)); |
| 212 | dma_fence_put(f); |
| 213 | } |
| 214 | kfree_rcu(old, rcu); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 215 | } |
| 216 | |
Rob Clark | dad6c39 | 2016-03-31 16:26:51 -0400 | [diff] [blame] | 217 | /** |
| 218 | * reservation_object_add_shared_fence - Add a fence to a shared slot |
| 219 | * @obj: the reservation object |
| 220 | * @fence: the shared fence to add |
| 221 | * |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 222 | * Add a fence to a shared slot, obj->lock must be held, and |
Rob Clark | f5bef0b | 2016-08-19 16:55:34 -0400 | [diff] [blame] | 223 | * reservation_object_reserve_shared() has been called. |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 224 | */ |
| 225 | void reservation_object_add_shared_fence(struct reservation_object *obj, |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 226 | struct dma_fence *fence) |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 227 | { |
| 228 | struct reservation_object_list *old, *fobj = obj->staged; |
| 229 | |
| 230 | old = reservation_object_get_list(obj); |
| 231 | obj->staged = NULL; |
| 232 | |
| 233 | if (!fobj) { |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 234 | BUG_ON(old->shared_count >= old->shared_max); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 235 | reservation_object_add_shared_inplace(obj, old, fence); |
| 236 | } else |
| 237 | reservation_object_add_shared_replace(obj, old, fobj, fence); |
| 238 | } |
| 239 | EXPORT_SYMBOL(reservation_object_add_shared_fence); |
| 240 | |
Rob Clark | dad6c39 | 2016-03-31 16:26:51 -0400 | [diff] [blame] | 241 | /** |
| 242 | * reservation_object_add_excl_fence - Add an exclusive fence. |
| 243 | * @obj: the reservation object |
| 244 | * @fence: the shared fence to add |
| 245 | * |
| 246 | * Add a fence to the exclusive slot. The obj->lock must be held. |
| 247 | */ |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 248 | void reservation_object_add_excl_fence(struct reservation_object *obj, |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 249 | struct dma_fence *fence) |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 250 | { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 251 | struct dma_fence *old_fence = reservation_object_get_excl(obj); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 252 | struct reservation_object_list *old; |
| 253 | u32 i = 0; |
| 254 | |
| 255 | old = reservation_object_get_list(obj); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 256 | if (old) |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 257 | i = old->shared_count; |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 258 | |
| 259 | if (fence) |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 260 | dma_fence_get(fence); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 261 | |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 262 | preempt_disable(); |
| 263 | write_seqcount_begin(&obj->seq); |
| 264 | /* write_seqcount_begin provides the necessary memory barrier */ |
| 265 | RCU_INIT_POINTER(obj->fence_excl, fence); |
| 266 | if (old) |
| 267 | old->shared_count = 0; |
| 268 | write_seqcount_end(&obj->seq); |
| 269 | preempt_enable(); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 270 | |
| 271 | /* inplace update, no shared fences */ |
| 272 | while (i--) |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 273 | dma_fence_put(rcu_dereference_protected(old->shared[i], |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 274 | reservation_object_held(obj))); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 275 | |
Christian König | f3e31b7 | 2017-08-07 17:32:21 -0400 | [diff] [blame] | 276 | dma_fence_put(old_fence); |
Maarten Lankhorst | 04a5faa | 2014-07-01 12:57:54 +0200 | [diff] [blame] | 277 | } |
| 278 | EXPORT_SYMBOL(reservation_object_add_excl_fence); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 279 | |
Rob Clark | dad6c39 | 2016-03-31 16:26:51 -0400 | [diff] [blame] | 280 | /** |
Christian König | 7faf952 | 2017-08-10 13:01:48 -0400 | [diff] [blame] | 281 | * reservation_object_copy_fences - Copy all fences from src to dst. |
| 282 | * @dst: the destination reservation object |
| 283 | * @src: the source reservation object |
| 284 | * |
Christian König | 39e16ba | 2017-09-04 21:02:45 +0200 | [diff] [blame] | 285 | * Copy all fences from src to dst. dst-lock must be held. |
Christian König | 7faf952 | 2017-08-10 13:01:48 -0400 | [diff] [blame] | 286 | */ |
| 287 | int reservation_object_copy_fences(struct reservation_object *dst, |
| 288 | struct reservation_object *src) |
| 289 | { |
| 290 | struct reservation_object_list *src_list, *dst_list; |
| 291 | struct dma_fence *old, *new; |
| 292 | size_t size; |
| 293 | unsigned i; |
| 294 | |
Christian König | 39e16ba | 2017-09-04 21:02:45 +0200 | [diff] [blame] | 295 | rcu_read_lock(); |
| 296 | src_list = rcu_dereference(src->fence); |
Christian König | 7faf952 | 2017-08-10 13:01:48 -0400 | [diff] [blame] | 297 | |
Christian König | 39e16ba | 2017-09-04 21:02:45 +0200 | [diff] [blame] | 298 | retry: |
Christian König | 7faf952 | 2017-08-10 13:01:48 -0400 | [diff] [blame] | 299 | if (src_list) { |
Christian König | 39e16ba | 2017-09-04 21:02:45 +0200 | [diff] [blame] | 300 | unsigned shared_count = src_list->shared_count; |
| 301 | |
| 302 | size = offsetof(typeof(*src_list), shared[shared_count]); |
| 303 | rcu_read_unlock(); |
| 304 | |
Christian König | 7faf952 | 2017-08-10 13:01:48 -0400 | [diff] [blame] | 305 | dst_list = kmalloc(size, GFP_KERNEL); |
| 306 | if (!dst_list) |
| 307 | return -ENOMEM; |
| 308 | |
Christian König | 39e16ba | 2017-09-04 21:02:45 +0200 | [diff] [blame] | 309 | rcu_read_lock(); |
| 310 | src_list = rcu_dereference(src->fence); |
| 311 | if (!src_list || src_list->shared_count > shared_count) { |
| 312 | kfree(dst_list); |
| 313 | goto retry; |
| 314 | } |
| 315 | |
| 316 | dst_list->shared_count = 0; |
| 317 | dst_list->shared_max = shared_count; |
| 318 | for (i = 0; i < src_list->shared_count; ++i) { |
| 319 | struct dma_fence *fence; |
| 320 | |
| 321 | fence = rcu_dereference(src_list->shared[i]); |
| 322 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, |
| 323 | &fence->flags)) |
| 324 | continue; |
| 325 | |
| 326 | if (!dma_fence_get_rcu(fence)) { |
| 327 | kfree(dst_list); |
| 328 | src_list = rcu_dereference(src->fence); |
| 329 | goto retry; |
| 330 | } |
| 331 | |
| 332 | if (dma_fence_is_signaled(fence)) { |
| 333 | dma_fence_put(fence); |
| 334 | continue; |
| 335 | } |
| 336 | |
Ville Syrjälä | ad46d7b | 2017-11-02 22:03:36 +0200 | [diff] [blame] | 337 | rcu_assign_pointer(dst_list->shared[dst_list->shared_count++], fence); |
Christian König | 39e16ba | 2017-09-04 21:02:45 +0200 | [diff] [blame] | 338 | } |
Christian König | 7faf952 | 2017-08-10 13:01:48 -0400 | [diff] [blame] | 339 | } else { |
| 340 | dst_list = NULL; |
| 341 | } |
| 342 | |
Christian König | 39e16ba | 2017-09-04 21:02:45 +0200 | [diff] [blame] | 343 | new = dma_fence_get_rcu_safe(&src->fence_excl); |
| 344 | rcu_read_unlock(); |
| 345 | |
Christian König | 7faf952 | 2017-08-10 13:01:48 -0400 | [diff] [blame] | 346 | kfree(dst->staged); |
| 347 | dst->staged = NULL; |
| 348 | |
| 349 | src_list = reservation_object_get_list(dst); |
Christian König | 7faf952 | 2017-08-10 13:01:48 -0400 | [diff] [blame] | 350 | old = reservation_object_get_excl(dst); |
Christian König | 7faf952 | 2017-08-10 13:01:48 -0400 | [diff] [blame] | 351 | |
| 352 | preempt_disable(); |
| 353 | write_seqcount_begin(&dst->seq); |
| 354 | /* write_seqcount_begin provides the necessary memory barrier */ |
| 355 | RCU_INIT_POINTER(dst->fence_excl, new); |
| 356 | RCU_INIT_POINTER(dst->fence, dst_list); |
| 357 | write_seqcount_end(&dst->seq); |
| 358 | preempt_enable(); |
| 359 | |
| 360 | if (src_list) |
| 361 | kfree_rcu(src_list, rcu); |
| 362 | dma_fence_put(old); |
| 363 | |
| 364 | return 0; |
| 365 | } |
| 366 | EXPORT_SYMBOL(reservation_object_copy_fences); |
| 367 | |
| 368 | /** |
Rob Clark | dad6c39 | 2016-03-31 16:26:51 -0400 | [diff] [blame] | 369 | * reservation_object_get_fences_rcu - Get an object's shared and exclusive |
| 370 | * fences without update side lock held |
| 371 | * @obj: the reservation object |
| 372 | * @pfence_excl: the returned exclusive fence (or NULL) |
| 373 | * @pshared_count: the number of shared fences returned |
| 374 | * @pshared: the array of shared fence ptrs returned (array is krealloc'd to |
| 375 | * the required size, and must be freed by caller) |
| 376 | * |
| 377 | * RETURNS |
| 378 | * Zero or -errno |
| 379 | */ |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 380 | int reservation_object_get_fences_rcu(struct reservation_object *obj, |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 381 | struct dma_fence **pfence_excl, |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 382 | unsigned *pshared_count, |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 383 | struct dma_fence ***pshared) |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 384 | { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 385 | struct dma_fence **shared = NULL; |
| 386 | struct dma_fence *fence_excl; |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 387 | unsigned int shared_count; |
| 388 | int ret = 1; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 389 | |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 390 | do { |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 391 | struct reservation_object_list *fobj; |
| 392 | unsigned seq; |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 393 | unsigned int i; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 394 | |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 395 | shared_count = i = 0; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 396 | |
| 397 | rcu_read_lock(); |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 398 | seq = read_seqcount_begin(&obj->seq); |
| 399 | |
| 400 | fence_excl = rcu_dereference(obj->fence_excl); |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 401 | if (fence_excl && !dma_fence_get_rcu(fence_excl)) |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 402 | goto unlock; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 403 | |
| 404 | fobj = rcu_dereference(obj->fence); |
| 405 | if (fobj) { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 406 | struct dma_fence **nshared; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 407 | size_t sz = sizeof(*shared) * fobj->shared_max; |
| 408 | |
| 409 | nshared = krealloc(shared, sz, |
| 410 | GFP_NOWAIT | __GFP_NOWARN); |
| 411 | if (!nshared) { |
| 412 | rcu_read_unlock(); |
| 413 | nshared = krealloc(shared, sz, GFP_KERNEL); |
| 414 | if (nshared) { |
| 415 | shared = nshared; |
| 416 | continue; |
| 417 | } |
| 418 | |
| 419 | ret = -ENOMEM; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 420 | break; |
| 421 | } |
| 422 | shared = nshared; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 423 | shared_count = fobj->shared_count; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 424 | |
| 425 | for (i = 0; i < shared_count; ++i) { |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 426 | shared[i] = rcu_dereference(fobj->shared[i]); |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 427 | if (!dma_fence_get_rcu(shared[i])) |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 428 | break; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 429 | } |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 430 | } |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 431 | |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 432 | if (i != shared_count || read_seqcount_retry(&obj->seq, seq)) { |
| 433 | while (i--) |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 434 | dma_fence_put(shared[i]); |
| 435 | dma_fence_put(fence_excl); |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 436 | goto unlock; |
| 437 | } |
| 438 | |
| 439 | ret = 0; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 440 | unlock: |
| 441 | rcu_read_unlock(); |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 442 | } while (ret); |
| 443 | |
| 444 | if (!shared_count) { |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 445 | kfree(shared); |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 446 | shared = NULL; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 447 | } |
Chris Wilson | fedf541 | 2016-08-29 08:08:30 +0100 | [diff] [blame] | 448 | |
| 449 | *pshared_count = shared_count; |
| 450 | *pshared = shared; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 451 | *pfence_excl = fence_excl; |
| 452 | |
| 453 | return ret; |
| 454 | } |
| 455 | EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu); |
| 456 | |
Rob Clark | dad6c39 | 2016-03-31 16:26:51 -0400 | [diff] [blame] | 457 | /** |
| 458 | * reservation_object_wait_timeout_rcu - Wait on reservation's objects |
| 459 | * shared and/or exclusive fences. |
| 460 | * @obj: the reservation object |
| 461 | * @wait_all: if true, wait on all fences, else wait on just exclusive fence |
| 462 | * @intr: if true, do interruptible wait |
| 463 | * @timeout: timeout value in jiffies or zero to return immediately |
| 464 | * |
| 465 | * RETURNS |
| 466 | * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or |
| 467 | * greater than zer on success. |
| 468 | */ |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 469 | long reservation_object_wait_timeout_rcu(struct reservation_object *obj, |
| 470 | bool wait_all, bool intr, |
| 471 | unsigned long timeout) |
| 472 | { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 473 | struct dma_fence *fence; |
Christian König | 5bffee8 | 2018-01-22 21:00:03 +0100 | [diff] [blame] | 474 | unsigned seq, shared_count; |
Christian König | 06a66b5 | 2016-11-07 16:16:16 -0500 | [diff] [blame] | 475 | long ret = timeout ? timeout : 1; |
Christian König | 5bffee8 | 2018-01-22 21:00:03 +0100 | [diff] [blame] | 476 | int i; |
Jammy Zhou | fb8b7d2 | 2015-01-21 18:35:47 +0800 | [diff] [blame] | 477 | |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 478 | retry: |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 479 | shared_count = 0; |
| 480 | seq = read_seqcount_begin(&obj->seq); |
| 481 | rcu_read_lock(); |
Christian König | 5bffee8 | 2018-01-22 21:00:03 +0100 | [diff] [blame] | 482 | i = -1; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 483 | |
Christian König | b88fa00 | 2017-08-10 13:01:49 -0400 | [diff] [blame] | 484 | fence = rcu_dereference(obj->fence_excl); |
| 485 | if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { |
| 486 | if (!dma_fence_get_rcu(fence)) |
| 487 | goto unlock_retry; |
| 488 | |
| 489 | if (dma_fence_is_signaled(fence)) { |
| 490 | dma_fence_put(fence); |
| 491 | fence = NULL; |
| 492 | } |
| 493 | |
| 494 | } else { |
| 495 | fence = NULL; |
| 496 | } |
| 497 | |
Christian König | 5bffee8 | 2018-01-22 21:00:03 +0100 | [diff] [blame] | 498 | if (wait_all) { |
Jagan Teki | 5136629 | 2015-05-21 01:09:31 +0530 | [diff] [blame] | 499 | struct reservation_object_list *fobj = |
| 500 | rcu_dereference(obj->fence); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 501 | |
| 502 | if (fobj) |
| 503 | shared_count = fobj->shared_count; |
| 504 | |
Christian König | 5bffee8 | 2018-01-22 21:00:03 +0100 | [diff] [blame] | 505 | for (i = 0; !fence && i < shared_count; ++i) { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 506 | struct dma_fence *lfence = rcu_dereference(fobj->shared[i]); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 507 | |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 508 | if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, |
| 509 | &lfence->flags)) |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 510 | continue; |
| 511 | |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 512 | if (!dma_fence_get_rcu(lfence)) |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 513 | goto unlock_retry; |
| 514 | |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 515 | if (dma_fence_is_signaled(lfence)) { |
| 516 | dma_fence_put(lfence); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 517 | continue; |
| 518 | } |
| 519 | |
| 520 | fence = lfence; |
| 521 | break; |
| 522 | } |
| 523 | } |
| 524 | |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 525 | rcu_read_unlock(); |
| 526 | if (fence) { |
Chris Wilson | 1cec20f | 2016-08-29 08:08:31 +0100 | [diff] [blame] | 527 | if (read_seqcount_retry(&obj->seq, seq)) { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 528 | dma_fence_put(fence); |
Chris Wilson | 1cec20f | 2016-08-29 08:08:31 +0100 | [diff] [blame] | 529 | goto retry; |
| 530 | } |
| 531 | |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 532 | ret = dma_fence_wait_timeout(fence, intr, ret); |
| 533 | dma_fence_put(fence); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 534 | if (ret > 0 && wait_all && (i + 1 < shared_count)) |
| 535 | goto retry; |
| 536 | } |
| 537 | return ret; |
| 538 | |
| 539 | unlock_retry: |
| 540 | rcu_read_unlock(); |
| 541 | goto retry; |
| 542 | } |
| 543 | EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu); |
| 544 | |
| 545 | |
| 546 | static inline int |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 547 | reservation_object_test_signaled_single(struct dma_fence *passed_fence) |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 548 | { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 549 | struct dma_fence *fence, *lfence = passed_fence; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 550 | int ret = 1; |
| 551 | |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 552 | if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) { |
| 553 | fence = dma_fence_get_rcu(lfence); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 554 | if (!fence) |
| 555 | return -1; |
| 556 | |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 557 | ret = !!dma_fence_is_signaled(fence); |
| 558 | dma_fence_put(fence); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 559 | } |
| 560 | return ret; |
| 561 | } |
| 562 | |
Rob Clark | dad6c39 | 2016-03-31 16:26:51 -0400 | [diff] [blame] | 563 | /** |
| 564 | * reservation_object_test_signaled_rcu - Test if a reservation object's |
| 565 | * fences have been signaled. |
| 566 | * @obj: the reservation object |
| 567 | * @test_all: if true, test all fences, otherwise only test the exclusive |
| 568 | * fence |
| 569 | * |
| 570 | * RETURNS |
| 571 | * true if all fences signaled, else false |
| 572 | */ |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 573 | bool reservation_object_test_signaled_rcu(struct reservation_object *obj, |
| 574 | bool test_all) |
| 575 | { |
| 576 | unsigned seq, shared_count; |
Chris Wilson | b68d837 | 2016-08-29 08:08:32 +0100 | [diff] [blame] | 577 | int ret; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 578 | |
Chris Wilson | b68d837 | 2016-08-29 08:08:32 +0100 | [diff] [blame] | 579 | rcu_read_lock(); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 580 | retry: |
Chris Wilson | b68d837 | 2016-08-29 08:08:32 +0100 | [diff] [blame] | 581 | ret = true; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 582 | shared_count = 0; |
| 583 | seq = read_seqcount_begin(&obj->seq); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 584 | |
| 585 | if (test_all) { |
| 586 | unsigned i; |
| 587 | |
Jagan Teki | 5136629 | 2015-05-21 01:09:31 +0530 | [diff] [blame] | 588 | struct reservation_object_list *fobj = |
| 589 | rcu_dereference(obj->fence); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 590 | |
| 591 | if (fobj) |
| 592 | shared_count = fobj->shared_count; |
| 593 | |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 594 | for (i = 0; i < shared_count; ++i) { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 595 | struct dma_fence *fence = rcu_dereference(fobj->shared[i]); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 596 | |
| 597 | ret = reservation_object_test_signaled_single(fence); |
| 598 | if (ret < 0) |
Chris Wilson | b68d837 | 2016-08-29 08:08:32 +0100 | [diff] [blame] | 599 | goto retry; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 600 | else if (!ret) |
| 601 | break; |
| 602 | } |
| 603 | |
Chris Wilson | b68d837 | 2016-08-29 08:08:32 +0100 | [diff] [blame] | 604 | if (read_seqcount_retry(&obj->seq, seq)) |
| 605 | goto retry; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 606 | } |
| 607 | |
| 608 | if (!shared_count) { |
Chris Wilson | f54d186 | 2016-10-25 13:00:45 +0100 | [diff] [blame] | 609 | struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 610 | |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 611 | if (fence_excl) { |
Jagan Teki | 5136629 | 2015-05-21 01:09:31 +0530 | [diff] [blame] | 612 | ret = reservation_object_test_signaled_single( |
| 613 | fence_excl); |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 614 | if (ret < 0) |
Chris Wilson | b68d837 | 2016-08-29 08:08:32 +0100 | [diff] [blame] | 615 | goto retry; |
| 616 | |
| 617 | if (read_seqcount_retry(&obj->seq, seq)) |
| 618 | goto retry; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 619 | } |
| 620 | } |
| 621 | |
| 622 | rcu_read_unlock(); |
| 623 | return ret; |
Maarten Lankhorst | 3c3b177 | 2014-07-01 12:58:00 +0200 | [diff] [blame] | 624 | } |
| 625 | EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu); |