blob: 40ffcb94e3797c295347f2d0f24faed65b2664d3 [file] [log] [blame]
Chris Wilson24f90d62021-01-22 19:29:04 +00001// SPDX-License-Identifier: MIT
Chris Wilson2871ea82019-10-24 11:03:44 +01002/*
Chris Wilson2871ea82019-10-24 11:03:44 +01003 * Copyright © 2019 Intel Corporation
4 */
5
Jani Nikulab508d012022-02-10 17:45:39 +02006#include "gem/i915_gem_internal.h"
Michel Thierryd712f4c2021-01-27 13:14:17 +00007#include "gem/i915_gem_lmem.h"
Chris Wilson2871ea82019-10-24 11:03:44 +01008#include "gem/i915_gem_object.h"
Chris Wilson45233ab2020-12-16 13:54:52 +00009
Chris Wilson2871ea82019-10-24 11:03:44 +010010#include "i915_drv.h"
11#include "i915_vma.h"
12#include "intel_engine.h"
Matt Roper202b1f42022-01-10 21:15:56 -080013#include "intel_engine_regs.h"
Chris Wilson45233ab2020-12-16 13:54:52 +000014#include "intel_gpu_commands.h"
Chris Wilson2871ea82019-10-24 11:03:44 +010015#include "intel_ring.h"
16#include "intel_timeline.h"
17
18unsigned int intel_ring_update_space(struct intel_ring *ring)
19{
20 unsigned int space;
21
22 space = __intel_ring_space(ring->head, ring->emit, ring->size);
23
24 ring->space = space;
25 return space;
26}
27
Maarten Lankhorst47b08692020-08-19 16:08:54 +020028void __intel_ring_pin(struct intel_ring *ring)
29{
30 GEM_BUG_ON(!atomic_read(&ring->pin_count));
31 atomic_inc(&ring->pin_count);
32}
33
34int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww)
Chris Wilson2871ea82019-10-24 11:03:44 +010035{
36 struct i915_vma *vma = ring->vma;
37 unsigned int flags;
38 void *addr;
39 int ret;
40
41 if (atomic_fetch_inc(&ring->pin_count))
42 return 0;
43
Chris Wilson2871ea82019-10-24 11:03:44 +010044 /* Ring wraparound at offset 0 sometimes hangs. No idea why. */
Chris Wilsone3793462020-01-30 18:17:10 +000045 flags = PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
Chris Wilson2871ea82019-10-24 11:03:44 +010046
Chris Wilson41a9c752021-01-19 21:43:33 +000047 if (i915_gem_object_is_stolen(vma->obj))
Chris Wilson2871ea82019-10-24 11:03:44 +010048 flags |= PIN_MAPPABLE;
49 else
50 flags |= PIN_HIGH;
51
Maarten Lankhorst47b08692020-08-19 16:08:54 +020052 ret = i915_ggtt_pin(vma, ww, 0, flags);
Chris Wilson2871ea82019-10-24 11:03:44 +010053 if (unlikely(ret))
54 goto err_unpin;
55
Venkata Sandeep Dhanalakotafa85bfd2021-04-27 09:54:12 +010056 if (i915_vma_is_map_and_fenceable(vma)) {
Chris Wilson2871ea82019-10-24 11:03:44 +010057 addr = (void __force *)i915_vma_pin_iomap(vma);
Venkata Sandeep Dhanalakotafa85bfd2021-04-27 09:54:12 +010058 } else {
59 int type = i915_coherent_map_type(vma->vm->i915, vma->obj, false);
60
61 addr = i915_gem_object_pin_map(vma->obj, type);
62 }
63
Chris Wilson2871ea82019-10-24 11:03:44 +010064 if (IS_ERR(addr)) {
65 ret = PTR_ERR(addr);
66 goto err_ring;
67 }
68
69 i915_vma_make_unshrinkable(vma);
70
Chris Wilsona266bf42019-11-18 23:02:40 +000071 /* Discard any unused bytes beyond that submitted to hw. */
72 intel_ring_reset(ring, ring->emit);
Chris Wilson2871ea82019-10-24 11:03:44 +010073
Chris Wilsona266bf42019-11-18 23:02:40 +000074 ring->vaddr = addr;
Chris Wilson2871ea82019-10-24 11:03:44 +010075 return 0;
76
77err_ring:
78 i915_vma_unpin(vma);
79err_unpin:
80 atomic_dec(&ring->pin_count);
81 return ret;
82}
83
84void intel_ring_reset(struct intel_ring *ring, u32 tail)
85{
86 tail = intel_ring_wrap(ring, tail);
87 ring->tail = tail;
88 ring->head = tail;
89 ring->emit = tail;
90 intel_ring_update_space(ring);
91}
92
93void intel_ring_unpin(struct intel_ring *ring)
94{
95 struct i915_vma *vma = ring->vma;
96
97 if (!atomic_dec_and_test(&ring->pin_count))
98 return;
99
Chris Wilson2871ea82019-10-24 11:03:44 +0100100 i915_vma_unset_ggtt_write(vma);
101 if (i915_vma_is_map_and_fenceable(vma))
102 i915_vma_unpin_iomap(vma);
103 else
104 i915_gem_object_unpin_map(vma->obj);
105
Chris Wilson2871ea82019-10-24 11:03:44 +0100106 i915_vma_make_purgeable(vma);
Chris Wilsona266bf42019-11-18 23:02:40 +0000107 i915_vma_unpin(vma);
Chris Wilson2871ea82019-10-24 11:03:44 +0100108}
109
110static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size)
111{
112 struct i915_address_space *vm = &ggtt->vm;
113 struct drm_i915_private *i915 = vm->i915;
114 struct drm_i915_gem_object *obj;
115 struct i915_vma *vma;
116
Thomas Hellström0d8ee5b2021-09-22 08:25:24 +0200117 obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE |
118 I915_BO_ALLOC_PM_VOLATILE);
Michel Thierryd712f4c2021-01-27 13:14:17 +0000119 if (IS_ERR(obj) && i915_ggtt_has_aperture(ggtt))
Matthew Auld34a6baa2019-10-29 09:58:55 +0000120 obj = i915_gem_object_create_stolen(i915, size);
Chris Wilson2871ea82019-10-24 11:03:44 +0100121 if (IS_ERR(obj))
122 obj = i915_gem_object_create_internal(i915, size);
123 if (IS_ERR(obj))
124 return ERR_CAST(obj);
125
126 /*
127 * Mark ring buffers as read-only from GPU side (so no stray overwrites)
128 * if supported by the platform's GGTT.
129 */
130 if (vm->has_read_only)
131 i915_gem_object_set_readonly(obj);
132
133 vma = i915_vma_instance(obj, vm, NULL);
134 if (IS_ERR(vma))
135 goto err;
136
137 return vma;
138
139err:
140 i915_gem_object_put(obj);
141 return vma;
142}
143
144struct intel_ring *
145intel_engine_create_ring(struct intel_engine_cs *engine, int size)
146{
147 struct drm_i915_private *i915 = engine->i915;
148 struct intel_ring *ring;
149 struct i915_vma *vma;
150
151 GEM_BUG_ON(!is_power_of_2(size));
152 GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
153
154 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
155 if (!ring)
156 return ERR_PTR(-ENOMEM);
157
158 kref_init(&ring->ref);
159 ring->size = size;
Chris Wilson5ba32c72020-02-07 21:14:52 +0000160 ring->wrap = BITS_PER_TYPE(ring->size) - ilog2(size);
Chris Wilson2871ea82019-10-24 11:03:44 +0100161
162 /*
163 * Workaround an erratum on the i830 which causes a hang if
164 * the TAIL pointer points to within the last 2 cachelines
165 * of the buffer.
166 */
167 ring->effective_size = size;
168 if (IS_I830(i915) || IS_I845G(i915))
169 ring->effective_size -= 2 * CACHELINE_BYTES;
170
171 intel_ring_update_space(ring);
172
173 vma = create_ring_vma(engine->gt->ggtt, size);
174 if (IS_ERR(vma)) {
175 kfree(ring);
176 return ERR_CAST(vma);
177 }
178 ring->vma = vma;
179
180 return ring;
181}
182
183void intel_ring_free(struct kref *ref)
184{
185 struct intel_ring *ring = container_of(ref, typeof(*ring), ref);
186
187 i915_vma_put(ring->vma);
188 kfree(ring);
189}
190
191static noinline int
192wait_for_space(struct intel_ring *ring,
193 struct intel_timeline *tl,
194 unsigned int bytes)
195{
196 struct i915_request *target;
197 long timeout;
198
199 if (intel_ring_update_space(ring) >= bytes)
200 return 0;
201
202 GEM_BUG_ON(list_empty(&tl->requests));
203 list_for_each_entry(target, &tl->requests, link) {
204 if (target->ring != ring)
205 continue;
206
207 /* Would completion of this request free enough space? */
208 if (bytes <= __intel_ring_space(target->postfix,
209 ring->emit, ring->size))
210 break;
211 }
212
213 if (GEM_WARN_ON(&target->link == &tl->requests))
214 return -ENOSPC;
215
216 timeout = i915_request_wait(target,
217 I915_WAIT_INTERRUPTIBLE,
218 MAX_SCHEDULE_TIMEOUT);
219 if (timeout < 0)
220 return timeout;
221
222 i915_request_retire_upto(target);
223
224 intel_ring_update_space(ring);
225 GEM_BUG_ON(ring->space < bytes);
226 return 0;
227}
228
229u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
230{
231 struct intel_ring *ring = rq->ring;
232 const unsigned int remain_usable = ring->effective_size - ring->emit;
233 const unsigned int bytes = num_dwords * sizeof(u32);
234 unsigned int need_wrap = 0;
235 unsigned int total_bytes;
236 u32 *cs;
237
238 /* Packets must be qword aligned. */
239 GEM_BUG_ON(num_dwords & 1);
240
241 total_bytes = bytes + rq->reserved_space;
242 GEM_BUG_ON(total_bytes > ring->effective_size);
243
244 if (unlikely(total_bytes > remain_usable)) {
245 const int remain_actual = ring->size - ring->emit;
246
247 if (bytes > remain_usable) {
248 /*
249 * Not enough space for the basic request. So need to
250 * flush out the remainder and then wait for
251 * base + reserved.
252 */
253 total_bytes += remain_actual;
254 need_wrap = remain_actual | 1;
255 } else {
256 /*
257 * The base request will fit but the reserved space
258 * falls off the end. So we don't need an immediate
259 * wrap and only need to effectively wait for the
260 * reserved size from the start of ringbuffer.
261 */
262 total_bytes = rq->reserved_space + remain_actual;
263 }
264 }
265
266 if (unlikely(total_bytes > ring->space)) {
267 int ret;
268
269 /*
270 * Space is reserved in the ringbuffer for finalising the
271 * request, as that cannot be allowed to fail. During request
272 * finalisation, reserved_space is set to 0 to stop the
273 * overallocation and the assumption is that then we never need
274 * to wait (which has the risk of failing with EINTR).
275 *
276 * See also i915_request_alloc() and i915_request_add().
277 */
278 GEM_BUG_ON(!rq->reserved_space);
279
280 ret = wait_for_space(ring,
281 i915_request_timeline(rq),
282 total_bytes);
283 if (unlikely(ret))
284 return ERR_PTR(ret);
285 }
286
287 if (unlikely(need_wrap)) {
288 need_wrap &= ~1;
289 GEM_BUG_ON(need_wrap > ring->space);
290 GEM_BUG_ON(ring->emit + need_wrap > ring->size);
291 GEM_BUG_ON(!IS_ALIGNED(need_wrap, sizeof(u64)));
292
293 /* Fill the tail with MI_NOOP */
294 memset64(ring->vaddr + ring->emit, 0, need_wrap / sizeof(u64));
295 ring->space -= need_wrap;
296 ring->emit = 0;
297 }
298
299 GEM_BUG_ON(ring->emit > ring->size - bytes);
300 GEM_BUG_ON(ring->space < bytes);
301 cs = ring->vaddr + ring->emit;
302 GEM_DEBUG_EXEC(memset32(cs, POISON_INUSE, bytes / sizeof(*cs)));
303 ring->emit += bytes;
304 ring->space -= bytes;
305
306 return cs;
307}
308
309/* Align the ring tail to a cacheline boundary */
310int intel_ring_cacheline_align(struct i915_request *rq)
311{
312 int num_dwords;
313 void *cs;
314
315 num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
316 if (num_dwords == 0)
317 return 0;
318
319 num_dwords = CACHELINE_DWORDS - num_dwords;
320 GEM_BUG_ON(num_dwords & 1);
321
322 cs = intel_ring_begin(rq, num_dwords);
323 if (IS_ERR(cs))
324 return PTR_ERR(cs);
325
326 memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);
327 intel_ring_advance(rq, cs + num_dwords);
328
329 GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));
330 return 0;
331}
Chris Wilson8ab3a382020-06-09 16:17:23 +0100332
333#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
334#include "selftest_ring.c"
335#endif