Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 1 | // SPDX-License-Identifier: MIT |
| 2 | /* |
| 3 | * Copyright © 2022 Intel Corporation |
| 4 | */ |
| 5 | |
Lucas De Marchi | cb30cfd | 2023-02-21 15:33:48 -0800 | [diff] [blame] | 6 | #include "xe_wait_user_fence.h" |
| 7 | |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 8 | #include <drm/drm_device.h> |
| 9 | #include <drm/drm_file.h> |
Zbigniew Kempczyński | 5572a00 | 2023-06-28 07:51:41 +0200 | [diff] [blame] | 10 | #include <drm/drm_utils.h> |
Jani Nikula | 87d8ecf | 2024-08-27 12:15:39 +0300 | [diff] [blame] | 11 | #include <uapi/drm/xe_drm.h> |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 12 | |
| 13 | #include "xe_device.h" |
| 14 | #include "xe_gt.h" |
| 15 | #include "xe_macros.h" |
Bommu Krishnaiah | e670f0b | 2023-12-15 15:45:34 +0000 | [diff] [blame] | 16 | #include "xe_exec_queue.h" |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 17 | |
| 18 | static int do_compare(u64 addr, u64 value, u64 mask, u16 op) |
| 19 | { |
| 20 | u64 rvalue; |
| 21 | int err; |
| 22 | bool passed; |
| 23 | |
| 24 | err = copy_from_user(&rvalue, u64_to_user_ptr(addr), sizeof(rvalue)); |
| 25 | if (err) |
| 26 | return -EFAULT; |
| 27 | |
| 28 | switch (op) { |
Rodrigo Vivi | 4a349c8 | 2023-11-14 13:34:33 +0000 | [diff] [blame] | 29 | case DRM_XE_UFENCE_WAIT_OP_EQ: |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 30 | passed = (rvalue & mask) == (value & mask); |
| 31 | break; |
Rodrigo Vivi | 4a349c8 | 2023-11-14 13:34:33 +0000 | [diff] [blame] | 32 | case DRM_XE_UFENCE_WAIT_OP_NEQ: |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 33 | passed = (rvalue & mask) != (value & mask); |
| 34 | break; |
Rodrigo Vivi | 4a349c8 | 2023-11-14 13:34:33 +0000 | [diff] [blame] | 35 | case DRM_XE_UFENCE_WAIT_OP_GT: |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 36 | passed = (rvalue & mask) > (value & mask); |
| 37 | break; |
Rodrigo Vivi | 4a349c8 | 2023-11-14 13:34:33 +0000 | [diff] [blame] | 38 | case DRM_XE_UFENCE_WAIT_OP_GTE: |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 39 | passed = (rvalue & mask) >= (value & mask); |
| 40 | break; |
Rodrigo Vivi | 4a349c8 | 2023-11-14 13:34:33 +0000 | [diff] [blame] | 41 | case DRM_XE_UFENCE_WAIT_OP_LT: |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 42 | passed = (rvalue & mask) < (value & mask); |
| 43 | break; |
Rodrigo Vivi | 4a349c8 | 2023-11-14 13:34:33 +0000 | [diff] [blame] | 44 | case DRM_XE_UFENCE_WAIT_OP_LTE: |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 45 | passed = (rvalue & mask) <= (value & mask); |
| 46 | break; |
| 47 | default: |
Francois Dugast | 99fea68 | 2023-07-27 14:55:29 +0000 | [diff] [blame] | 48 | XE_WARN_ON("Not possible"); |
Lucas De Marchi | 717cf0a | 2023-12-18 08:33:01 -0800 | [diff] [blame] | 49 | return -EINVAL; |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 50 | } |
| 51 | |
| 52 | return passed ? 0 : 1; |
| 53 | } |
| 54 | |
Bommu Krishnaiah | 9212da0 | 2023-12-15 15:45:33 +0000 | [diff] [blame] | 55 | #define VALID_FLAGS DRM_XE_UFENCE_WAIT_FLAG_ABSTIME |
Rodrigo Vivi | 4a349c8 | 2023-11-14 13:34:33 +0000 | [diff] [blame] | 56 | #define MAX_OP DRM_XE_UFENCE_WAIT_OP_LTE |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 57 | |
Fei Yang | e2e2d96 | 2023-09-21 15:05:00 -0700 | [diff] [blame] | 58 | static long to_jiffies_timeout(struct xe_device *xe, |
| 59 | struct drm_xe_wait_user_fence *args) |
Zbigniew Kempczyński | 5572a00 | 2023-06-28 07:51:41 +0200 | [diff] [blame] | 60 | { |
Fei Yang | e2e2d96 | 2023-09-21 15:05:00 -0700 | [diff] [blame] | 61 | unsigned long long t; |
| 62 | long timeout; |
Zbigniew Kempczyński | 5572a00 | 2023-06-28 07:51:41 +0200 | [diff] [blame] | 63 | |
Fei Yang | e2e2d96 | 2023-09-21 15:05:00 -0700 | [diff] [blame] | 64 | /* |
| 65 | * For negative timeout we want to wait "forever" by setting |
| 66 | * MAX_SCHEDULE_TIMEOUT. But we have to assign this value also |
| 67 | * to args->timeout to avoid being zeroed on the signal delivery |
| 68 | * (see arithmetics after wait). |
| 69 | */ |
| 70 | if (args->timeout < 0) { |
| 71 | args->timeout = MAX_SCHEDULE_TIMEOUT; |
| 72 | return MAX_SCHEDULE_TIMEOUT; |
| 73 | } |
| 74 | |
| 75 | if (args->timeout == 0) |
| 76 | return 0; |
| 77 | |
| 78 | /* |
| 79 | * Save the timeout to an u64 variable because nsecs_to_jiffies |
| 80 | * might return a value that overflows s32 variable. |
| 81 | */ |
Francois Dugast | 3ac4a78 | 2023-11-14 13:34:28 +0000 | [diff] [blame] | 82 | if (args->flags & DRM_XE_UFENCE_WAIT_FLAG_ABSTIME) |
Fei Yang | e2e2d96 | 2023-09-21 15:05:00 -0700 | [diff] [blame] | 83 | t = drm_timeout_abs_to_jiffies(args->timeout); |
| 84 | else |
| 85 | t = nsecs_to_jiffies(args->timeout); |
Zbigniew Kempczyński | 5572a00 | 2023-06-28 07:51:41 +0200 | [diff] [blame] | 86 | |
Fei Yang | e2e2d96 | 2023-09-21 15:05:00 -0700 | [diff] [blame] | 87 | /* |
| 88 | * Anything greater then MAX_SCHEDULE_TIMEOUT is meaningless, |
| 89 | * also we don't want to cap it at MAX_SCHEDULE_TIMEOUT because |
| 90 | * apparently user doesn't mean to wait forever, otherwise the |
| 91 | * args->timeout should have been set to a negative value. |
| 92 | */ |
| 93 | if (t > MAX_SCHEDULE_TIMEOUT) |
| 94 | timeout = MAX_SCHEDULE_TIMEOUT - 1; |
| 95 | else |
| 96 | timeout = t; |
Zbigniew Kempczyński | 5572a00 | 2023-06-28 07:51:41 +0200 | [diff] [blame] | 97 | |
| 98 | return timeout ?: 1; |
| 99 | } |
| 100 | |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 101 | int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, |
| 102 | struct drm_file *file) |
| 103 | { |
| 104 | struct xe_device *xe = to_xe_device(dev); |
Bommu Krishnaiah | e670f0b | 2023-12-15 15:45:34 +0000 | [diff] [blame] | 105 | struct xe_file *xef = to_xe_file(file); |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 106 | DEFINE_WAIT_FUNC(w_wait, woken_wake_function); |
| 107 | struct drm_xe_wait_user_fence *args = data; |
Bommu Krishnaiah | e670f0b | 2023-12-15 15:45:34 +0000 | [diff] [blame] | 108 | struct xe_exec_queue *q = NULL; |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 109 | u64 addr = args->addr; |
Bommu Krishnaiah | e670f0b | 2023-12-15 15:45:34 +0000 | [diff] [blame] | 110 | int err = 0; |
Fei Yang | e2e2d96 | 2023-09-21 15:05:00 -0700 | [diff] [blame] | 111 | long timeout; |
Zbigniew Kempczyński | 5572a00 | 2023-06-28 07:51:41 +0200 | [diff] [blame] | 112 | ktime_t start; |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 113 | |
Francois Dugast | b8c1ba8 | 2023-07-17 10:20:18 +0200 | [diff] [blame] | 114 | if (XE_IOCTL_DBG(xe, args->extensions) || XE_IOCTL_DBG(xe, args->pad) || |
Bommu Krishnaiah | 9212da0 | 2023-12-15 15:45:33 +0000 | [diff] [blame] | 115 | XE_IOCTL_DBG(xe, args->pad2) || |
Francois Dugast | b8c1ba8 | 2023-07-17 10:20:18 +0200 | [diff] [blame] | 116 | XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 117 | return -EINVAL; |
| 118 | |
Francois Dugast | b8c1ba8 | 2023-07-17 10:20:18 +0200 | [diff] [blame] | 119 | if (XE_IOCTL_DBG(xe, args->flags & ~VALID_FLAGS)) |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 120 | return -EINVAL; |
| 121 | |
Francois Dugast | b8c1ba8 | 2023-07-17 10:20:18 +0200 | [diff] [blame] | 122 | if (XE_IOCTL_DBG(xe, args->op > MAX_OP)) |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 123 | return -EINVAL; |
| 124 | |
Matthew Brost | b21ae51 | 2023-09-14 13:40:49 -0700 | [diff] [blame] | 125 | if (XE_IOCTL_DBG(xe, addr & 0x7)) |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 126 | return -EINVAL; |
| 127 | |
Bommu Krishnaiah | e670f0b | 2023-12-15 15:45:34 +0000 | [diff] [blame] | 128 | if (args->exec_queue_id) { |
| 129 | q = xe_exec_queue_lookup(xef, args->exec_queue_id); |
| 130 | if (XE_IOCTL_DBG(xe, !q)) |
| 131 | return -ENOENT; |
| 132 | } |
| 133 | |
Fei Yang | e2e2d96 | 2023-09-21 15:05:00 -0700 | [diff] [blame] | 134 | timeout = to_jiffies_timeout(xe, args); |
Zbigniew Kempczyński | 5572a00 | 2023-06-28 07:51:41 +0200 | [diff] [blame] | 135 | |
| 136 | start = ktime_get(); |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 137 | |
Matthew Brost | b21ae51 | 2023-09-14 13:40:49 -0700 | [diff] [blame] | 138 | add_wait_queue(&xe->ufence_wq, &w_wait); |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 139 | for (;;) { |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 140 | err = do_compare(addr, args->value, args->mask, args->op); |
| 141 | if (err <= 0) |
| 142 | break; |
| 143 | |
| 144 | if (signal_pending(current)) { |
| 145 | err = -ERESTARTSYS; |
| 146 | break; |
| 147 | } |
| 148 | |
Bommu Krishnaiah | e670f0b | 2023-12-15 15:45:34 +0000 | [diff] [blame] | 149 | if (q) { |
| 150 | if (q->ops->reset_status(q)) { |
Colin Ian King | 264ed17 | 2024-01-02 09:20:14 +0000 | [diff] [blame] | 151 | drm_info(&xe->drm, "exec queue reset detected\n"); |
Bommu Krishnaiah | e670f0b | 2023-12-15 15:45:34 +0000 | [diff] [blame] | 152 | err = -EIO; |
| 153 | break; |
| 154 | } |
| 155 | } |
| 156 | |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 157 | if (!timeout) { |
| 158 | err = -ETIME; |
| 159 | break; |
| 160 | } |
| 161 | |
| 162 | timeout = wait_woken(&w_wait, TASK_INTERRUPTIBLE, timeout); |
| 163 | } |
Matthew Brost | b21ae51 | 2023-09-14 13:40:49 -0700 | [diff] [blame] | 164 | remove_wait_queue(&xe->ufence_wq, &w_wait); |
Zbigniew Kempczyński | 5572a00 | 2023-06-28 07:51:41 +0200 | [diff] [blame] | 165 | |
Francois Dugast | 3ac4a78 | 2023-11-14 13:34:28 +0000 | [diff] [blame] | 166 | if (!(args->flags & DRM_XE_UFENCE_WAIT_FLAG_ABSTIME)) { |
Zbigniew Kempczyński | 5572a00 | 2023-06-28 07:51:41 +0200 | [diff] [blame] | 167 | args->timeout -= ktime_to_ns(ktime_sub(ktime_get(), start)); |
| 168 | if (args->timeout < 0) |
| 169 | args->timeout = 0; |
| 170 | } |
| 171 | |
Bommu Krishnaiah | e670f0b | 2023-12-15 15:45:34 +0000 | [diff] [blame] | 172 | if (!timeout && !(err < 0)) |
| 173 | err = -ETIME; |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 174 | |
Bommu Krishnaiah | e670f0b | 2023-12-15 15:45:34 +0000 | [diff] [blame] | 175 | if (q) |
| 176 | xe_exec_queue_put(q); |
| 177 | |
| 178 | return err; |
Matthew Brost | dd08ebf | 2023-03-30 17:31:57 -0400 | [diff] [blame] | 179 | } |