Thomas Gleixner | 2874c5f | 2019-05-27 08:55:01 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
Uma Krishnan | 76ebe01 | 2018-03-26 11:30:51 -0500 | [diff] [blame] | 2 | /* |
| 3 | * CXL Flash Device Driver |
| 4 | * |
| 5 | * Written by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation |
| 6 | * Uma Krishnan <ukrishn@linux.vnet.ibm.com>, IBM Corporation |
| 7 | * |
| 8 | * Copyright (C) 2018 IBM Corporation |
Uma Krishnan | 76ebe01 | 2018-03-26 11:30:51 -0500 | [diff] [blame] | 9 | */ |
| 10 | |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 11 | #include <linux/file.h> |
Uma Krishnan | 429ebfa | 2018-03-26 11:32:00 -0500 | [diff] [blame] | 12 | #include <linux/idr.h> |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 13 | #include <linux/module.h> |
| 14 | #include <linux/mount.h> |
David Howells | 219e696 | 2019-03-25 16:38:25 +0000 | [diff] [blame] | 15 | #include <linux/pseudo_fs.h> |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 16 | #include <linux/poll.h> |
Uma Krishnan | 03aa9c5 | 2018-03-26 11:34:12 -0500 | [diff] [blame] | 17 | #include <linux/sched/signal.h> |
Frederic Barrat | 1e89da5 | 2020-04-03 17:38:35 +0200 | [diff] [blame] | 18 | #include <linux/interrupt.h> |
Christophe Leroy | 61657dc | 2022-07-08 09:11:06 +0200 | [diff] [blame] | 19 | #include <linux/irqdomain.h> |
Frederic Barrat | 1e89da5 | 2020-04-03 17:38:35 +0200 | [diff] [blame] | 20 | #include <asm/xive.h> |
Uma Krishnan | 76ebe01 | 2018-03-26 11:30:51 -0500 | [diff] [blame] | 21 | #include <misc/ocxl.h> |
| 22 | |
Uma Krishnan | 03aa9c5 | 2018-03-26 11:34:12 -0500 | [diff] [blame] | 23 | #include <uapi/misc/cxl.h> |
| 24 | |
Uma Krishnan | 76ebe01 | 2018-03-26 11:30:51 -0500 | [diff] [blame] | 25 | #include "backend.h" |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 26 | #include "ocxl_hw.h" |
| 27 | |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 28 | /* |
| 29 | * Pseudo-filesystem to allocate inodes. |
| 30 | */ |
| 31 | |
| 32 | #define OCXLFLASH_FS_MAGIC 0x1697698f |
| 33 | |
| 34 | static int ocxlflash_fs_cnt; |
| 35 | static struct vfsmount *ocxlflash_vfs_mount; |
| 36 | |
David Howells | 219e696 | 2019-03-25 16:38:25 +0000 | [diff] [blame] | 37 | static int ocxlflash_fs_init_fs_context(struct fs_context *fc) |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 38 | { |
David Howells | 219e696 | 2019-03-25 16:38:25 +0000 | [diff] [blame] | 39 | return init_pseudo(fc, OCXLFLASH_FS_MAGIC) ? 0 : -ENOMEM; |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 40 | } |
| 41 | |
| 42 | static struct file_system_type ocxlflash_fs_type = { |
| 43 | .name = "ocxlflash", |
| 44 | .owner = THIS_MODULE, |
David Howells | 219e696 | 2019-03-25 16:38:25 +0000 | [diff] [blame] | 45 | .init_fs_context = ocxlflash_fs_init_fs_context, |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 46 | .kill_sb = kill_anon_super, |
| 47 | }; |
| 48 | |
| 49 | /* |
| 50 | * ocxlflash_release_mapping() - release the memory mapping |
| 51 | * @ctx: Context whose mapping is to be released. |
| 52 | */ |
| 53 | static void ocxlflash_release_mapping(struct ocxlflash_context *ctx) |
| 54 | { |
| 55 | if (ctx->mapping) |
| 56 | simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt); |
| 57 | ctx->mapping = NULL; |
| 58 | } |
| 59 | |
| 60 | /* |
| 61 | * ocxlflash_getfile() - allocate pseudo filesystem, inode, and the file |
| 62 | * @dev: Generic device of the host. |
| 63 | * @name: Name of the pseudo filesystem. |
| 64 | * @fops: File operations. |
| 65 | * @priv: Private data. |
| 66 | * @flags: Flags for the file. |
| 67 | * |
| 68 | * Return: pointer to the file on success, ERR_PTR on failure |
| 69 | */ |
| 70 | static struct file *ocxlflash_getfile(struct device *dev, const char *name, |
| 71 | const struct file_operations *fops, |
| 72 | void *priv, int flags) |
| 73 | { |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 74 | struct file *file; |
Al Viro | 118f486 | 2018-06-17 12:42:49 -0400 | [diff] [blame] | 75 | struct inode *inode; |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 76 | int rc; |
| 77 | |
| 78 | if (fops->owner && !try_module_get(fops->owner)) { |
| 79 | dev_err(dev, "%s: Owner does not exist\n", __func__); |
| 80 | rc = -ENOENT; |
| 81 | goto err1; |
| 82 | } |
| 83 | |
| 84 | rc = simple_pin_fs(&ocxlflash_fs_type, &ocxlflash_vfs_mount, |
| 85 | &ocxlflash_fs_cnt); |
| 86 | if (unlikely(rc < 0)) { |
| 87 | dev_err(dev, "%s: Cannot mount ocxlflash pseudofs rc=%d\n", |
| 88 | __func__, rc); |
| 89 | goto err2; |
| 90 | } |
| 91 | |
| 92 | inode = alloc_anon_inode(ocxlflash_vfs_mount->mnt_sb); |
| 93 | if (IS_ERR(inode)) { |
| 94 | rc = PTR_ERR(inode); |
| 95 | dev_err(dev, "%s: alloc_anon_inode failed rc=%d\n", |
| 96 | __func__, rc); |
| 97 | goto err3; |
| 98 | } |
| 99 | |
Al Viro | 118f486 | 2018-06-17 12:42:49 -0400 | [diff] [blame] | 100 | file = alloc_file_pseudo(inode, ocxlflash_vfs_mount, name, |
| 101 | flags & (O_ACCMODE | O_NONBLOCK), fops); |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 102 | if (IS_ERR(file)) { |
| 103 | rc = PTR_ERR(file); |
| 104 | dev_err(dev, "%s: alloc_file failed rc=%d\n", |
| 105 | __func__, rc); |
Al Viro | 118f486 | 2018-06-17 12:42:49 -0400 | [diff] [blame] | 106 | goto err4; |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 107 | } |
| 108 | |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 109 | file->private_data = priv; |
| 110 | out: |
| 111 | return file; |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 112 | err4: |
| 113 | iput(inode); |
| 114 | err3: |
| 115 | simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt); |
| 116 | err2: |
| 117 | module_put(fops->owner); |
| 118 | err1: |
| 119 | file = ERR_PTR(rc); |
| 120 | goto out; |
| 121 | } |
| 122 | |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 123 | /** |
Uma Krishnan | 012f394 | 2018-03-26 11:32:56 -0500 | [diff] [blame] | 124 | * ocxlflash_psa_map() - map the process specific MMIO space |
| 125 | * @ctx_cookie: Adapter context for which the mapping needs to be done. |
| 126 | * |
| 127 | * Return: MMIO pointer of the mapped region |
| 128 | */ |
| 129 | static void __iomem *ocxlflash_psa_map(void *ctx_cookie) |
| 130 | { |
| 131 | struct ocxlflash_context *ctx = ctx_cookie; |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 132 | struct device *dev = ctx->hw_afu->dev; |
| 133 | |
| 134 | mutex_lock(&ctx->state_mutex); |
| 135 | if (ctx->state != STARTED) { |
| 136 | dev_err(dev, "%s: Context not started, state=%d\n", __func__, |
| 137 | ctx->state); |
| 138 | mutex_unlock(&ctx->state_mutex); |
| 139 | return NULL; |
| 140 | } |
| 141 | mutex_unlock(&ctx->state_mutex); |
Uma Krishnan | 012f394 | 2018-03-26 11:32:56 -0500 | [diff] [blame] | 142 | |
| 143 | return ioremap(ctx->psn_phys, ctx->psn_size); |
| 144 | } |
| 145 | |
| 146 | /** |
| 147 | * ocxlflash_psa_unmap() - unmap the process specific MMIO space |
| 148 | * @addr: MMIO pointer to unmap. |
| 149 | */ |
| 150 | static void ocxlflash_psa_unmap(void __iomem *addr) |
| 151 | { |
| 152 | iounmap(addr); |
| 153 | } |
| 154 | |
| 155 | /** |
Uma Krishnan | b18718c | 2018-03-26 11:32:20 -0500 | [diff] [blame] | 156 | * ocxlflash_process_element() - get process element of the adapter context |
| 157 | * @ctx_cookie: Adapter context associated with the process element. |
| 158 | * |
| 159 | * Return: process element of the adapter context |
| 160 | */ |
| 161 | static int ocxlflash_process_element(void *ctx_cookie) |
| 162 | { |
| 163 | struct ocxlflash_context *ctx = ctx_cookie; |
| 164 | |
| 165 | return ctx->pe; |
| 166 | } |
| 167 | |
| 168 | /** |
Uma Krishnan | a06b1cf | 2018-03-26 11:33:48 -0500 | [diff] [blame] | 169 | * afu_map_irq() - map the interrupt of the adapter context |
| 170 | * @flags: Flags. |
| 171 | * @ctx: Adapter context. |
| 172 | * @num: Per-context AFU interrupt number. |
| 173 | * @handler: Interrupt handler to register. |
| 174 | * @cookie: Interrupt handler private data. |
| 175 | * @name: Name of the interrupt. |
| 176 | * |
| 177 | * Return: 0 on success, -errno on failure |
| 178 | */ |
| 179 | static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num, |
| 180 | irq_handler_t handler, void *cookie, char *name) |
| 181 | { |
| 182 | struct ocxl_hw_afu *afu = ctx->hw_afu; |
| 183 | struct device *dev = afu->dev; |
| 184 | struct ocxlflash_irqs *irq; |
Frederic Barrat | 1e89da5 | 2020-04-03 17:38:35 +0200 | [diff] [blame] | 185 | struct xive_irq_data *xd; |
Uma Krishnan | a06b1cf | 2018-03-26 11:33:48 -0500 | [diff] [blame] | 186 | u32 virq; |
| 187 | int rc = 0; |
| 188 | |
| 189 | if (num < 0 || num >= ctx->num_irqs) { |
| 190 | dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num); |
| 191 | rc = -ENOENT; |
| 192 | goto out; |
| 193 | } |
| 194 | |
| 195 | irq = &ctx->irqs[num]; |
| 196 | virq = irq_create_mapping(NULL, irq->hwirq); |
| 197 | if (unlikely(!virq)) { |
| 198 | dev_err(dev, "%s: irq_create_mapping failed\n", __func__); |
| 199 | rc = -ENOMEM; |
| 200 | goto out; |
| 201 | } |
| 202 | |
| 203 | rc = request_irq(virq, handler, 0, name, cookie); |
| 204 | if (unlikely(rc)) { |
| 205 | dev_err(dev, "%s: request_irq failed rc=%d\n", __func__, rc); |
| 206 | goto err1; |
| 207 | } |
| 208 | |
Frederic Barrat | 1e89da5 | 2020-04-03 17:38:35 +0200 | [diff] [blame] | 209 | xd = irq_get_handler_data(virq); |
| 210 | if (unlikely(!xd)) { |
| 211 | dev_err(dev, "%s: Can't get interrupt data\n", __func__); |
| 212 | rc = -ENXIO; |
Uma Krishnan | a06b1cf | 2018-03-26 11:33:48 -0500 | [diff] [blame] | 213 | goto err2; |
| 214 | } |
| 215 | |
| 216 | irq->virq = virq; |
Frederic Barrat | 1e89da5 | 2020-04-03 17:38:35 +0200 | [diff] [blame] | 217 | irq->vtrig = xd->trig_mmio; |
Uma Krishnan | a06b1cf | 2018-03-26 11:33:48 -0500 | [diff] [blame] | 218 | out: |
| 219 | return rc; |
| 220 | err2: |
| 221 | free_irq(virq, cookie); |
| 222 | err1: |
| 223 | irq_dispose_mapping(virq); |
| 224 | goto out; |
| 225 | } |
| 226 | |
| 227 | /** |
| 228 | * ocxlflash_map_afu_irq() - map the interrupt of the adapter context |
| 229 | * @ctx_cookie: Adapter context. |
| 230 | * @num: Per-context AFU interrupt number. |
| 231 | * @handler: Interrupt handler to register. |
| 232 | * @cookie: Interrupt handler private data. |
| 233 | * @name: Name of the interrupt. |
| 234 | * |
| 235 | * Return: 0 on success, -errno on failure |
| 236 | */ |
| 237 | static int ocxlflash_map_afu_irq(void *ctx_cookie, int num, |
| 238 | irq_handler_t handler, void *cookie, |
| 239 | char *name) |
| 240 | { |
| 241 | return afu_map_irq(0, ctx_cookie, num, handler, cookie, name); |
| 242 | } |
| 243 | |
| 244 | /** |
| 245 | * afu_unmap_irq() - unmap the interrupt |
| 246 | * @flags: Flags. |
| 247 | * @ctx: Adapter context. |
| 248 | * @num: Per-context AFU interrupt number. |
| 249 | * @cookie: Interrupt handler private data. |
| 250 | */ |
| 251 | static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num, |
| 252 | void *cookie) |
| 253 | { |
| 254 | struct ocxl_hw_afu *afu = ctx->hw_afu; |
| 255 | struct device *dev = afu->dev; |
| 256 | struct ocxlflash_irqs *irq; |
| 257 | |
| 258 | if (num < 0 || num >= ctx->num_irqs) { |
| 259 | dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num); |
| 260 | return; |
| 261 | } |
| 262 | |
| 263 | irq = &ctx->irqs[num]; |
Uma Krishnan | a06b1cf | 2018-03-26 11:33:48 -0500 | [diff] [blame] | 264 | |
| 265 | if (irq_find_mapping(NULL, irq->hwirq)) { |
| 266 | free_irq(irq->virq, cookie); |
| 267 | irq_dispose_mapping(irq->virq); |
| 268 | } |
| 269 | |
| 270 | memset(irq, 0, sizeof(*irq)); |
| 271 | } |
| 272 | |
| 273 | /** |
| 274 | * ocxlflash_unmap_afu_irq() - unmap the interrupt |
| 275 | * @ctx_cookie: Adapter context. |
| 276 | * @num: Per-context AFU interrupt number. |
| 277 | * @cookie: Interrupt handler private data. |
| 278 | */ |
| 279 | static void ocxlflash_unmap_afu_irq(void *ctx_cookie, int num, void *cookie) |
| 280 | { |
| 281 | return afu_unmap_irq(0, ctx_cookie, num, cookie); |
| 282 | } |
| 283 | |
| 284 | /** |
Uma Krishnan | 402a55e | 2018-03-26 11:34:35 -0500 | [diff] [blame] | 285 | * ocxlflash_get_irq_objhndl() - get the object handle for an interrupt |
| 286 | * @ctx_cookie: Context associated with the interrupt. |
| 287 | * @irq: Interrupt number. |
| 288 | * |
| 289 | * Return: effective address of the mapped region |
| 290 | */ |
| 291 | static u64 ocxlflash_get_irq_objhndl(void *ctx_cookie, int irq) |
| 292 | { |
| 293 | struct ocxlflash_context *ctx = ctx_cookie; |
| 294 | |
| 295 | if (irq < 0 || irq >= ctx->num_irqs) |
| 296 | return 0; |
| 297 | |
| 298 | return (__force u64)ctx->irqs[irq].vtrig; |
| 299 | } |
| 300 | |
| 301 | /** |
Uma Krishnan | 66ae644 | 2018-03-26 11:35:07 -0500 | [diff] [blame] | 302 | * ocxlflash_xsl_fault() - callback when translation error is triggered |
| 303 | * @data: Private data provided at callback registration, the context. |
| 304 | * @addr: Address that triggered the error. |
| 305 | * @dsisr: Value of dsisr register. |
| 306 | */ |
| 307 | static void ocxlflash_xsl_fault(void *data, u64 addr, u64 dsisr) |
| 308 | { |
| 309 | struct ocxlflash_context *ctx = data; |
| 310 | |
| 311 | spin_lock(&ctx->slock); |
| 312 | ctx->fault_addr = addr; |
| 313 | ctx->fault_dsisr = dsisr; |
| 314 | ctx->pending_fault = true; |
| 315 | spin_unlock(&ctx->slock); |
| 316 | |
| 317 | wake_up_all(&ctx->wq); |
| 318 | } |
| 319 | |
| 320 | /** |
Uma Krishnan | 6b938ac | 2018-03-26 11:32:48 -0500 | [diff] [blame] | 321 | * start_context() - local routine to start a context |
| 322 | * @ctx: Adapter context to be started. |
| 323 | * |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 324 | * Assign the context specific MMIO space, add and enable the PE. |
Uma Krishnan | 6b938ac | 2018-03-26 11:32:48 -0500 | [diff] [blame] | 325 | * |
| 326 | * Return: 0 on success, -errno on failure |
| 327 | */ |
| 328 | static int start_context(struct ocxlflash_context *ctx) |
| 329 | { |
| 330 | struct ocxl_hw_afu *afu = ctx->hw_afu; |
| 331 | struct ocxl_afu_config *acfg = &afu->acfg; |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 332 | void *link_token = afu->link_token; |
Christophe Lombard | d731fee | 2020-11-25 16:50:11 +0100 | [diff] [blame] | 333 | struct pci_dev *pdev = afu->pdev; |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 334 | struct device *dev = afu->dev; |
Uma Krishnan | 6b938ac | 2018-03-26 11:32:48 -0500 | [diff] [blame] | 335 | bool master = ctx->master; |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 336 | struct mm_struct *mm; |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 337 | int rc = 0; |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 338 | u32 pid; |
Uma Krishnan | 6b938ac | 2018-03-26 11:32:48 -0500 | [diff] [blame] | 339 | |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 340 | mutex_lock(&ctx->state_mutex); |
| 341 | if (ctx->state != OPENED) { |
| 342 | dev_err(dev, "%s: Context state invalid, state=%d\n", |
| 343 | __func__, ctx->state); |
| 344 | rc = -EINVAL; |
| 345 | goto out; |
| 346 | } |
| 347 | |
Uma Krishnan | 6b938ac | 2018-03-26 11:32:48 -0500 | [diff] [blame] | 348 | if (master) { |
| 349 | ctx->psn_size = acfg->global_mmio_size; |
| 350 | ctx->psn_phys = afu->gmmio_phys; |
| 351 | } else { |
| 352 | ctx->psn_size = acfg->pp_mmio_stride; |
| 353 | ctx->psn_phys = afu->ppmmio_phys + (ctx->pe * ctx->psn_size); |
| 354 | } |
| 355 | |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 356 | /* pid and mm not set for master contexts */ |
| 357 | if (master) { |
| 358 | pid = 0; |
| 359 | mm = NULL; |
| 360 | } else { |
| 361 | pid = current->mm->context.id; |
| 362 | mm = current->mm; |
| 363 | } |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 364 | |
Christophe Lombard | d731fee | 2020-11-25 16:50:11 +0100 | [diff] [blame] | 365 | rc = ocxl_link_add_pe(link_token, ctx->pe, pid, 0, 0, |
| 366 | pci_dev_id(pdev), mm, ocxlflash_xsl_fault, |
| 367 | ctx); |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 368 | if (unlikely(rc)) { |
| 369 | dev_err(dev, "%s: ocxl_link_add_pe failed rc=%d\n", |
| 370 | __func__, rc); |
| 371 | goto out; |
| 372 | } |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 373 | |
| 374 | ctx->state = STARTED; |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 375 | out: |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 376 | mutex_unlock(&ctx->state_mutex); |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 377 | return rc; |
Uma Krishnan | 6b938ac | 2018-03-26 11:32:48 -0500 | [diff] [blame] | 378 | } |
| 379 | |
| 380 | /** |
| 381 | * ocxlflash_start_context() - start a kernel context |
| 382 | * @ctx_cookie: Adapter context to be started. |
| 383 | * |
| 384 | * Return: 0 on success, -errno on failure |
| 385 | */ |
| 386 | static int ocxlflash_start_context(void *ctx_cookie) |
| 387 | { |
| 388 | struct ocxlflash_context *ctx = ctx_cookie; |
| 389 | |
| 390 | return start_context(ctx); |
| 391 | } |
| 392 | |
| 393 | /** |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 394 | * ocxlflash_stop_context() - stop a context |
| 395 | * @ctx_cookie: Adapter context to be stopped. |
| 396 | * |
| 397 | * Return: 0 on success, -errno on failure |
| 398 | */ |
| 399 | static int ocxlflash_stop_context(void *ctx_cookie) |
| 400 | { |
| 401 | struct ocxlflash_context *ctx = ctx_cookie; |
| 402 | struct ocxl_hw_afu *afu = ctx->hw_afu; |
| 403 | struct ocxl_afu_config *acfg = &afu->acfg; |
| 404 | struct pci_dev *pdev = afu->pdev; |
| 405 | struct device *dev = afu->dev; |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 406 | enum ocxlflash_ctx_state state; |
| 407 | int rc = 0; |
| 408 | |
| 409 | mutex_lock(&ctx->state_mutex); |
| 410 | state = ctx->state; |
| 411 | ctx->state = CLOSED; |
| 412 | mutex_unlock(&ctx->state_mutex); |
| 413 | if (state != STARTED) |
| 414 | goto out; |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 415 | |
| 416 | rc = ocxl_config_terminate_pasid(pdev, acfg->dvsec_afu_control_pos, |
| 417 | ctx->pe); |
| 418 | if (unlikely(rc)) { |
| 419 | dev_err(dev, "%s: ocxl_config_terminate_pasid failed rc=%d\n", |
| 420 | __func__, rc); |
| 421 | /* If EBUSY, PE could be referenced in future by the AFU */ |
| 422 | if (rc == -EBUSY) |
| 423 | goto out; |
| 424 | } |
| 425 | |
| 426 | rc = ocxl_link_remove_pe(afu->link_token, ctx->pe); |
| 427 | if (unlikely(rc)) { |
| 428 | dev_err(dev, "%s: ocxl_link_remove_pe failed rc=%d\n", |
| 429 | __func__, rc); |
| 430 | goto out; |
| 431 | } |
| 432 | out: |
| 433 | return rc; |
| 434 | } |
| 435 | |
| 436 | /** |
Uma Krishnan | 9433fb3 | 2018-03-26 11:35:15 -0500 | [diff] [blame] | 437 | * ocxlflash_afu_reset() - reset the AFU |
| 438 | * @ctx_cookie: Adapter context. |
| 439 | */ |
| 440 | static int ocxlflash_afu_reset(void *ctx_cookie) |
| 441 | { |
| 442 | struct ocxlflash_context *ctx = ctx_cookie; |
| 443 | struct device *dev = ctx->hw_afu->dev; |
| 444 | |
| 445 | /* Pending implementation from OCXL transport services */ |
| 446 | dev_err_once(dev, "%s: afu_reset() fop not supported\n", __func__); |
| 447 | |
| 448 | /* Silently return success until it is implemented */ |
| 449 | return 0; |
| 450 | } |
| 451 | |
| 452 | /** |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 453 | * ocxlflash_set_master() - sets the context as master |
| 454 | * @ctx_cookie: Adapter context to set as master. |
| 455 | */ |
| 456 | static void ocxlflash_set_master(void *ctx_cookie) |
| 457 | { |
| 458 | struct ocxlflash_context *ctx = ctx_cookie; |
| 459 | |
| 460 | ctx->master = true; |
| 461 | } |
| 462 | |
| 463 | /** |
| 464 | * ocxlflash_get_context() - obtains the context associated with the host |
| 465 | * @pdev: PCI device associated with the host. |
| 466 | * @afu_cookie: Hardware AFU associated with the host. |
| 467 | * |
| 468 | * Return: returns the pointer to host adapter context |
| 469 | */ |
| 470 | static void *ocxlflash_get_context(struct pci_dev *pdev, void *afu_cookie) |
| 471 | { |
| 472 | struct ocxl_hw_afu *afu = afu_cookie; |
| 473 | |
| 474 | return afu->ocxl_ctx; |
| 475 | } |
| 476 | |
| 477 | /** |
| 478 | * ocxlflash_dev_context_init() - allocate and initialize an adapter context |
| 479 | * @pdev: PCI device associated with the host. |
| 480 | * @afu_cookie: Hardware AFU associated with the host. |
| 481 | * |
| 482 | * Return: returns the adapter context on success, ERR_PTR on failure |
| 483 | */ |
| 484 | static void *ocxlflash_dev_context_init(struct pci_dev *pdev, void *afu_cookie) |
| 485 | { |
| 486 | struct ocxl_hw_afu *afu = afu_cookie; |
| 487 | struct device *dev = afu->dev; |
| 488 | struct ocxlflash_context *ctx; |
| 489 | int rc; |
| 490 | |
| 491 | ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); |
| 492 | if (unlikely(!ctx)) { |
| 493 | dev_err(dev, "%s: Context allocation failed\n", __func__); |
| 494 | rc = -ENOMEM; |
Uma Krishnan | 429ebfa | 2018-03-26 11:32:00 -0500 | [diff] [blame] | 495 | goto err1; |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 496 | } |
| 497 | |
Uma Krishnan | 429ebfa | 2018-03-26 11:32:00 -0500 | [diff] [blame] | 498 | idr_preload(GFP_KERNEL); |
| 499 | rc = idr_alloc(&afu->idr, ctx, 0, afu->max_pasid, GFP_NOWAIT); |
| 500 | idr_preload_end(); |
| 501 | if (unlikely(rc < 0)) { |
| 502 | dev_err(dev, "%s: idr_alloc failed rc=%d\n", __func__, rc); |
| 503 | goto err2; |
| 504 | } |
| 505 | |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 506 | spin_lock_init(&ctx->slock); |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 507 | init_waitqueue_head(&ctx->wq); |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 508 | mutex_init(&ctx->state_mutex); |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 509 | |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 510 | ctx->state = OPENED; |
Uma Krishnan | 429ebfa | 2018-03-26 11:32:00 -0500 | [diff] [blame] | 511 | ctx->pe = rc; |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 512 | ctx->master = false; |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 513 | ctx->mapping = NULL; |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 514 | ctx->hw_afu = afu; |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 515 | ctx->irq_bitmap = 0; |
| 516 | ctx->pending_irq = false; |
Uma Krishnan | 66ae644 | 2018-03-26 11:35:07 -0500 | [diff] [blame] | 517 | ctx->pending_fault = false; |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 518 | out: |
| 519 | return ctx; |
Uma Krishnan | 429ebfa | 2018-03-26 11:32:00 -0500 | [diff] [blame] | 520 | err2: |
| 521 | kfree(ctx); |
| 522 | err1: |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 523 | ctx = ERR_PTR(rc); |
| 524 | goto out; |
| 525 | } |
| 526 | |
| 527 | /** |
| 528 | * ocxlflash_release_context() - releases an adapter context |
| 529 | * @ctx_cookie: Adapter context to be released. |
| 530 | * |
| 531 | * Return: 0 on success, -errno on failure |
| 532 | */ |
| 533 | static int ocxlflash_release_context(void *ctx_cookie) |
| 534 | { |
| 535 | struct ocxlflash_context *ctx = ctx_cookie; |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 536 | struct device *dev; |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 537 | int rc = 0; |
| 538 | |
| 539 | if (!ctx) |
| 540 | goto out; |
| 541 | |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 542 | dev = ctx->hw_afu->dev; |
| 543 | mutex_lock(&ctx->state_mutex); |
| 544 | if (ctx->state >= STARTED) { |
| 545 | dev_err(dev, "%s: Context in use, state=%d\n", __func__, |
| 546 | ctx->state); |
| 547 | mutex_unlock(&ctx->state_mutex); |
| 548 | rc = -EBUSY; |
| 549 | goto out; |
| 550 | } |
| 551 | mutex_unlock(&ctx->state_mutex); |
| 552 | |
Uma Krishnan | 429ebfa | 2018-03-26 11:32:00 -0500 | [diff] [blame] | 553 | idr_remove(&ctx->hw_afu->idr, ctx->pe); |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 554 | ocxlflash_release_mapping(ctx); |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 555 | kfree(ctx); |
| 556 | out: |
| 557 | return rc; |
| 558 | } |
| 559 | |
| 560 | /** |
Uma Krishnan | 8b7a5521 | 2018-03-26 11:32:29 -0500 | [diff] [blame] | 561 | * ocxlflash_perst_reloads_same_image() - sets the image reload policy |
| 562 | * @afu_cookie: Hardware AFU associated with the host. |
| 563 | * @image: Whether to load the same image on PERST. |
| 564 | */ |
| 565 | static void ocxlflash_perst_reloads_same_image(void *afu_cookie, bool image) |
| 566 | { |
| 567 | struct ocxl_hw_afu *afu = afu_cookie; |
| 568 | |
| 569 | afu->perst_same_image = image; |
| 570 | } |
| 571 | |
| 572 | /** |
Uma Krishnan | 119c920 | 2018-03-26 11:33:14 -0500 | [diff] [blame] | 573 | * ocxlflash_read_adapter_vpd() - reads the adapter VPD |
| 574 | * @pdev: PCI device associated with the host. |
| 575 | * @buf: Buffer to get the VPD data. |
| 576 | * @count: Size of buffer (maximum bytes that can be read). |
| 577 | * |
| 578 | * Return: size of VPD on success, -errno on failure |
| 579 | */ |
| 580 | static ssize_t ocxlflash_read_adapter_vpd(struct pci_dev *pdev, void *buf, |
| 581 | size_t count) |
| 582 | { |
| 583 | return pci_read_vpd(pdev, 0, count, buf); |
| 584 | } |
| 585 | |
| 586 | /** |
Uma Krishnan | bc65c1c | 2018-03-26 11:33:41 -0500 | [diff] [blame] | 587 | * free_afu_irqs() - internal service to free interrupts |
| 588 | * @ctx: Adapter context. |
| 589 | */ |
| 590 | static void free_afu_irqs(struct ocxlflash_context *ctx) |
| 591 | { |
| 592 | struct ocxl_hw_afu *afu = ctx->hw_afu; |
| 593 | struct device *dev = afu->dev; |
| 594 | int i; |
| 595 | |
| 596 | if (!ctx->irqs) { |
| 597 | dev_err(dev, "%s: Interrupts not allocated\n", __func__); |
| 598 | return; |
| 599 | } |
| 600 | |
| 601 | for (i = ctx->num_irqs; i >= 0; i--) |
| 602 | ocxl_link_free_irq(afu->link_token, ctx->irqs[i].hwirq); |
| 603 | |
| 604 | kfree(ctx->irqs); |
| 605 | ctx->irqs = NULL; |
| 606 | } |
| 607 | |
| 608 | /** |
| 609 | * alloc_afu_irqs() - internal service to allocate interrupts |
| 610 | * @ctx: Context associated with the request. |
| 611 | * @num: Number of interrupts requested. |
| 612 | * |
| 613 | * Return: 0 on success, -errno on failure |
| 614 | */ |
| 615 | static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num) |
| 616 | { |
| 617 | struct ocxl_hw_afu *afu = ctx->hw_afu; |
| 618 | struct device *dev = afu->dev; |
| 619 | struct ocxlflash_irqs *irqs; |
Uma Krishnan | bc65c1c | 2018-03-26 11:33:41 -0500 | [diff] [blame] | 620 | int rc = 0; |
| 621 | int hwirq; |
| 622 | int i; |
| 623 | |
| 624 | if (ctx->irqs) { |
| 625 | dev_err(dev, "%s: Interrupts already allocated\n", __func__); |
| 626 | rc = -EEXIST; |
| 627 | goto out; |
| 628 | } |
| 629 | |
| 630 | if (num > OCXL_MAX_IRQS) { |
| 631 | dev_err(dev, "%s: Too many interrupts num=%d\n", __func__, num); |
| 632 | rc = -EINVAL; |
| 633 | goto out; |
| 634 | } |
| 635 | |
| 636 | irqs = kcalloc(num, sizeof(*irqs), GFP_KERNEL); |
| 637 | if (unlikely(!irqs)) { |
| 638 | dev_err(dev, "%s: Context irqs allocation failed\n", __func__); |
| 639 | rc = -ENOMEM; |
| 640 | goto out; |
| 641 | } |
| 642 | |
| 643 | for (i = 0; i < num; i++) { |
Frederic Barrat | dde6f18 | 2020-04-03 17:38:37 +0200 | [diff] [blame] | 644 | rc = ocxl_link_irq_alloc(afu->link_token, &hwirq); |
Uma Krishnan | bc65c1c | 2018-03-26 11:33:41 -0500 | [diff] [blame] | 645 | if (unlikely(rc)) { |
| 646 | dev_err(dev, "%s: ocxl_link_irq_alloc failed rc=%d\n", |
| 647 | __func__, rc); |
| 648 | goto err; |
| 649 | } |
| 650 | |
| 651 | irqs[i].hwirq = hwirq; |
Uma Krishnan | bc65c1c | 2018-03-26 11:33:41 -0500 | [diff] [blame] | 652 | } |
| 653 | |
| 654 | ctx->irqs = irqs; |
| 655 | ctx->num_irqs = num; |
| 656 | out: |
| 657 | return rc; |
| 658 | err: |
| 659 | for (i = i-1; i >= 0; i--) |
| 660 | ocxl_link_free_irq(afu->link_token, irqs[i].hwirq); |
| 661 | kfree(irqs); |
| 662 | goto out; |
| 663 | } |
| 664 | |
| 665 | /** |
| 666 | * ocxlflash_allocate_afu_irqs() - allocates the requested number of interrupts |
| 667 | * @ctx_cookie: Context associated with the request. |
| 668 | * @num: Number of interrupts requested. |
| 669 | * |
| 670 | * Return: 0 on success, -errno on failure |
| 671 | */ |
| 672 | static int ocxlflash_allocate_afu_irqs(void *ctx_cookie, int num) |
| 673 | { |
| 674 | return alloc_afu_irqs(ctx_cookie, num); |
| 675 | } |
| 676 | |
| 677 | /** |
| 678 | * ocxlflash_free_afu_irqs() - frees the interrupts of an adapter context |
| 679 | * @ctx_cookie: Adapter context. |
| 680 | */ |
| 681 | static void ocxlflash_free_afu_irqs(void *ctx_cookie) |
| 682 | { |
| 683 | free_afu_irqs(ctx_cookie); |
| 684 | } |
| 685 | |
| 686 | /** |
Uma Krishnan | 5437050 | 2018-03-26 11:32:37 -0500 | [diff] [blame] | 687 | * ocxlflash_unconfig_afu() - unconfigure the AFU |
| 688 | * @afu: AFU associated with the host. |
| 689 | */ |
| 690 | static void ocxlflash_unconfig_afu(struct ocxl_hw_afu *afu) |
| 691 | { |
| 692 | if (afu->gmmio_virt) { |
| 693 | iounmap(afu->gmmio_virt); |
| 694 | afu->gmmio_virt = NULL; |
| 695 | } |
| 696 | } |
| 697 | |
| 698 | /** |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 699 | * ocxlflash_destroy_afu() - destroy the AFU structure |
| 700 | * @afu_cookie: AFU to be freed. |
| 701 | */ |
| 702 | static void ocxlflash_destroy_afu(void *afu_cookie) |
| 703 | { |
| 704 | struct ocxl_hw_afu *afu = afu_cookie; |
Uma Krishnan | 3351e4f | 2018-03-26 11:33:05 -0500 | [diff] [blame] | 705 | int pos; |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 706 | |
| 707 | if (!afu) |
| 708 | return; |
| 709 | |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 710 | ocxlflash_release_context(afu->ocxl_ctx); |
Uma Krishnan | 429ebfa | 2018-03-26 11:32:00 -0500 | [diff] [blame] | 711 | idr_destroy(&afu->idr); |
Uma Krishnan | 3351e4f | 2018-03-26 11:33:05 -0500 | [diff] [blame] | 712 | |
| 713 | /* Disable the AFU */ |
| 714 | pos = afu->acfg.dvsec_afu_control_pos; |
| 715 | ocxl_config_set_afu_state(afu->pdev, pos, 0); |
| 716 | |
Uma Krishnan | 5437050 | 2018-03-26 11:32:37 -0500 | [diff] [blame] | 717 | ocxlflash_unconfig_afu(afu); |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 718 | kfree(afu); |
| 719 | } |
| 720 | |
| 721 | /** |
Uma Krishnan | e9dfced | 2018-03-26 11:31:09 -0500 | [diff] [blame] | 722 | * ocxlflash_config_fn() - configure the host function |
| 723 | * @pdev: PCI device associated with the host. |
| 724 | * @afu: AFU associated with the host. |
| 725 | * |
| 726 | * Return: 0 on success, -errno on failure |
| 727 | */ |
| 728 | static int ocxlflash_config_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu) |
| 729 | { |
| 730 | struct ocxl_fn_config *fcfg = &afu->fcfg; |
| 731 | struct device *dev = &pdev->dev; |
Uma Krishnan | 2e22277 | 2018-03-26 11:31:21 -0500 | [diff] [blame] | 732 | u16 base, enabled, supported; |
Uma Krishnan | e9dfced | 2018-03-26 11:31:09 -0500 | [diff] [blame] | 733 | int rc = 0; |
| 734 | |
| 735 | /* Read DVSEC config of the function */ |
| 736 | rc = ocxl_config_read_function(pdev, fcfg); |
| 737 | if (unlikely(rc)) { |
| 738 | dev_err(dev, "%s: ocxl_config_read_function failed rc=%d\n", |
| 739 | __func__, rc); |
| 740 | goto out; |
| 741 | } |
| 742 | |
| 743 | /* Check if function has AFUs defined, only 1 per function supported */ |
| 744 | if (fcfg->max_afu_index >= 0) { |
| 745 | afu->is_present = true; |
| 746 | if (fcfg->max_afu_index != 0) |
| 747 | dev_warn(dev, "%s: Unexpected AFU index value %d\n", |
| 748 | __func__, fcfg->max_afu_index); |
| 749 | } |
Uma Krishnan | 2e22277 | 2018-03-26 11:31:21 -0500 | [diff] [blame] | 750 | |
| 751 | rc = ocxl_config_get_actag_info(pdev, &base, &enabled, &supported); |
| 752 | if (unlikely(rc)) { |
| 753 | dev_err(dev, "%s: ocxl_config_get_actag_info failed rc=%d\n", |
| 754 | __func__, rc); |
| 755 | goto out; |
| 756 | } |
| 757 | |
| 758 | afu->fn_actag_base = base; |
| 759 | afu->fn_actag_enabled = enabled; |
| 760 | |
| 761 | ocxl_config_set_actag(pdev, fcfg->dvsec_function_pos, base, enabled); |
| 762 | dev_dbg(dev, "%s: Function acTag range base=%u enabled=%u\n", |
| 763 | __func__, base, enabled); |
Uma Krishnan | 7390482 | 2018-03-26 11:33:21 -0500 | [diff] [blame] | 764 | |
| 765 | rc = ocxl_link_setup(pdev, 0, &afu->link_token); |
| 766 | if (unlikely(rc)) { |
| 767 | dev_err(dev, "%s: ocxl_link_setup failed rc=%d\n", |
| 768 | __func__, rc); |
| 769 | goto out; |
| 770 | } |
Uma Krishnan | c52bf5b | 2018-03-26 11:33:28 -0500 | [diff] [blame] | 771 | |
| 772 | rc = ocxl_config_set_TL(pdev, fcfg->dvsec_tl_pos); |
| 773 | if (unlikely(rc)) { |
| 774 | dev_err(dev, "%s: ocxl_config_set_TL failed rc=%d\n", |
| 775 | __func__, rc); |
| 776 | goto err; |
| 777 | } |
Uma Krishnan | e9dfced | 2018-03-26 11:31:09 -0500 | [diff] [blame] | 778 | out: |
| 779 | return rc; |
Uma Krishnan | c52bf5b | 2018-03-26 11:33:28 -0500 | [diff] [blame] | 780 | err: |
| 781 | ocxl_link_release(pdev, afu->link_token); |
| 782 | goto out; |
Uma Krishnan | e9dfced | 2018-03-26 11:31:09 -0500 | [diff] [blame] | 783 | } |
| 784 | |
| 785 | /** |
Uma Krishnan | 7390482 | 2018-03-26 11:33:21 -0500 | [diff] [blame] | 786 | * ocxlflash_unconfig_fn() - unconfigure the host function |
| 787 | * @pdev: PCI device associated with the host. |
| 788 | * @afu: AFU associated with the host. |
| 789 | */ |
| 790 | static void ocxlflash_unconfig_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu) |
| 791 | { |
| 792 | ocxl_link_release(pdev, afu->link_token); |
| 793 | } |
| 794 | |
| 795 | /** |
Uma Krishnan | 5437050 | 2018-03-26 11:32:37 -0500 | [diff] [blame] | 796 | * ocxlflash_map_mmio() - map the AFU MMIO space |
| 797 | * @afu: AFU associated with the host. |
| 798 | * |
| 799 | * Return: 0 on success, -errno on failure |
| 800 | */ |
| 801 | static int ocxlflash_map_mmio(struct ocxl_hw_afu *afu) |
| 802 | { |
| 803 | struct ocxl_afu_config *acfg = &afu->acfg; |
| 804 | struct pci_dev *pdev = afu->pdev; |
| 805 | struct device *dev = afu->dev; |
| 806 | phys_addr_t gmmio, ppmmio; |
| 807 | int rc = 0; |
| 808 | |
| 809 | rc = pci_request_region(pdev, acfg->global_mmio_bar, "ocxlflash"); |
| 810 | if (unlikely(rc)) { |
| 811 | dev_err(dev, "%s: pci_request_region for global failed rc=%d\n", |
| 812 | __func__, rc); |
| 813 | goto out; |
| 814 | } |
| 815 | gmmio = pci_resource_start(pdev, acfg->global_mmio_bar); |
| 816 | gmmio += acfg->global_mmio_offset; |
| 817 | |
| 818 | rc = pci_request_region(pdev, acfg->pp_mmio_bar, "ocxlflash"); |
| 819 | if (unlikely(rc)) { |
| 820 | dev_err(dev, "%s: pci_request_region for pp bar failed rc=%d\n", |
| 821 | __func__, rc); |
| 822 | goto err1; |
| 823 | } |
| 824 | ppmmio = pci_resource_start(pdev, acfg->pp_mmio_bar); |
| 825 | ppmmio += acfg->pp_mmio_offset; |
| 826 | |
| 827 | afu->gmmio_virt = ioremap(gmmio, acfg->global_mmio_size); |
| 828 | if (unlikely(!afu->gmmio_virt)) { |
| 829 | dev_err(dev, "%s: MMIO mapping failed\n", __func__); |
| 830 | rc = -ENOMEM; |
| 831 | goto err2; |
| 832 | } |
| 833 | |
| 834 | afu->gmmio_phys = gmmio; |
| 835 | afu->ppmmio_phys = ppmmio; |
| 836 | out: |
| 837 | return rc; |
| 838 | err2: |
| 839 | pci_release_region(pdev, acfg->pp_mmio_bar); |
| 840 | err1: |
| 841 | pci_release_region(pdev, acfg->global_mmio_bar); |
| 842 | goto out; |
| 843 | } |
| 844 | |
| 845 | /** |
Uma Krishnan | 9cc8429 | 2018-03-26 11:31:29 -0500 | [diff] [blame] | 846 | * ocxlflash_config_afu() - configure the host AFU |
| 847 | * @pdev: PCI device associated with the host. |
| 848 | * @afu: AFU associated with the host. |
| 849 | * |
| 850 | * Must be called _after_ host function configuration. |
| 851 | * |
| 852 | * Return: 0 on success, -errno on failure |
| 853 | */ |
| 854 | static int ocxlflash_config_afu(struct pci_dev *pdev, struct ocxl_hw_afu *afu) |
| 855 | { |
| 856 | struct ocxl_afu_config *acfg = &afu->acfg; |
| 857 | struct ocxl_fn_config *fcfg = &afu->fcfg; |
| 858 | struct device *dev = &pdev->dev; |
Uma Krishnan | d926519e | 2018-03-26 11:31:36 -0500 | [diff] [blame] | 859 | int count; |
| 860 | int base; |
| 861 | int pos; |
Uma Krishnan | 9cc8429 | 2018-03-26 11:31:29 -0500 | [diff] [blame] | 862 | int rc = 0; |
| 863 | |
| 864 | /* This HW AFU function does not have any AFUs defined */ |
| 865 | if (!afu->is_present) |
| 866 | goto out; |
| 867 | |
| 868 | /* Read AFU config at index 0 */ |
| 869 | rc = ocxl_config_read_afu(pdev, fcfg, acfg, 0); |
| 870 | if (unlikely(rc)) { |
| 871 | dev_err(dev, "%s: ocxl_config_read_afu failed rc=%d\n", |
| 872 | __func__, rc); |
| 873 | goto out; |
| 874 | } |
Uma Krishnan | d926519e | 2018-03-26 11:31:36 -0500 | [diff] [blame] | 875 | |
| 876 | /* Only one AFU per function is supported, so actag_base is same */ |
| 877 | base = afu->fn_actag_base; |
| 878 | count = min_t(int, acfg->actag_supported, afu->fn_actag_enabled); |
| 879 | pos = acfg->dvsec_afu_control_pos; |
| 880 | |
| 881 | ocxl_config_set_afu_actag(pdev, pos, base, count); |
| 882 | dev_dbg(dev, "%s: acTag base=%d enabled=%d\n", __func__, base, count); |
| 883 | afu->afu_actag_base = base; |
| 884 | afu->afu_actag_enabled = count; |
Uma Krishnan | 41df40d | 2018-03-26 11:31:44 -0500 | [diff] [blame] | 885 | afu->max_pasid = 1 << acfg->pasid_supported_log; |
| 886 | |
| 887 | ocxl_config_set_afu_pasid(pdev, pos, 0, acfg->pasid_supported_log); |
Uma Krishnan | 5437050 | 2018-03-26 11:32:37 -0500 | [diff] [blame] | 888 | |
| 889 | rc = ocxlflash_map_mmio(afu); |
| 890 | if (unlikely(rc)) { |
| 891 | dev_err(dev, "%s: ocxlflash_map_mmio failed rc=%d\n", |
| 892 | __func__, rc); |
| 893 | goto out; |
| 894 | } |
Uma Krishnan | 3351e4f | 2018-03-26 11:33:05 -0500 | [diff] [blame] | 895 | |
| 896 | /* Enable the AFU */ |
| 897 | ocxl_config_set_afu_state(pdev, acfg->dvsec_afu_control_pos, 1); |
Uma Krishnan | 9cc8429 | 2018-03-26 11:31:29 -0500 | [diff] [blame] | 898 | out: |
| 899 | return rc; |
| 900 | } |
| 901 | |
| 902 | /** |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 903 | * ocxlflash_create_afu() - create the AFU for OCXL |
| 904 | * @pdev: PCI device associated with the host. |
| 905 | * |
| 906 | * Return: AFU on success, NULL on failure |
| 907 | */ |
| 908 | static void *ocxlflash_create_afu(struct pci_dev *pdev) |
| 909 | { |
| 910 | struct device *dev = &pdev->dev; |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 911 | struct ocxlflash_context *ctx; |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 912 | struct ocxl_hw_afu *afu; |
Uma Krishnan | e9dfced | 2018-03-26 11:31:09 -0500 | [diff] [blame] | 913 | int rc; |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 914 | |
| 915 | afu = kzalloc(sizeof(*afu), GFP_KERNEL); |
| 916 | if (unlikely(!afu)) { |
| 917 | dev_err(dev, "%s: HW AFU allocation failed\n", __func__); |
| 918 | goto out; |
| 919 | } |
| 920 | |
| 921 | afu->pdev = pdev; |
| 922 | afu->dev = dev; |
Uma Krishnan | 429ebfa | 2018-03-26 11:32:00 -0500 | [diff] [blame] | 923 | idr_init(&afu->idr); |
Uma Krishnan | e9dfced | 2018-03-26 11:31:09 -0500 | [diff] [blame] | 924 | |
| 925 | rc = ocxlflash_config_fn(pdev, afu); |
| 926 | if (unlikely(rc)) { |
| 927 | dev_err(dev, "%s: Function configuration failed rc=%d\n", |
| 928 | __func__, rc); |
| 929 | goto err1; |
| 930 | } |
Uma Krishnan | 9cc8429 | 2018-03-26 11:31:29 -0500 | [diff] [blame] | 931 | |
| 932 | rc = ocxlflash_config_afu(pdev, afu); |
| 933 | if (unlikely(rc)) { |
| 934 | dev_err(dev, "%s: AFU configuration failed rc=%d\n", |
| 935 | __func__, rc); |
Uma Krishnan | 7390482 | 2018-03-26 11:33:21 -0500 | [diff] [blame] | 936 | goto err2; |
Uma Krishnan | 9cc8429 | 2018-03-26 11:31:29 -0500 | [diff] [blame] | 937 | } |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 938 | |
| 939 | ctx = ocxlflash_dev_context_init(pdev, afu); |
| 940 | if (IS_ERR(ctx)) { |
| 941 | rc = PTR_ERR(ctx); |
| 942 | dev_err(dev, "%s: ocxlflash_dev_context_init failed rc=%d\n", |
| 943 | __func__, rc); |
Uma Krishnan | 7390482 | 2018-03-26 11:33:21 -0500 | [diff] [blame] | 944 | goto err3; |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 945 | } |
| 946 | |
| 947 | afu->ocxl_ctx = ctx; |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 948 | out: |
| 949 | return afu; |
Uma Krishnan | 7390482 | 2018-03-26 11:33:21 -0500 | [diff] [blame] | 950 | err3: |
Uma Krishnan | 5437050 | 2018-03-26 11:32:37 -0500 | [diff] [blame] | 951 | ocxlflash_unconfig_afu(afu); |
Uma Krishnan | 7390482 | 2018-03-26 11:33:21 -0500 | [diff] [blame] | 952 | err2: |
| 953 | ocxlflash_unconfig_fn(pdev, afu); |
Uma Krishnan | e9dfced | 2018-03-26 11:31:09 -0500 | [diff] [blame] | 954 | err1: |
Uma Krishnan | 429ebfa | 2018-03-26 11:32:00 -0500 | [diff] [blame] | 955 | idr_destroy(&afu->idr); |
Uma Krishnan | e9dfced | 2018-03-26 11:31:09 -0500 | [diff] [blame] | 956 | kfree(afu); |
| 957 | afu = NULL; |
| 958 | goto out; |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 959 | } |
Uma Krishnan | 76ebe01 | 2018-03-26 11:30:51 -0500 | [diff] [blame] | 960 | |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 961 | /** |
| 962 | * ctx_event_pending() - check for any event pending on the context |
| 963 | * @ctx: Context to be checked. |
| 964 | * |
| 965 | * Return: true if there is an event pending, false if none pending |
| 966 | */ |
| 967 | static inline bool ctx_event_pending(struct ocxlflash_context *ctx) |
| 968 | { |
Uma Krishnan | 66ae644 | 2018-03-26 11:35:07 -0500 | [diff] [blame] | 969 | if (ctx->pending_irq || ctx->pending_fault) |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 970 | return true; |
| 971 | |
| 972 | return false; |
| 973 | } |
| 974 | |
| 975 | /** |
| 976 | * afu_poll() - poll the AFU for events on the context |
| 977 | * @file: File associated with the adapter context. |
| 978 | * @poll: Poll structure from the user. |
| 979 | * |
| 980 | * Return: poll mask |
| 981 | */ |
| 982 | static unsigned int afu_poll(struct file *file, struct poll_table_struct *poll) |
| 983 | { |
| 984 | struct ocxlflash_context *ctx = file->private_data; |
| 985 | struct device *dev = ctx->hw_afu->dev; |
| 986 | ulong lock_flags; |
| 987 | int mask = 0; |
| 988 | |
| 989 | poll_wait(file, &ctx->wq, poll); |
| 990 | |
| 991 | spin_lock_irqsave(&ctx->slock, lock_flags); |
| 992 | if (ctx_event_pending(ctx)) |
| 993 | mask |= POLLIN | POLLRDNORM; |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 994 | else if (ctx->state == CLOSED) |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 995 | mask |= POLLERR; |
| 996 | spin_unlock_irqrestore(&ctx->slock, lock_flags); |
| 997 | |
| 998 | dev_dbg(dev, "%s: Poll wait completed for pe %i mask %i\n", |
| 999 | __func__, ctx->pe, mask); |
| 1000 | |
| 1001 | return mask; |
| 1002 | } |
| 1003 | |
Uma Krishnan | 03aa9c5 | 2018-03-26 11:34:12 -0500 | [diff] [blame] | 1004 | /** |
| 1005 | * afu_read() - perform a read on the context for any event |
| 1006 | * @file: File associated with the adapter context. |
| 1007 | * @buf: Buffer to receive the data. |
| 1008 | * @count: Size of buffer (maximum bytes that can be read). |
| 1009 | * @off: Offset. |
| 1010 | * |
| 1011 | * Return: size of the data read on success, -errno on failure |
| 1012 | */ |
| 1013 | static ssize_t afu_read(struct file *file, char __user *buf, size_t count, |
| 1014 | loff_t *off) |
| 1015 | { |
| 1016 | struct ocxlflash_context *ctx = file->private_data; |
| 1017 | struct device *dev = ctx->hw_afu->dev; |
| 1018 | struct cxl_event event; |
| 1019 | ulong lock_flags; |
| 1020 | ssize_t esize; |
| 1021 | ssize_t rc; |
| 1022 | int bit; |
| 1023 | DEFINE_WAIT(event_wait); |
| 1024 | |
| 1025 | if (*off != 0) { |
| 1026 | dev_err(dev, "%s: Non-zero offset not supported, off=%lld\n", |
| 1027 | __func__, *off); |
| 1028 | rc = -EINVAL; |
| 1029 | goto out; |
| 1030 | } |
| 1031 | |
| 1032 | spin_lock_irqsave(&ctx->slock, lock_flags); |
| 1033 | |
| 1034 | for (;;) { |
| 1035 | prepare_to_wait(&ctx->wq, &event_wait, TASK_INTERRUPTIBLE); |
| 1036 | |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 1037 | if (ctx_event_pending(ctx) || (ctx->state == CLOSED)) |
Uma Krishnan | 03aa9c5 | 2018-03-26 11:34:12 -0500 | [diff] [blame] | 1038 | break; |
| 1039 | |
| 1040 | if (file->f_flags & O_NONBLOCK) { |
| 1041 | dev_err(dev, "%s: File cannot be blocked on I/O\n", |
| 1042 | __func__); |
| 1043 | rc = -EAGAIN; |
| 1044 | goto err; |
| 1045 | } |
| 1046 | |
| 1047 | if (signal_pending(current)) { |
| 1048 | dev_err(dev, "%s: Signal pending on the process\n", |
| 1049 | __func__); |
| 1050 | rc = -ERESTARTSYS; |
| 1051 | goto err; |
| 1052 | } |
| 1053 | |
| 1054 | spin_unlock_irqrestore(&ctx->slock, lock_flags); |
| 1055 | schedule(); |
| 1056 | spin_lock_irqsave(&ctx->slock, lock_flags); |
| 1057 | } |
| 1058 | |
| 1059 | finish_wait(&ctx->wq, &event_wait); |
| 1060 | |
| 1061 | memset(&event, 0, sizeof(event)); |
| 1062 | event.header.process_element = ctx->pe; |
| 1063 | event.header.size = sizeof(struct cxl_event_header); |
| 1064 | if (ctx->pending_irq) { |
| 1065 | esize = sizeof(struct cxl_event_afu_interrupt); |
| 1066 | event.header.size += esize; |
| 1067 | event.header.type = CXL_EVENT_AFU_INTERRUPT; |
| 1068 | |
| 1069 | bit = find_first_bit(&ctx->irq_bitmap, ctx->num_irqs); |
| 1070 | clear_bit(bit, &ctx->irq_bitmap); |
| 1071 | event.irq.irq = bit + 1; |
| 1072 | if (bitmap_empty(&ctx->irq_bitmap, ctx->num_irqs)) |
| 1073 | ctx->pending_irq = false; |
Uma Krishnan | 66ae644 | 2018-03-26 11:35:07 -0500 | [diff] [blame] | 1074 | } else if (ctx->pending_fault) { |
| 1075 | event.header.size += sizeof(struct cxl_event_data_storage); |
| 1076 | event.header.type = CXL_EVENT_DATA_STORAGE; |
| 1077 | event.fault.addr = ctx->fault_addr; |
| 1078 | event.fault.dsisr = ctx->fault_dsisr; |
| 1079 | ctx->pending_fault = false; |
Uma Krishnan | 03aa9c5 | 2018-03-26 11:34:12 -0500 | [diff] [blame] | 1080 | } |
| 1081 | |
| 1082 | spin_unlock_irqrestore(&ctx->slock, lock_flags); |
| 1083 | |
| 1084 | if (copy_to_user(buf, &event, event.header.size)) { |
| 1085 | dev_err(dev, "%s: copy_to_user failed\n", __func__); |
| 1086 | rc = -EFAULT; |
| 1087 | goto out; |
| 1088 | } |
| 1089 | |
| 1090 | rc = event.header.size; |
| 1091 | out: |
| 1092 | return rc; |
| 1093 | err: |
| 1094 | finish_wait(&ctx->wq, &event_wait); |
| 1095 | spin_unlock_irqrestore(&ctx->slock, lock_flags); |
| 1096 | goto out; |
| 1097 | } |
| 1098 | |
Uma Krishnan | 93b8f8d | 2018-03-26 11:34:20 -0500 | [diff] [blame] | 1099 | /** |
| 1100 | * afu_release() - release and free the context |
| 1101 | * @inode: File inode pointer. |
| 1102 | * @file: File associated with the context. |
| 1103 | * |
| 1104 | * Return: 0 on success, -errno on failure |
| 1105 | */ |
| 1106 | static int afu_release(struct inode *inode, struct file *file) |
| 1107 | { |
| 1108 | struct ocxlflash_context *ctx = file->private_data; |
| 1109 | int i; |
| 1110 | |
| 1111 | /* Unmap and free the interrupts associated with the context */ |
| 1112 | for (i = ctx->num_irqs; i >= 0; i--) |
| 1113 | afu_unmap_irq(0, ctx, i, ctx); |
| 1114 | free_afu_irqs(ctx); |
| 1115 | |
| 1116 | return ocxlflash_release_context(ctx); |
| 1117 | } |
| 1118 | |
| 1119 | /** |
| 1120 | * ocxlflash_mmap_fault() - mmap fault handler |
| 1121 | * @vmf: VM fault associated with current fault. |
| 1122 | * |
| 1123 | * Return: 0 on success, -errno on failure |
| 1124 | */ |
Souptick Joarder | a38b80c | 2018-06-20 23:55:07 +0530 | [diff] [blame] | 1125 | static vm_fault_t ocxlflash_mmap_fault(struct vm_fault *vmf) |
Uma Krishnan | 93b8f8d | 2018-03-26 11:34:20 -0500 | [diff] [blame] | 1126 | { |
| 1127 | struct vm_area_struct *vma = vmf->vma; |
| 1128 | struct ocxlflash_context *ctx = vma->vm_file->private_data; |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 1129 | struct device *dev = ctx->hw_afu->dev; |
Uma Krishnan | 93b8f8d | 2018-03-26 11:34:20 -0500 | [diff] [blame] | 1130 | u64 mmio_area, offset; |
| 1131 | |
| 1132 | offset = vmf->pgoff << PAGE_SHIFT; |
| 1133 | if (offset >= ctx->psn_size) |
| 1134 | return VM_FAULT_SIGBUS; |
| 1135 | |
Uma Krishnan | f81face | 2018-03-26 11:35:00 -0500 | [diff] [blame] | 1136 | mutex_lock(&ctx->state_mutex); |
| 1137 | if (ctx->state != STARTED) { |
| 1138 | dev_err(dev, "%s: Context not started, state=%d\n", |
| 1139 | __func__, ctx->state); |
| 1140 | mutex_unlock(&ctx->state_mutex); |
| 1141 | return VM_FAULT_SIGBUS; |
| 1142 | } |
| 1143 | mutex_unlock(&ctx->state_mutex); |
| 1144 | |
Uma Krishnan | 93b8f8d | 2018-03-26 11:34:20 -0500 | [diff] [blame] | 1145 | mmio_area = ctx->psn_phys; |
| 1146 | mmio_area += offset; |
| 1147 | |
Souptick Joarder | a38b80c | 2018-06-20 23:55:07 +0530 | [diff] [blame] | 1148 | return vmf_insert_pfn(vma, vmf->address, mmio_area >> PAGE_SHIFT); |
Uma Krishnan | 93b8f8d | 2018-03-26 11:34:20 -0500 | [diff] [blame] | 1149 | } |
| 1150 | |
| 1151 | static const struct vm_operations_struct ocxlflash_vmops = { |
| 1152 | .fault = ocxlflash_mmap_fault, |
| 1153 | }; |
| 1154 | |
| 1155 | /** |
| 1156 | * afu_mmap() - map the fault handler operations |
| 1157 | * @file: File associated with the context. |
| 1158 | * @vma: VM area associated with mapping. |
| 1159 | * |
| 1160 | * Return: 0 on success, -errno on failure |
| 1161 | */ |
| 1162 | static int afu_mmap(struct file *file, struct vm_area_struct *vma) |
| 1163 | { |
| 1164 | struct ocxlflash_context *ctx = file->private_data; |
| 1165 | |
| 1166 | if ((vma_pages(vma) + vma->vm_pgoff) > |
| 1167 | (ctx->psn_size >> PAGE_SHIFT)) |
| 1168 | return -EINVAL; |
| 1169 | |
Suren Baghdasaryan | 1c71222 | 2023-01-26 11:37:49 -0800 | [diff] [blame] | 1170 | vm_flags_set(vma, VM_IO | VM_PFNMAP); |
Uma Krishnan | 93b8f8d | 2018-03-26 11:34:20 -0500 | [diff] [blame] | 1171 | vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); |
| 1172 | vma->vm_ops = &ocxlflash_vmops; |
| 1173 | return 0; |
| 1174 | } |
| 1175 | |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 1176 | static const struct file_operations ocxl_afu_fops = { |
| 1177 | .owner = THIS_MODULE, |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 1178 | .poll = afu_poll, |
Uma Krishnan | 03aa9c5 | 2018-03-26 11:34:12 -0500 | [diff] [blame] | 1179 | .read = afu_read, |
Uma Krishnan | 93b8f8d | 2018-03-26 11:34:20 -0500 | [diff] [blame] | 1180 | .release = afu_release, |
| 1181 | .mmap = afu_mmap, |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 1182 | }; |
| 1183 | |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 1184 | #define PATCH_FOPS(NAME) \ |
| 1185 | do { if (!fops->NAME) fops->NAME = ocxl_afu_fops.NAME; } while (0) |
| 1186 | |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 1187 | /** |
| 1188 | * ocxlflash_get_fd() - get file descriptor for an adapter context |
| 1189 | * @ctx_cookie: Adapter context. |
| 1190 | * @fops: File operations to be associated. |
| 1191 | * @fd: File descriptor to be returned back. |
| 1192 | * |
| 1193 | * Return: pointer to the file on success, ERR_PTR on failure |
| 1194 | */ |
| 1195 | static struct file *ocxlflash_get_fd(void *ctx_cookie, |
| 1196 | struct file_operations *fops, int *fd) |
| 1197 | { |
| 1198 | struct ocxlflash_context *ctx = ctx_cookie; |
| 1199 | struct device *dev = ctx->hw_afu->dev; |
| 1200 | struct file *file; |
| 1201 | int flags, fdtmp; |
| 1202 | int rc = 0; |
| 1203 | char *name = NULL; |
| 1204 | |
| 1205 | /* Only allow one fd per context */ |
| 1206 | if (ctx->mapping) { |
| 1207 | dev_err(dev, "%s: Context is already mapped to an fd\n", |
| 1208 | __func__); |
| 1209 | rc = -EEXIST; |
| 1210 | goto err1; |
| 1211 | } |
| 1212 | |
| 1213 | flags = O_RDWR | O_CLOEXEC; |
| 1214 | |
| 1215 | /* This code is similar to anon_inode_getfd() */ |
| 1216 | rc = get_unused_fd_flags(flags); |
| 1217 | if (unlikely(rc < 0)) { |
| 1218 | dev_err(dev, "%s: get_unused_fd_flags failed rc=%d\n", |
| 1219 | __func__, rc); |
| 1220 | goto err1; |
| 1221 | } |
| 1222 | fdtmp = rc; |
| 1223 | |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 1224 | /* Patch the file ops that are not defined */ |
| 1225 | if (fops) { |
| 1226 | PATCH_FOPS(poll); |
Uma Krishnan | 03aa9c5 | 2018-03-26 11:34:12 -0500 | [diff] [blame] | 1227 | PATCH_FOPS(read); |
Uma Krishnan | 93b8f8d | 2018-03-26 11:34:20 -0500 | [diff] [blame] | 1228 | PATCH_FOPS(release); |
| 1229 | PATCH_FOPS(mmap); |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 1230 | } else /* Use default ops */ |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 1231 | fops = (struct file_operations *)&ocxl_afu_fops; |
| 1232 | |
| 1233 | name = kasprintf(GFP_KERNEL, "ocxlflash:%d", ctx->pe); |
| 1234 | file = ocxlflash_getfile(dev, name, fops, ctx, flags); |
| 1235 | kfree(name); |
| 1236 | if (IS_ERR(file)) { |
| 1237 | rc = PTR_ERR(file); |
| 1238 | dev_err(dev, "%s: ocxlflash_getfile failed rc=%d\n", |
| 1239 | __func__, rc); |
| 1240 | goto err2; |
| 1241 | } |
| 1242 | |
| 1243 | ctx->mapping = file->f_mapping; |
| 1244 | *fd = fdtmp; |
| 1245 | out: |
| 1246 | return file; |
| 1247 | err2: |
| 1248 | put_unused_fd(fdtmp); |
| 1249 | err1: |
| 1250 | file = ERR_PTR(rc); |
| 1251 | goto out; |
| 1252 | } |
| 1253 | |
Uma Krishnan | b18718c | 2018-03-26 11:32:20 -0500 | [diff] [blame] | 1254 | /** |
| 1255 | * ocxlflash_fops_get_context() - get the context associated with the file |
| 1256 | * @file: File associated with the adapter context. |
| 1257 | * |
| 1258 | * Return: pointer to the context |
| 1259 | */ |
| 1260 | static void *ocxlflash_fops_get_context(struct file *file) |
| 1261 | { |
| 1262 | return file->private_data; |
| 1263 | } |
| 1264 | |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 1265 | /** |
| 1266 | * ocxlflash_afu_irq() - interrupt handler for user contexts |
| 1267 | * @irq: Interrupt number. |
| 1268 | * @data: Private data provided at interrupt registration, the context. |
| 1269 | * |
| 1270 | * Return: Always return IRQ_HANDLED. |
| 1271 | */ |
| 1272 | static irqreturn_t ocxlflash_afu_irq(int irq, void *data) |
| 1273 | { |
| 1274 | struct ocxlflash_context *ctx = data; |
| 1275 | struct device *dev = ctx->hw_afu->dev; |
| 1276 | int i; |
| 1277 | |
| 1278 | dev_dbg(dev, "%s: Interrupt raised for pe %i virq %i\n", |
| 1279 | __func__, ctx->pe, irq); |
| 1280 | |
| 1281 | for (i = 0; i < ctx->num_irqs; i++) { |
| 1282 | if (ctx->irqs[i].virq == irq) |
| 1283 | break; |
| 1284 | } |
| 1285 | if (unlikely(i >= ctx->num_irqs)) { |
| 1286 | dev_err(dev, "%s: Received AFU IRQ out of range\n", __func__); |
| 1287 | goto out; |
| 1288 | } |
| 1289 | |
| 1290 | spin_lock(&ctx->slock); |
| 1291 | set_bit(i - 1, &ctx->irq_bitmap); |
| 1292 | ctx->pending_irq = true; |
| 1293 | spin_unlock(&ctx->slock); |
Uma Krishnan | 56f1db1 | 2018-03-26 11:34:03 -0500 | [diff] [blame] | 1294 | |
| 1295 | wake_up_all(&ctx->wq); |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 1296 | out: |
| 1297 | return IRQ_HANDLED; |
| 1298 | } |
| 1299 | |
| 1300 | /** |
| 1301 | * ocxlflash_start_work() - start a user context |
| 1302 | * @ctx_cookie: Context to be started. |
| 1303 | * @num_irqs: Number of interrupts requested. |
| 1304 | * |
| 1305 | * Return: 0 on success, -errno on failure |
| 1306 | */ |
| 1307 | static int ocxlflash_start_work(void *ctx_cookie, u64 num_irqs) |
| 1308 | { |
| 1309 | struct ocxlflash_context *ctx = ctx_cookie; |
| 1310 | struct ocxl_hw_afu *afu = ctx->hw_afu; |
| 1311 | struct device *dev = afu->dev; |
| 1312 | char *name; |
| 1313 | int rc = 0; |
| 1314 | int i; |
| 1315 | |
| 1316 | rc = alloc_afu_irqs(ctx, num_irqs); |
| 1317 | if (unlikely(rc < 0)) { |
| 1318 | dev_err(dev, "%s: alloc_afu_irqs failed rc=%d\n", __func__, rc); |
| 1319 | goto out; |
| 1320 | } |
| 1321 | |
| 1322 | for (i = 0; i < num_irqs; i++) { |
| 1323 | name = kasprintf(GFP_KERNEL, "ocxlflash-%s-pe%i-%i", |
| 1324 | dev_name(dev), ctx->pe, i); |
| 1325 | rc = afu_map_irq(0, ctx, i, ocxlflash_afu_irq, ctx, name); |
| 1326 | kfree(name); |
| 1327 | if (unlikely(rc < 0)) { |
| 1328 | dev_err(dev, "%s: afu_map_irq failed rc=%d\n", |
| 1329 | __func__, rc); |
| 1330 | goto err; |
| 1331 | } |
| 1332 | } |
| 1333 | |
| 1334 | rc = start_context(ctx); |
| 1335 | if (unlikely(rc)) { |
| 1336 | dev_err(dev, "%s: start_context failed rc=%d\n", __func__, rc); |
| 1337 | goto err; |
| 1338 | } |
| 1339 | out: |
| 1340 | return rc; |
| 1341 | err: |
| 1342 | for (i = i-1; i >= 0; i--) |
| 1343 | afu_unmap_irq(0, ctx, i, ctx); |
| 1344 | free_afu_irqs(ctx); |
| 1345 | goto out; |
Uma Krishnan | e117c3c | 2018-03-26 11:34:27 -0500 | [diff] [blame] | 1346 | }; |
| 1347 | |
| 1348 | /** |
| 1349 | * ocxlflash_fd_mmap() - mmap handler for adapter file descriptor |
| 1350 | * @file: File installed with adapter file descriptor. |
| 1351 | * @vma: VM area associated with mapping. |
| 1352 | * |
| 1353 | * Return: 0 on success, -errno on failure |
| 1354 | */ |
| 1355 | static int ocxlflash_fd_mmap(struct file *file, struct vm_area_struct *vma) |
| 1356 | { |
| 1357 | return afu_mmap(file, vma); |
| 1358 | } |
| 1359 | |
| 1360 | /** |
| 1361 | * ocxlflash_fd_release() - release the context associated with the file |
| 1362 | * @inode: File inode pointer. |
| 1363 | * @file: File associated with the adapter context. |
| 1364 | * |
| 1365 | * Return: 0 on success, -errno on failure |
| 1366 | */ |
| 1367 | static int ocxlflash_fd_release(struct inode *inode, struct file *file) |
| 1368 | { |
| 1369 | return afu_release(inode, file); |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 1370 | } |
| 1371 | |
Uma Krishnan | 76ebe01 | 2018-03-26 11:30:51 -0500 | [diff] [blame] | 1372 | /* Backend ops to ocxlflash services */ |
| 1373 | const struct cxlflash_backend_ops cxlflash_ocxl_ops = { |
| 1374 | .module = THIS_MODULE, |
Uma Krishnan | 012f394 | 2018-03-26 11:32:56 -0500 | [diff] [blame] | 1375 | .psa_map = ocxlflash_psa_map, |
| 1376 | .psa_unmap = ocxlflash_psa_unmap, |
Uma Krishnan | b18718c | 2018-03-26 11:32:20 -0500 | [diff] [blame] | 1377 | .process_element = ocxlflash_process_element, |
Uma Krishnan | a06b1cf | 2018-03-26 11:33:48 -0500 | [diff] [blame] | 1378 | .map_afu_irq = ocxlflash_map_afu_irq, |
| 1379 | .unmap_afu_irq = ocxlflash_unmap_afu_irq, |
Uma Krishnan | 402a55e | 2018-03-26 11:34:35 -0500 | [diff] [blame] | 1380 | .get_irq_objhndl = ocxlflash_get_irq_objhndl, |
Uma Krishnan | 6b938ac | 2018-03-26 11:32:48 -0500 | [diff] [blame] | 1381 | .start_context = ocxlflash_start_context, |
Uma Krishnan | c207b57 | 2018-03-26 11:33:35 -0500 | [diff] [blame] | 1382 | .stop_context = ocxlflash_stop_context, |
Uma Krishnan | 9433fb3 | 2018-03-26 11:35:15 -0500 | [diff] [blame] | 1383 | .afu_reset = ocxlflash_afu_reset, |
Uma Krishnan | f6b4557c | 2018-03-26 11:31:53 -0500 | [diff] [blame] | 1384 | .set_master = ocxlflash_set_master, |
| 1385 | .get_context = ocxlflash_get_context, |
| 1386 | .dev_context_init = ocxlflash_dev_context_init, |
| 1387 | .release_context = ocxlflash_release_context, |
Uma Krishnan | 8b7a5521 | 2018-03-26 11:32:29 -0500 | [diff] [blame] | 1388 | .perst_reloads_same_image = ocxlflash_perst_reloads_same_image, |
Uma Krishnan | 119c920 | 2018-03-26 11:33:14 -0500 | [diff] [blame] | 1389 | .read_adapter_vpd = ocxlflash_read_adapter_vpd, |
Uma Krishnan | bc65c1c | 2018-03-26 11:33:41 -0500 | [diff] [blame] | 1390 | .allocate_afu_irqs = ocxlflash_allocate_afu_irqs, |
| 1391 | .free_afu_irqs = ocxlflash_free_afu_irqs, |
Uma Krishnan | 48e077d | 2018-03-26 11:31:01 -0500 | [diff] [blame] | 1392 | .create_afu = ocxlflash_create_afu, |
| 1393 | .destroy_afu = ocxlflash_destroy_afu, |
Uma Krishnan | 926a62f | 2018-03-26 11:32:09 -0500 | [diff] [blame] | 1394 | .get_fd = ocxlflash_get_fd, |
Uma Krishnan | b18718c | 2018-03-26 11:32:20 -0500 | [diff] [blame] | 1395 | .fops_get_context = ocxlflash_fops_get_context, |
Uma Krishnan | 762c7e9 | 2018-03-26 11:33:55 -0500 | [diff] [blame] | 1396 | .start_work = ocxlflash_start_work, |
Uma Krishnan | e117c3c | 2018-03-26 11:34:27 -0500 | [diff] [blame] | 1397 | .fd_mmap = ocxlflash_fd_mmap, |
| 1398 | .fd_release = ocxlflash_fd_release, |
Uma Krishnan | 76ebe01 | 2018-03-26 11:30:51 -0500 | [diff] [blame] | 1399 | }; |