Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is provided under a dual BSD/GPLv2 license. When using or |
| 3 | * redistributing this file, you may do so under either license. |
| 4 | * |
| 5 | * GPL LICENSE SUMMARY |
| 6 | * |
| 7 | * Copyright(c) 2012 Intel Corporation. All rights reserved. |
| 8 | * |
| 9 | * This program is free software; you can redistribute it and/or modify |
| 10 | * it under the terms of version 2 of the GNU General Public License as |
| 11 | * published by the Free Software Foundation. |
| 12 | * |
| 13 | * BSD LICENSE |
| 14 | * |
| 15 | * Copyright(c) 2012 Intel Corporation. All rights reserved. |
| 16 | * |
| 17 | * Redistribution and use in source and binary forms, with or without |
| 18 | * modification, are permitted provided that the following conditions |
| 19 | * are met: |
| 20 | * |
| 21 | * * Redistributions of source code must retain the above copyright |
| 22 | * notice, this list of conditions and the following disclaimer. |
| 23 | * * Redistributions in binary form must reproduce the above copy |
| 24 | * notice, this list of conditions and the following disclaimer in |
| 25 | * the documentation and/or other materials provided with the |
| 26 | * distribution. |
| 27 | * * Neither the name of Intel Corporation nor the names of its |
| 28 | * contributors may be used to endorse or promote products derived |
| 29 | * from this software without specific prior written permission. |
| 30 | * |
| 31 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 32 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 33 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 34 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 35 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 36 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 37 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 38 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 39 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 40 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 41 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 42 | * |
| 43 | * Intel PCIe NTB Linux driver |
| 44 | * |
| 45 | * Contact Information: |
| 46 | * Jon Mason <jon.mason@intel.com> |
| 47 | */ |
| 48 | #include <linux/debugfs.h> |
| 49 | #include <linux/delay.h> |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 50 | #include <linux/dmaengine.h> |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 51 | #include <linux/dma-mapping.h> |
| 52 | #include <linux/errno.h> |
| 53 | #include <linux/export.h> |
| 54 | #include <linux/interrupt.h> |
| 55 | #include <linux/module.h> |
| 56 | #include <linux/pci.h> |
| 57 | #include <linux/slab.h> |
| 58 | #include <linux/types.h> |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 59 | #include "ntb_hw.h" |
| 60 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 61 | #define NTB_TRANSPORT_VERSION 3 |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 62 | |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 63 | static unsigned int transport_mtu = 0x401E; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 64 | module_param(transport_mtu, uint, 0644); |
| 65 | MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets"); |
| 66 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 67 | static unsigned char max_num_clients; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 68 | module_param(max_num_clients, byte, 0644); |
| 69 | MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients"); |
| 70 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 71 | static unsigned int copy_bytes = 1024; |
| 72 | module_param(copy_bytes, uint, 0644); |
| 73 | MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); |
| 74 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 75 | struct ntb_queue_entry { |
| 76 | /* ntb_queue list reference */ |
| 77 | struct list_head entry; |
| 78 | /* pointers to data to be transfered */ |
| 79 | void *cb_data; |
| 80 | void *buf; |
| 81 | unsigned int len; |
| 82 | unsigned int flags; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 83 | |
| 84 | struct ntb_transport_qp *qp; |
| 85 | union { |
| 86 | struct ntb_payload_header __iomem *tx_hdr; |
| 87 | struct ntb_payload_header *rx_hdr; |
| 88 | }; |
| 89 | unsigned int index; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 90 | }; |
| 91 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 92 | struct ntb_rx_info { |
| 93 | unsigned int entry; |
| 94 | }; |
| 95 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 96 | struct ntb_transport_qp { |
| 97 | struct ntb_transport *transport; |
| 98 | struct ntb_device *ndev; |
| 99 | void *cb_data; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 100 | struct dma_chan *dma_chan; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 101 | |
| 102 | bool client_ready; |
| 103 | bool qp_link; |
| 104 | u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ |
| 105 | |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 106 | struct ntb_rx_info __iomem *rx_info; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 107 | struct ntb_rx_info *remote_rx_info; |
| 108 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 109 | void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, |
| 110 | void *data, int len); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 111 | struct list_head tx_free_q; |
| 112 | spinlock_t ntb_tx_free_q_lock; |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 113 | void __iomem *tx_mw; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 114 | dma_addr_t tx_mw_phys; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 115 | unsigned int tx_index; |
| 116 | unsigned int tx_max_entry; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 117 | unsigned int tx_max_frame; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 118 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 119 | void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, |
| 120 | void *data, int len); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 121 | struct list_head rx_pend_q; |
| 122 | struct list_head rx_free_q; |
| 123 | spinlock_t ntb_rx_pend_q_lock; |
| 124 | spinlock_t ntb_rx_free_q_lock; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 125 | void *rx_buff; |
| 126 | unsigned int rx_index; |
| 127 | unsigned int rx_max_entry; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 128 | unsigned int rx_max_frame; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 129 | dma_cookie_t last_cookie; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 130 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 131 | void (*event_handler)(void *data, int status); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 132 | struct delayed_work link_work; |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 133 | struct work_struct link_cleanup; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 134 | |
| 135 | struct dentry *debugfs_dir; |
| 136 | struct dentry *debugfs_stats; |
| 137 | |
| 138 | /* Stats */ |
| 139 | u64 rx_bytes; |
| 140 | u64 rx_pkts; |
| 141 | u64 rx_ring_empty; |
| 142 | u64 rx_err_no_buf; |
| 143 | u64 rx_err_oflow; |
| 144 | u64 rx_err_ver; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 145 | u64 rx_memcpy; |
| 146 | u64 rx_async; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 147 | u64 tx_bytes; |
| 148 | u64 tx_pkts; |
| 149 | u64 tx_ring_full; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 150 | u64 tx_err_no_buf; |
| 151 | u64 tx_memcpy; |
| 152 | u64 tx_async; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 153 | }; |
| 154 | |
| 155 | struct ntb_transport_mw { |
| 156 | size_t size; |
| 157 | void *virt_addr; |
| 158 | dma_addr_t dma_addr; |
| 159 | }; |
| 160 | |
| 161 | struct ntb_transport_client_dev { |
| 162 | struct list_head entry; |
| 163 | struct device dev; |
| 164 | }; |
| 165 | |
| 166 | struct ntb_transport { |
| 167 | struct list_head entry; |
| 168 | struct list_head client_devs; |
| 169 | |
| 170 | struct ntb_device *ndev; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 171 | struct ntb_transport_mw *mw; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 172 | struct ntb_transport_qp *qps; |
| 173 | unsigned int max_qps; |
| 174 | unsigned long qp_bitmap; |
| 175 | bool transport_link; |
| 176 | struct delayed_work link_work; |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 177 | struct work_struct link_cleanup; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 178 | }; |
| 179 | |
| 180 | enum { |
| 181 | DESC_DONE_FLAG = 1 << 0, |
| 182 | LINK_DOWN_FLAG = 1 << 1, |
| 183 | }; |
| 184 | |
| 185 | struct ntb_payload_header { |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 186 | unsigned int ver; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 187 | unsigned int len; |
| 188 | unsigned int flags; |
| 189 | }; |
| 190 | |
| 191 | enum { |
| 192 | VERSION = 0, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 193 | QP_LINKS, |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 194 | NUM_QPS, |
| 195 | NUM_MWS, |
| 196 | MW0_SZ_HIGH, |
| 197 | MW0_SZ_LOW, |
| 198 | MW1_SZ_HIGH, |
| 199 | MW1_SZ_LOW, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 200 | MAX_SPAD, |
| 201 | }; |
| 202 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 203 | #define QP_TO_MW(ndev, qp) ((qp) % ntb_max_mw(ndev)) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 204 | #define NTB_QP_DEF_NUM_ENTRIES 100 |
| 205 | #define NTB_LINK_DOWN_TIMEOUT 10 |
| 206 | |
| 207 | static int ntb_match_bus(struct device *dev, struct device_driver *drv) |
| 208 | { |
| 209 | return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); |
| 210 | } |
| 211 | |
| 212 | static int ntb_client_probe(struct device *dev) |
| 213 | { |
| 214 | const struct ntb_client *drv = container_of(dev->driver, |
| 215 | struct ntb_client, driver); |
| 216 | struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev); |
| 217 | int rc = -EINVAL; |
| 218 | |
| 219 | get_device(dev); |
| 220 | if (drv && drv->probe) |
| 221 | rc = drv->probe(pdev); |
| 222 | if (rc) |
| 223 | put_device(dev); |
| 224 | |
| 225 | return rc; |
| 226 | } |
| 227 | |
| 228 | static int ntb_client_remove(struct device *dev) |
| 229 | { |
| 230 | const struct ntb_client *drv = container_of(dev->driver, |
| 231 | struct ntb_client, driver); |
| 232 | struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev); |
| 233 | |
| 234 | if (drv && drv->remove) |
| 235 | drv->remove(pdev); |
| 236 | |
| 237 | put_device(dev); |
| 238 | |
| 239 | return 0; |
| 240 | } |
| 241 | |
Jon Mason | 170d35a | 2013-01-19 02:02:23 -0700 | [diff] [blame] | 242 | static struct bus_type ntb_bus_type = { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 243 | .name = "ntb_bus", |
| 244 | .match = ntb_match_bus, |
| 245 | .probe = ntb_client_probe, |
| 246 | .remove = ntb_client_remove, |
| 247 | }; |
| 248 | |
| 249 | static LIST_HEAD(ntb_transport_list); |
| 250 | |
Greg Kroah-Hartman | 78a61ab | 2013-01-17 19:17:42 -0800 | [diff] [blame] | 251 | static int ntb_bus_init(struct ntb_transport *nt) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 252 | { |
| 253 | if (list_empty(&ntb_transport_list)) { |
| 254 | int rc = bus_register(&ntb_bus_type); |
| 255 | if (rc) |
| 256 | return rc; |
| 257 | } |
| 258 | |
| 259 | list_add(&nt->entry, &ntb_transport_list); |
| 260 | |
| 261 | return 0; |
| 262 | } |
| 263 | |
Greg Kroah-Hartman | 78a61ab | 2013-01-17 19:17:42 -0800 | [diff] [blame] | 264 | static void ntb_bus_remove(struct ntb_transport *nt) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 265 | { |
| 266 | struct ntb_transport_client_dev *client_dev, *cd; |
| 267 | |
| 268 | list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) { |
| 269 | dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n", |
| 270 | dev_name(&client_dev->dev)); |
| 271 | list_del(&client_dev->entry); |
| 272 | device_unregister(&client_dev->dev); |
| 273 | } |
| 274 | |
| 275 | list_del(&nt->entry); |
| 276 | |
| 277 | if (list_empty(&ntb_transport_list)) |
| 278 | bus_unregister(&ntb_bus_type); |
| 279 | } |
| 280 | |
| 281 | static void ntb_client_release(struct device *dev) |
| 282 | { |
| 283 | struct ntb_transport_client_dev *client_dev; |
| 284 | client_dev = container_of(dev, struct ntb_transport_client_dev, dev); |
| 285 | |
| 286 | kfree(client_dev); |
| 287 | } |
| 288 | |
| 289 | /** |
| 290 | * ntb_unregister_client_dev - Unregister NTB client device |
| 291 | * @device_name: Name of NTB client device |
| 292 | * |
| 293 | * Unregister an NTB client device with the NTB transport layer |
| 294 | */ |
| 295 | void ntb_unregister_client_dev(char *device_name) |
| 296 | { |
| 297 | struct ntb_transport_client_dev *client, *cd; |
| 298 | struct ntb_transport *nt; |
| 299 | |
| 300 | list_for_each_entry(nt, &ntb_transport_list, entry) |
| 301 | list_for_each_entry_safe(client, cd, &nt->client_devs, entry) |
| 302 | if (!strncmp(dev_name(&client->dev), device_name, |
| 303 | strlen(device_name))) { |
| 304 | list_del(&client->entry); |
| 305 | device_unregister(&client->dev); |
| 306 | } |
| 307 | } |
| 308 | EXPORT_SYMBOL_GPL(ntb_unregister_client_dev); |
| 309 | |
| 310 | /** |
| 311 | * ntb_register_client_dev - Register NTB client device |
| 312 | * @device_name: Name of NTB client device |
| 313 | * |
| 314 | * Register an NTB client device with the NTB transport layer |
| 315 | */ |
| 316 | int ntb_register_client_dev(char *device_name) |
| 317 | { |
| 318 | struct ntb_transport_client_dev *client_dev; |
| 319 | struct ntb_transport *nt; |
Jon Mason | 8b19d45 | 2013-04-26 14:51:57 -0700 | [diff] [blame] | 320 | int rc, i = 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 321 | |
Jon Mason | 8222b40 | 2013-01-19 02:02:16 -0700 | [diff] [blame] | 322 | if (list_empty(&ntb_transport_list)) |
| 323 | return -ENODEV; |
| 324 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 325 | list_for_each_entry(nt, &ntb_transport_list, entry) { |
| 326 | struct device *dev; |
| 327 | |
| 328 | client_dev = kzalloc(sizeof(struct ntb_transport_client_dev), |
| 329 | GFP_KERNEL); |
| 330 | if (!client_dev) { |
| 331 | rc = -ENOMEM; |
| 332 | goto err; |
| 333 | } |
| 334 | |
| 335 | dev = &client_dev->dev; |
| 336 | |
| 337 | /* setup and register client devices */ |
Jon Mason | 8b19d45 | 2013-04-26 14:51:57 -0700 | [diff] [blame] | 338 | dev_set_name(dev, "%s%d", device_name, i); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 339 | dev->bus = &ntb_bus_type; |
| 340 | dev->release = ntb_client_release; |
| 341 | dev->parent = &ntb_query_pdev(nt->ndev)->dev; |
| 342 | |
| 343 | rc = device_register(dev); |
| 344 | if (rc) { |
| 345 | kfree(client_dev); |
| 346 | goto err; |
| 347 | } |
| 348 | |
| 349 | list_add_tail(&client_dev->entry, &nt->client_devs); |
Jon Mason | 8b19d45 | 2013-04-26 14:51:57 -0700 | [diff] [blame] | 350 | i++; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 351 | } |
| 352 | |
| 353 | return 0; |
| 354 | |
| 355 | err: |
| 356 | ntb_unregister_client_dev(device_name); |
| 357 | |
| 358 | return rc; |
| 359 | } |
| 360 | EXPORT_SYMBOL_GPL(ntb_register_client_dev); |
| 361 | |
| 362 | /** |
| 363 | * ntb_register_client - Register NTB client driver |
| 364 | * @drv: NTB client driver to be registered |
| 365 | * |
| 366 | * Register an NTB client driver with the NTB transport layer |
| 367 | * |
| 368 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 369 | */ |
| 370 | int ntb_register_client(struct ntb_client *drv) |
| 371 | { |
| 372 | drv->driver.bus = &ntb_bus_type; |
| 373 | |
Jon Mason | 8222b40 | 2013-01-19 02:02:16 -0700 | [diff] [blame] | 374 | if (list_empty(&ntb_transport_list)) |
| 375 | return -ENODEV; |
| 376 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 377 | return driver_register(&drv->driver); |
| 378 | } |
| 379 | EXPORT_SYMBOL_GPL(ntb_register_client); |
| 380 | |
| 381 | /** |
| 382 | * ntb_unregister_client - Unregister NTB client driver |
| 383 | * @drv: NTB client driver to be unregistered |
| 384 | * |
| 385 | * Unregister an NTB client driver with the NTB transport layer |
| 386 | * |
| 387 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 388 | */ |
| 389 | void ntb_unregister_client(struct ntb_client *drv) |
| 390 | { |
| 391 | driver_unregister(&drv->driver); |
| 392 | } |
| 393 | EXPORT_SYMBOL_GPL(ntb_unregister_client); |
| 394 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 395 | static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count, |
| 396 | loff_t *offp) |
| 397 | { |
| 398 | struct ntb_transport_qp *qp; |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 399 | char *buf; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 400 | ssize_t ret, out_offset, out_count; |
| 401 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 402 | out_count = 1000; |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 403 | |
| 404 | buf = kmalloc(out_count, GFP_KERNEL); |
| 405 | if (!buf) |
| 406 | return -ENOMEM; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 407 | |
| 408 | qp = filp->private_data; |
| 409 | out_offset = 0; |
| 410 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 411 | "NTB QP stats\n"); |
| 412 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 413 | "rx_bytes - \t%llu\n", qp->rx_bytes); |
| 414 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 415 | "rx_pkts - \t%llu\n", qp->rx_pkts); |
| 416 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 417 | "rx_memcpy - \t%llu\n", qp->rx_memcpy); |
| 418 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 419 | "rx_async - \t%llu\n", qp->rx_async); |
| 420 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 421 | "rx_ring_empty - %llu\n", qp->rx_ring_empty); |
| 422 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 423 | "rx_err_no_buf - %llu\n", qp->rx_err_no_buf); |
| 424 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 425 | "rx_err_oflow - \t%llu\n", qp->rx_err_oflow); |
| 426 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 427 | "rx_err_ver - \t%llu\n", qp->rx_err_ver); |
| 428 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 429 | "rx_buff - \t%p\n", qp->rx_buff); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 430 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 431 | "rx_index - \t%u\n", qp->rx_index); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 432 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 433 | "rx_max_entry - \t%u\n", qp->rx_max_entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 434 | |
| 435 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 436 | "tx_bytes - \t%llu\n", qp->tx_bytes); |
| 437 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 438 | "tx_pkts - \t%llu\n", qp->tx_pkts); |
| 439 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 440 | "tx_memcpy - \t%llu\n", qp->tx_memcpy); |
| 441 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
| 442 | "tx_async - \t%llu\n", qp->tx_async); |
| 443 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 444 | "tx_ring_full - \t%llu\n", qp->tx_ring_full); |
| 445 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 446 | "tx_err_no_buf - %llu\n", qp->tx_err_no_buf); |
| 447 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 448 | "tx_mw - \t%p\n", qp->tx_mw); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 449 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 450 | "tx_index - \t%u\n", qp->tx_index); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 451 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 452 | "tx_max_entry - \t%u\n", qp->tx_max_entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 453 | |
| 454 | out_offset += snprintf(buf + out_offset, out_count - out_offset, |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 455 | "\nQP Link %s\n", (qp->qp_link == NTB_LINK_UP) ? |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 456 | "Up" : "Down"); |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 457 | if (out_offset > out_count) |
| 458 | out_offset = out_count; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 459 | |
| 460 | ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); |
Jon Mason | d7237e2 | 2013-01-19 02:02:25 -0700 | [diff] [blame] | 461 | kfree(buf); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 462 | return ret; |
| 463 | } |
| 464 | |
| 465 | static const struct file_operations ntb_qp_debugfs_stats = { |
| 466 | .owner = THIS_MODULE, |
Jon Mason | d66d7ac | 2013-01-19 02:02:20 -0700 | [diff] [blame] | 467 | .open = simple_open, |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 468 | .read = debugfs_read, |
| 469 | }; |
| 470 | |
| 471 | static void ntb_list_add(spinlock_t *lock, struct list_head *entry, |
| 472 | struct list_head *list) |
| 473 | { |
| 474 | unsigned long flags; |
| 475 | |
| 476 | spin_lock_irqsave(lock, flags); |
| 477 | list_add_tail(entry, list); |
| 478 | spin_unlock_irqrestore(lock, flags); |
| 479 | } |
| 480 | |
| 481 | static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock, |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 482 | struct list_head *list) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 483 | { |
| 484 | struct ntb_queue_entry *entry; |
| 485 | unsigned long flags; |
| 486 | |
| 487 | spin_lock_irqsave(lock, flags); |
| 488 | if (list_empty(list)) { |
| 489 | entry = NULL; |
| 490 | goto out; |
| 491 | } |
| 492 | entry = list_first_entry(list, struct ntb_queue_entry, entry); |
| 493 | list_del(&entry->entry); |
| 494 | out: |
| 495 | spin_unlock_irqrestore(lock, flags); |
| 496 | |
| 497 | return entry; |
| 498 | } |
| 499 | |
| 500 | static void ntb_transport_setup_qp_mw(struct ntb_transport *nt, |
| 501 | unsigned int qp_num) |
| 502 | { |
| 503 | struct ntb_transport_qp *qp = &nt->qps[qp_num]; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 504 | unsigned int rx_size, num_qps_mw; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 505 | u8 mw_num, mw_max; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 506 | unsigned int i; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 507 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 508 | mw_max = ntb_max_mw(nt->ndev); |
| 509 | mw_num = QP_TO_MW(nt->ndev, qp_num); |
| 510 | |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 511 | WARN_ON(nt->mw[mw_num].virt_addr == NULL); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 512 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 513 | if (nt->max_qps % mw_max && mw_num < nt->max_qps % mw_max) |
| 514 | num_qps_mw = nt->max_qps / mw_max + 1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 515 | else |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 516 | num_qps_mw = nt->max_qps / mw_max; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 517 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 518 | rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 519 | qp->rx_buff = nt->mw[mw_num].virt_addr + qp_num / mw_max * rx_size; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 520 | rx_size -= sizeof(struct ntb_rx_info); |
| 521 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 522 | qp->remote_rx_info = qp->rx_buff + rx_size; |
| 523 | |
Jon Mason | c9d534c | 2013-02-01 15:45:16 -0700 | [diff] [blame] | 524 | /* Due to housekeeping, there must be atleast 2 buffs */ |
| 525 | qp->rx_max_frame = min(transport_mtu, rx_size / 2); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 526 | qp->rx_max_entry = rx_size / qp->rx_max_frame; |
| 527 | qp->rx_index = 0; |
| 528 | |
Jon Mason | c9d534c | 2013-02-01 15:45:16 -0700 | [diff] [blame] | 529 | qp->remote_rx_info->entry = qp->rx_max_entry - 1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 530 | |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 531 | /* setup the hdr offsets with 0's */ |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 532 | for (i = 0; i < qp->rx_max_entry; i++) { |
| 533 | void *offset = qp->rx_buff + qp->rx_max_frame * (i + 1) - |
| 534 | sizeof(struct ntb_payload_header); |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 535 | memset(offset, 0, sizeof(struct ntb_payload_header)); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 536 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 537 | |
| 538 | qp->rx_pkts = 0; |
| 539 | qp->tx_pkts = 0; |
Jon Mason | 90f9e93 | 2013-02-01 15:34:35 -0700 | [diff] [blame] | 540 | qp->tx_index = 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 541 | } |
| 542 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 543 | static void ntb_free_mw(struct ntb_transport *nt, int num_mw) |
| 544 | { |
| 545 | struct ntb_transport_mw *mw = &nt->mw[num_mw]; |
| 546 | struct pci_dev *pdev = ntb_query_pdev(nt->ndev); |
| 547 | |
| 548 | if (!mw->virt_addr) |
| 549 | return; |
| 550 | |
| 551 | dma_free_coherent(&pdev->dev, mw->size, mw->virt_addr, mw->dma_addr); |
| 552 | mw->virt_addr = NULL; |
| 553 | } |
| 554 | |
Jon Mason | b77b263 | 2013-02-01 15:25:37 -0700 | [diff] [blame] | 555 | static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size) |
| 556 | { |
| 557 | struct ntb_transport_mw *mw = &nt->mw[num_mw]; |
| 558 | struct pci_dev *pdev = ntb_query_pdev(nt->ndev); |
| 559 | |
| 560 | /* No need to re-setup */ |
| 561 | if (mw->size == ALIGN(size, 4096)) |
| 562 | return 0; |
| 563 | |
| 564 | if (mw->size != 0) |
| 565 | ntb_free_mw(nt, num_mw); |
| 566 | |
| 567 | /* Alloc memory for receiving data. Must be 4k aligned */ |
| 568 | mw->size = ALIGN(size, 4096); |
| 569 | |
| 570 | mw->virt_addr = dma_alloc_coherent(&pdev->dev, mw->size, &mw->dma_addr, |
| 571 | GFP_KERNEL); |
| 572 | if (!mw->virt_addr) { |
| 573 | mw->size = 0; |
| 574 | dev_err(&pdev->dev, "Unable to allocate MW buffer of size %d\n", |
| 575 | (int) mw->size); |
| 576 | return -ENOMEM; |
| 577 | } |
| 578 | |
| 579 | /* Notify HW the memory location of the receive buffer */ |
| 580 | ntb_set_mw_addr(nt->ndev, num_mw, mw->dma_addr); |
| 581 | |
| 582 | return 0; |
| 583 | } |
| 584 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 585 | static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 586 | { |
| 587 | struct ntb_transport *nt = qp->transport; |
| 588 | struct pci_dev *pdev = ntb_query_pdev(nt->ndev); |
| 589 | |
| 590 | if (qp->qp_link == NTB_LINK_DOWN) { |
| 591 | cancel_delayed_work_sync(&qp->link_work); |
| 592 | return; |
| 593 | } |
| 594 | |
| 595 | if (qp->event_handler) |
| 596 | qp->event_handler(qp->cb_data, NTB_LINK_DOWN); |
| 597 | |
| 598 | dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); |
| 599 | qp->qp_link = NTB_LINK_DOWN; |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 600 | } |
| 601 | |
| 602 | static void ntb_qp_link_cleanup_work(struct work_struct *work) |
| 603 | { |
| 604 | struct ntb_transport_qp *qp = container_of(work, |
| 605 | struct ntb_transport_qp, |
| 606 | link_cleanup); |
| 607 | struct ntb_transport *nt = qp->transport; |
| 608 | |
| 609 | ntb_qp_link_cleanup(qp); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 610 | |
| 611 | if (nt->transport_link == NTB_LINK_UP) |
| 612 | schedule_delayed_work(&qp->link_work, |
| 613 | msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); |
| 614 | } |
| 615 | |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 616 | static void ntb_qp_link_down(struct ntb_transport_qp *qp) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 617 | { |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 618 | schedule_work(&qp->link_cleanup); |
| 619 | } |
| 620 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 621 | static void ntb_transport_link_cleanup(struct ntb_transport *nt) |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 622 | { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 623 | int i; |
| 624 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 625 | /* Pass along the info to any clients */ |
| 626 | for (i = 0; i < nt->max_qps; i++) |
| 627 | if (!test_bit(i, &nt->qp_bitmap)) |
| 628 | ntb_qp_link_cleanup(&nt->qps[i]); |
| 629 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 630 | if (nt->transport_link == NTB_LINK_DOWN) |
| 631 | cancel_delayed_work_sync(&nt->link_work); |
| 632 | else |
| 633 | nt->transport_link = NTB_LINK_DOWN; |
| 634 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 635 | /* The scratchpad registers keep the values if the remote side |
| 636 | * goes down, blast them now to give them a sane value the next |
| 637 | * time they are accessed |
| 638 | */ |
| 639 | for (i = 0; i < MAX_SPAD; i++) |
| 640 | ntb_write_local_spad(nt->ndev, i, 0); |
| 641 | } |
| 642 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 643 | static void ntb_transport_link_cleanup_work(struct work_struct *work) |
| 644 | { |
| 645 | struct ntb_transport *nt = container_of(work, struct ntb_transport, |
| 646 | link_cleanup); |
| 647 | |
| 648 | ntb_transport_link_cleanup(nt); |
| 649 | } |
| 650 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 651 | static void ntb_transport_event_callback(void *data, enum ntb_hw_event event) |
| 652 | { |
| 653 | struct ntb_transport *nt = data; |
| 654 | |
| 655 | switch (event) { |
| 656 | case NTB_EVENT_HW_LINK_UP: |
| 657 | schedule_delayed_work(&nt->link_work, 0); |
| 658 | break; |
| 659 | case NTB_EVENT_HW_LINK_DOWN: |
Jon Mason | 7b4f2d3 | 2013-01-19 02:02:19 -0700 | [diff] [blame] | 660 | schedule_work(&nt->link_cleanup); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 661 | break; |
| 662 | default: |
| 663 | BUG(); |
| 664 | } |
| 665 | } |
| 666 | |
| 667 | static void ntb_transport_link_work(struct work_struct *work) |
| 668 | { |
| 669 | struct ntb_transport *nt = container_of(work, struct ntb_transport, |
| 670 | link_work.work); |
| 671 | struct ntb_device *ndev = nt->ndev; |
| 672 | struct pci_dev *pdev = ntb_query_pdev(ndev); |
| 673 | u32 val; |
| 674 | int rc, i; |
| 675 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 676 | /* send the local info, in the opposite order of the way we read it */ |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 677 | for (i = 0; i < ntb_max_mw(ndev); i++) { |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 678 | rc = ntb_write_remote_spad(ndev, MW0_SZ_HIGH + (i * 2), |
| 679 | ntb_get_mw_size(ndev, i) >> 32); |
| 680 | if (rc) { |
| 681 | dev_err(&pdev->dev, "Error writing %u to remote spad %d\n", |
| 682 | (u32)(ntb_get_mw_size(ndev, i) >> 32), |
| 683 | MW0_SZ_HIGH + (i * 2)); |
| 684 | goto out; |
| 685 | } |
| 686 | |
| 687 | rc = ntb_write_remote_spad(ndev, MW0_SZ_LOW + (i * 2), |
| 688 | (u32) ntb_get_mw_size(ndev, i)); |
| 689 | if (rc) { |
| 690 | dev_err(&pdev->dev, "Error writing %u to remote spad %d\n", |
| 691 | (u32) ntb_get_mw_size(ndev, i), |
| 692 | MW0_SZ_LOW + (i * 2)); |
| 693 | goto out; |
| 694 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 695 | } |
| 696 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 697 | rc = ntb_write_remote_spad(ndev, NUM_MWS, ntb_max_mw(ndev)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 698 | if (rc) { |
| 699 | dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 700 | ntb_max_mw(ndev), NUM_MWS); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 701 | goto out; |
| 702 | } |
| 703 | |
| 704 | rc = ntb_write_remote_spad(ndev, NUM_QPS, nt->max_qps); |
| 705 | if (rc) { |
| 706 | dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", |
| 707 | nt->max_qps, NUM_QPS); |
| 708 | goto out; |
| 709 | } |
| 710 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 711 | rc = ntb_write_remote_spad(ndev, VERSION, NTB_TRANSPORT_VERSION); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 712 | if (rc) { |
| 713 | dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 714 | NTB_TRANSPORT_VERSION, VERSION); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 715 | goto out; |
| 716 | } |
| 717 | |
| 718 | /* Query the remote side for its info */ |
| 719 | rc = ntb_read_remote_spad(ndev, VERSION, &val); |
| 720 | if (rc) { |
| 721 | dev_err(&pdev->dev, "Error reading remote spad %d\n", VERSION); |
| 722 | goto out; |
| 723 | } |
| 724 | |
| 725 | if (val != NTB_TRANSPORT_VERSION) |
| 726 | goto out; |
| 727 | dev_dbg(&pdev->dev, "Remote version = %d\n", val); |
| 728 | |
| 729 | rc = ntb_read_remote_spad(ndev, NUM_QPS, &val); |
| 730 | if (rc) { |
| 731 | dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_QPS); |
| 732 | goto out; |
| 733 | } |
| 734 | |
| 735 | if (val != nt->max_qps) |
| 736 | goto out; |
| 737 | dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); |
| 738 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 739 | rc = ntb_read_remote_spad(ndev, NUM_MWS, &val); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 740 | if (rc) { |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 741 | dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_MWS); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 742 | goto out; |
| 743 | } |
| 744 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 745 | if (val != ntb_max_mw(ndev)) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 746 | goto out; |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 747 | dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 748 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 749 | for (i = 0; i < ntb_max_mw(ndev); i++) { |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 750 | u64 val64; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 751 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 752 | rc = ntb_read_remote_spad(ndev, MW0_SZ_HIGH + (i * 2), &val); |
| 753 | if (rc) { |
| 754 | dev_err(&pdev->dev, "Error reading remote spad %d\n", |
| 755 | MW0_SZ_HIGH + (i * 2)); |
| 756 | goto out1; |
| 757 | } |
| 758 | |
| 759 | val64 = (u64) val << 32; |
| 760 | |
| 761 | rc = ntb_read_remote_spad(ndev, MW0_SZ_LOW + (i * 2), &val); |
| 762 | if (rc) { |
| 763 | dev_err(&pdev->dev, "Error reading remote spad %d\n", |
| 764 | MW0_SZ_LOW + (i * 2)); |
| 765 | goto out1; |
| 766 | } |
| 767 | |
| 768 | val64 |= val; |
| 769 | |
| 770 | dev_dbg(&pdev->dev, "Remote MW%d size = %llu\n", i, val64); |
| 771 | |
| 772 | rc = ntb_set_mw(nt, i, val64); |
| 773 | if (rc) |
| 774 | goto out1; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 775 | } |
| 776 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 777 | nt->transport_link = NTB_LINK_UP; |
| 778 | |
| 779 | for (i = 0; i < nt->max_qps; i++) { |
| 780 | struct ntb_transport_qp *qp = &nt->qps[i]; |
| 781 | |
| 782 | ntb_transport_setup_qp_mw(nt, i); |
| 783 | |
| 784 | if (qp->client_ready == NTB_LINK_UP) |
| 785 | schedule_delayed_work(&qp->link_work, 0); |
| 786 | } |
| 787 | |
| 788 | return; |
| 789 | |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 790 | out1: |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 791 | for (i = 0; i < ntb_max_mw(ndev); i++) |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 792 | ntb_free_mw(nt, i); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 793 | out: |
| 794 | if (ntb_hw_link_status(ndev)) |
| 795 | schedule_delayed_work(&nt->link_work, |
| 796 | msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); |
| 797 | } |
| 798 | |
| 799 | static void ntb_qp_link_work(struct work_struct *work) |
| 800 | { |
| 801 | struct ntb_transport_qp *qp = container_of(work, |
| 802 | struct ntb_transport_qp, |
| 803 | link_work.work); |
| 804 | struct pci_dev *pdev = ntb_query_pdev(qp->ndev); |
| 805 | struct ntb_transport *nt = qp->transport; |
| 806 | int rc, val; |
| 807 | |
| 808 | WARN_ON(nt->transport_link != NTB_LINK_UP); |
| 809 | |
| 810 | rc = ntb_read_local_spad(nt->ndev, QP_LINKS, &val); |
| 811 | if (rc) { |
| 812 | dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS); |
| 813 | return; |
| 814 | } |
| 815 | |
| 816 | rc = ntb_write_remote_spad(nt->ndev, QP_LINKS, val | 1 << qp->qp_num); |
| 817 | if (rc) |
| 818 | dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", |
| 819 | val | 1 << qp->qp_num, QP_LINKS); |
| 820 | |
| 821 | /* query remote spad for qp ready bits */ |
| 822 | rc = ntb_read_remote_spad(nt->ndev, QP_LINKS, &val); |
| 823 | if (rc) |
| 824 | dev_err(&pdev->dev, "Error reading remote spad %d\n", QP_LINKS); |
| 825 | |
| 826 | dev_dbg(&pdev->dev, "Remote QP link status = %x\n", val); |
| 827 | |
| 828 | /* See if the remote side is up */ |
| 829 | if (1 << qp->qp_num & val) { |
| 830 | qp->qp_link = NTB_LINK_UP; |
| 831 | |
| 832 | dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); |
| 833 | if (qp->event_handler) |
| 834 | qp->event_handler(qp->cb_data, NTB_LINK_UP); |
| 835 | } else if (nt->transport_link == NTB_LINK_UP) |
| 836 | schedule_delayed_work(&qp->link_work, |
| 837 | msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); |
| 838 | } |
| 839 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 840 | static int ntb_transport_init_queue(struct ntb_transport *nt, |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 841 | unsigned int qp_num) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 842 | { |
| 843 | struct ntb_transport_qp *qp; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 844 | unsigned int num_qps_mw, tx_size; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 845 | u8 mw_num, mw_max; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 846 | u64 qp_offset; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 847 | |
| 848 | mw_max = ntb_max_mw(nt->ndev); |
| 849 | mw_num = QP_TO_MW(nt->ndev, qp_num); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 850 | |
| 851 | qp = &nt->qps[qp_num]; |
| 852 | qp->qp_num = qp_num; |
| 853 | qp->transport = nt; |
| 854 | qp->ndev = nt->ndev; |
| 855 | qp->qp_link = NTB_LINK_DOWN; |
| 856 | qp->client_ready = NTB_LINK_DOWN; |
| 857 | qp->event_handler = NULL; |
| 858 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 859 | if (nt->max_qps % mw_max && mw_num < nt->max_qps % mw_max) |
| 860 | num_qps_mw = nt->max_qps / mw_max + 1; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 861 | else |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 862 | num_qps_mw = nt->max_qps / mw_max; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 863 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 864 | tx_size = (unsigned int) ntb_get_mw_size(qp->ndev, mw_num) / num_qps_mw; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 865 | qp_offset = qp_num / mw_max * tx_size; |
| 866 | qp->tx_mw = ntb_get_mw_vbase(nt->ndev, mw_num) + qp_offset; |
| 867 | if (!qp->tx_mw) |
| 868 | return -EINVAL; |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 869 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 870 | qp->tx_mw_phys = ntb_get_mw_base(qp->ndev, mw_num) + qp_offset; |
| 871 | if (!qp->tx_mw_phys) |
| 872 | return -EINVAL; |
| 873 | |
| 874 | tx_size -= sizeof(struct ntb_rx_info); |
| 875 | qp->rx_info = qp->tx_mw + tx_size; |
| 876 | |
Jon Mason | c9d534c | 2013-02-01 15:45:16 -0700 | [diff] [blame] | 877 | /* Due to housekeeping, there must be atleast 2 buffs */ |
| 878 | qp->tx_max_frame = min(transport_mtu, tx_size / 2); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 879 | qp->tx_max_entry = tx_size / qp->tx_max_frame; |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 880 | |
Jon Mason | 1517a3f | 2013-07-30 15:58:49 -0700 | [diff] [blame] | 881 | if (ntb_query_debugfs(nt->ndev)) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 882 | char debugfs_name[4]; |
| 883 | |
| 884 | snprintf(debugfs_name, 4, "qp%d", qp_num); |
| 885 | qp->debugfs_dir = debugfs_create_dir(debugfs_name, |
Jon Mason | 1517a3f | 2013-07-30 15:58:49 -0700 | [diff] [blame] | 886 | ntb_query_debugfs(nt->ndev)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 887 | |
| 888 | qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, |
| 889 | qp->debugfs_dir, qp, |
| 890 | &ntb_qp_debugfs_stats); |
| 891 | } |
| 892 | |
| 893 | INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 894 | INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 895 | |
| 896 | spin_lock_init(&qp->ntb_rx_pend_q_lock); |
| 897 | spin_lock_init(&qp->ntb_rx_free_q_lock); |
| 898 | spin_lock_init(&qp->ntb_tx_free_q_lock); |
| 899 | |
| 900 | INIT_LIST_HEAD(&qp->rx_pend_q); |
| 901 | INIT_LIST_HEAD(&qp->rx_free_q); |
| 902 | INIT_LIST_HEAD(&qp->tx_free_q); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 903 | |
| 904 | return 0; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 905 | } |
| 906 | |
| 907 | int ntb_transport_init(struct pci_dev *pdev) |
| 908 | { |
| 909 | struct ntb_transport *nt; |
| 910 | int rc, i; |
| 911 | |
| 912 | nt = kzalloc(sizeof(struct ntb_transport), GFP_KERNEL); |
| 913 | if (!nt) |
| 914 | return -ENOMEM; |
| 915 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 916 | nt->ndev = ntb_register_transport(pdev, nt); |
| 917 | if (!nt->ndev) { |
| 918 | rc = -EIO; |
| 919 | goto err; |
| 920 | } |
| 921 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 922 | nt->mw = kcalloc(ntb_max_mw(nt->ndev), sizeof(struct ntb_transport_mw), |
| 923 | GFP_KERNEL); |
| 924 | if (!nt->mw) { |
| 925 | rc = -ENOMEM; |
| 926 | goto err1; |
| 927 | } |
| 928 | |
| 929 | if (max_num_clients) |
| 930 | nt->max_qps = min(ntb_max_cbs(nt->ndev), max_num_clients); |
| 931 | else |
| 932 | nt->max_qps = min(ntb_max_cbs(nt->ndev), ntb_max_mw(nt->ndev)); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 933 | |
| 934 | nt->qps = kcalloc(nt->max_qps, sizeof(struct ntb_transport_qp), |
| 935 | GFP_KERNEL); |
| 936 | if (!nt->qps) { |
| 937 | rc = -ENOMEM; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 938 | goto err2; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 939 | } |
| 940 | |
| 941 | nt->qp_bitmap = ((u64) 1 << nt->max_qps) - 1; |
| 942 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 943 | for (i = 0; i < nt->max_qps; i++) { |
| 944 | rc = ntb_transport_init_queue(nt, i); |
| 945 | if (rc) |
| 946 | goto err3; |
| 947 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 948 | |
| 949 | INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 950 | INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 951 | |
| 952 | rc = ntb_register_event_callback(nt->ndev, |
| 953 | ntb_transport_event_callback); |
| 954 | if (rc) |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 955 | goto err3; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 956 | |
| 957 | INIT_LIST_HEAD(&nt->client_devs); |
| 958 | rc = ntb_bus_init(nt); |
| 959 | if (rc) |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 960 | goto err4; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 961 | |
| 962 | if (ntb_hw_link_status(nt->ndev)) |
| 963 | schedule_delayed_work(&nt->link_work, 0); |
| 964 | |
| 965 | return 0; |
| 966 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 967 | err4: |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 968 | ntb_unregister_event_callback(nt->ndev); |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 969 | err3: |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 970 | kfree(nt->qps); |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 971 | err2: |
| 972 | kfree(nt->mw); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 973 | err1: |
| 974 | ntb_unregister_transport(nt->ndev); |
| 975 | err: |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 976 | kfree(nt); |
| 977 | return rc; |
| 978 | } |
| 979 | |
| 980 | void ntb_transport_free(void *transport) |
| 981 | { |
| 982 | struct ntb_transport *nt = transport; |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 983 | struct ntb_device *ndev = nt->ndev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 984 | int i; |
| 985 | |
Jon Mason | fca4d51 | 2013-09-09 13:39:55 -0700 | [diff] [blame] | 986 | ntb_transport_link_cleanup(nt); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 987 | |
| 988 | /* verify that all the qp's are freed */ |
Jon Mason | 1517a3f | 2013-07-30 15:58:49 -0700 | [diff] [blame] | 989 | for (i = 0; i < nt->max_qps; i++) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 990 | if (!test_bit(i, &nt->qp_bitmap)) |
| 991 | ntb_transport_free_queue(&nt->qps[i]); |
Jon Mason | 1517a3f | 2013-07-30 15:58:49 -0700 | [diff] [blame] | 992 | debugfs_remove_recursive(nt->qps[i].debugfs_dir); |
| 993 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 994 | |
| 995 | ntb_bus_remove(nt); |
| 996 | |
| 997 | cancel_delayed_work_sync(&nt->link_work); |
| 998 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 999 | ntb_unregister_event_callback(ndev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1000 | |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 1001 | for (i = 0; i < ntb_max_mw(ndev); i++) |
Jon Mason | 113fc50 | 2013-01-30 11:40:52 -0700 | [diff] [blame] | 1002 | ntb_free_mw(nt, i); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1003 | |
| 1004 | kfree(nt->qps); |
Jon Mason | 948d3a6 | 2013-04-18 17:07:36 -0700 | [diff] [blame] | 1005 | kfree(nt->mw); |
| 1006 | ntb_unregister_transport(ndev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1007 | kfree(nt); |
| 1008 | } |
| 1009 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1010 | static void ntb_rx_copy_callback(void *data) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1011 | { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1012 | struct ntb_queue_entry *entry = data; |
| 1013 | struct ntb_transport_qp *qp = entry->qp; |
Jon Mason | 448c6fb | 2013-01-19 02:02:27 -0700 | [diff] [blame] | 1014 | void *cb_data = entry->cb_data; |
| 1015 | unsigned int len = entry->len; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1016 | struct ntb_payload_header *hdr = entry->rx_hdr; |
Jon Mason | 448c6fb | 2013-01-19 02:02:27 -0700 | [diff] [blame] | 1017 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1018 | /* Ensure that the data is fully copied out before clearing the flag */ |
| 1019 | wmb(); |
| 1020 | hdr->flags = 0; |
| 1021 | |
| 1022 | iowrite32(entry->index, &qp->rx_info->entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1023 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1024 | ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); |
Jon Mason | 448c6fb | 2013-01-19 02:02:27 -0700 | [diff] [blame] | 1025 | |
| 1026 | if (qp->rx_handler && qp->client_ready == NTB_LINK_UP) |
| 1027 | qp->rx_handler(qp, qp->cb_data, cb_data, len); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1028 | } |
| 1029 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1030 | static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset) |
| 1031 | { |
| 1032 | void *buf = entry->buf; |
| 1033 | size_t len = entry->len; |
| 1034 | |
| 1035 | memcpy(buf, offset, len); |
| 1036 | |
| 1037 | ntb_rx_copy_callback(entry); |
| 1038 | } |
| 1039 | |
| 1040 | static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset, |
| 1041 | size_t len) |
| 1042 | { |
| 1043 | struct dma_async_tx_descriptor *txd; |
| 1044 | struct ntb_transport_qp *qp = entry->qp; |
| 1045 | struct dma_chan *chan = qp->dma_chan; |
| 1046 | struct dma_device *device; |
| 1047 | size_t pay_off, buff_off; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1048 | struct dmaengine_unmap_data *unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1049 | dma_cookie_t cookie; |
| 1050 | void *buf = entry->buf; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1051 | |
| 1052 | entry->len = len; |
| 1053 | |
| 1054 | if (!chan) |
| 1055 | goto err; |
| 1056 | |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 1057 | if (len < copy_bytes) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1058 | goto err_wait; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1059 | |
| 1060 | device = chan->device; |
| 1061 | pay_off = (size_t) offset & ~PAGE_MASK; |
| 1062 | buff_off = (size_t) buf & ~PAGE_MASK; |
| 1063 | |
| 1064 | if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1065 | goto err_wait; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1066 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1067 | unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT); |
| 1068 | if (!unmap) |
| 1069 | goto err_wait; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1070 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1071 | unmap->len = len; |
| 1072 | unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset), |
| 1073 | pay_off, len, DMA_TO_DEVICE); |
| 1074 | if (dma_mapping_error(device->dev, unmap->addr[0])) |
| 1075 | goto err_get_unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1076 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1077 | unmap->to_cnt = 1; |
| 1078 | |
| 1079 | unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf), |
| 1080 | buff_off, len, DMA_FROM_DEVICE); |
| 1081 | if (dma_mapping_error(device->dev, unmap->addr[1])) |
| 1082 | goto err_get_unmap; |
| 1083 | |
| 1084 | unmap->from_cnt = 1; |
| 1085 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1086 | txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], |
Bartlomiej Zolnierkiewicz | 0776ae7 | 2013-10-18 19:35:33 +0200 | [diff] [blame] | 1087 | unmap->addr[0], len, |
| 1088 | DMA_PREP_INTERRUPT); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1089 | if (!txd) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1090 | goto err_get_unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1091 | |
| 1092 | txd->callback = ntb_rx_copy_callback; |
| 1093 | txd->callback_param = entry; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1094 | dma_set_unmap(txd, unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1095 | |
| 1096 | cookie = dmaengine_submit(txd); |
| 1097 | if (dma_submit_error(cookie)) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1098 | goto err_set_unmap; |
| 1099 | |
| 1100 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1101 | |
| 1102 | qp->last_cookie = cookie; |
| 1103 | |
| 1104 | qp->rx_async++; |
| 1105 | |
| 1106 | return; |
| 1107 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1108 | err_set_unmap: |
| 1109 | dmaengine_unmap_put(unmap); |
| 1110 | err_get_unmap: |
| 1111 | dmaengine_unmap_put(unmap); |
| 1112 | err_wait: |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1113 | /* If the callbacks come out of order, the writing of the index to the |
| 1114 | * last completed will be out of order. This may result in the |
| 1115 | * receive stalling forever. |
| 1116 | */ |
| 1117 | dma_sync_wait(chan, qp->last_cookie); |
| 1118 | err: |
| 1119 | ntb_memcpy_rx(entry, offset); |
| 1120 | qp->rx_memcpy++; |
| 1121 | } |
| 1122 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1123 | static int ntb_process_rxc(struct ntb_transport_qp *qp) |
| 1124 | { |
| 1125 | struct ntb_payload_header *hdr; |
| 1126 | struct ntb_queue_entry *entry; |
| 1127 | void *offset; |
| 1128 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1129 | offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; |
| 1130 | hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); |
| 1131 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1132 | entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); |
| 1133 | if (!entry) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1134 | dev_dbg(&ntb_query_pdev(qp->ndev)->dev, |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 1135 | "no buffer - HDR ver %u, len %d, flags %x\n", |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1136 | hdr->ver, hdr->len, hdr->flags); |
| 1137 | qp->rx_err_no_buf++; |
| 1138 | return -ENOMEM; |
| 1139 | } |
| 1140 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1141 | if (!(hdr->flags & DESC_DONE_FLAG)) { |
| 1142 | ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1143 | &qp->rx_pend_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1144 | qp->rx_ring_empty++; |
| 1145 | return -EAGAIN; |
| 1146 | } |
| 1147 | |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 1148 | if (hdr->ver != (u32) qp->rx_pkts) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1149 | dev_dbg(&ntb_query_pdev(qp->ndev)->dev, |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 1150 | "qp %d: version mismatch, expected %llu - got %u\n", |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1151 | qp->qp_num, qp->rx_pkts, hdr->ver); |
| 1152 | ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1153 | &qp->rx_pend_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1154 | qp->rx_err_ver++; |
| 1155 | return -EIO; |
| 1156 | } |
| 1157 | |
| 1158 | if (hdr->flags & LINK_DOWN_FLAG) { |
| 1159 | ntb_qp_link_down(qp); |
| 1160 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1161 | goto err; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1162 | } |
| 1163 | |
| 1164 | dev_dbg(&ntb_query_pdev(qp->ndev)->dev, |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 1165 | "rx offset %u, ver %u - %d payload received, buf size %d\n", |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1166 | qp->rx_index, hdr->ver, hdr->len, entry->len); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1167 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1168 | qp->rx_bytes += hdr->len; |
| 1169 | qp->rx_pkts++; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1170 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1171 | if (hdr->len > entry->len) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1172 | qp->rx_err_oflow++; |
| 1173 | dev_dbg(&ntb_query_pdev(qp->ndev)->dev, |
| 1174 | "RX overflow! Wanted %d got %d\n", |
| 1175 | hdr->len, entry->len); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1176 | |
| 1177 | goto err; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1178 | } |
| 1179 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1180 | entry->index = qp->rx_index; |
| 1181 | entry->rx_hdr = hdr; |
| 1182 | |
| 1183 | ntb_async_rx(entry, offset, hdr->len); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1184 | |
| 1185 | out: |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1186 | qp->rx_index++; |
| 1187 | qp->rx_index %= qp->rx_max_entry; |
| 1188 | |
| 1189 | return 0; |
| 1190 | |
| 1191 | err: |
Jon Mason | 53ca4fe | 2013-11-26 11:21:50 -0700 | [diff] [blame] | 1192 | ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, &qp->rx_pend_q); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1193 | /* Ensure that the data is fully copied out before clearing the flag */ |
| 1194 | wmb(); |
| 1195 | hdr->flags = 0; |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 1196 | iowrite32(qp->rx_index, &qp->rx_info->entry); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1197 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1198 | goto out; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1199 | } |
| 1200 | |
Jon Mason | e8aeb60 | 2013-04-18 17:59:44 -0700 | [diff] [blame] | 1201 | static int ntb_transport_rxc_db(void *data, int db_num) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1202 | { |
Jon Mason | e8aeb60 | 2013-04-18 17:59:44 -0700 | [diff] [blame] | 1203 | struct ntb_transport_qp *qp = data; |
Jon Mason | c336acd | 2013-01-17 15:28:45 -0700 | [diff] [blame] | 1204 | int rc, i; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1205 | |
Jon Mason | e8aeb60 | 2013-04-18 17:59:44 -0700 | [diff] [blame] | 1206 | dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n", |
| 1207 | __func__, db_num); |
| 1208 | |
Jon Mason | c336acd | 2013-01-17 15:28:45 -0700 | [diff] [blame] | 1209 | /* Limit the number of packets processed in a single interrupt to |
| 1210 | * provide fairness to others |
| 1211 | */ |
| 1212 | for (i = 0; i < qp->rx_max_entry; i++) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1213 | rc = ntb_process_rxc(qp); |
Jon Mason | c336acd | 2013-01-17 15:28:45 -0700 | [diff] [blame] | 1214 | if (rc) |
| 1215 | break; |
| 1216 | } |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1217 | |
| 1218 | if (qp->dma_chan) |
| 1219 | dma_async_issue_pending(qp->dma_chan); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1220 | |
Jon Mason | e8aeb60 | 2013-04-18 17:59:44 -0700 | [diff] [blame] | 1221 | return i; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1222 | } |
| 1223 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1224 | static void ntb_tx_copy_callback(void *data) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1225 | { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1226 | struct ntb_queue_entry *entry = data; |
| 1227 | struct ntb_transport_qp *qp = entry->qp; |
| 1228 | struct ntb_payload_header __iomem *hdr = entry->tx_hdr; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1229 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1230 | /* Ensure that the data is fully copied out before setting the flags */ |
Jon Mason | 842c1dd | 2013-01-19 02:02:17 -0700 | [diff] [blame] | 1231 | wmb(); |
Jon Mason | 7446564 | 2013-01-21 15:28:52 -0700 | [diff] [blame] | 1232 | iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1233 | |
Jon Mason | 4979388 | 2013-07-15 15:53:54 -0700 | [diff] [blame] | 1234 | ntb_ring_doorbell(qp->ndev, qp->qp_num); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1235 | |
| 1236 | /* The entry length can only be zero if the packet is intended to be a |
| 1237 | * "link down" or similar. Since no payload is being sent in these |
| 1238 | * cases, there is nothing to add to the completion queue. |
| 1239 | */ |
| 1240 | if (entry->len > 0) { |
| 1241 | qp->tx_bytes += entry->len; |
| 1242 | |
| 1243 | if (qp->tx_handler) |
| 1244 | qp->tx_handler(qp, qp->cb_data, entry->cb_data, |
| 1245 | entry->len); |
| 1246 | } |
| 1247 | |
| 1248 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); |
| 1249 | } |
| 1250 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1251 | static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) |
| 1252 | { |
| 1253 | memcpy_toio(offset, entry->buf, entry->len); |
| 1254 | |
| 1255 | ntb_tx_copy_callback(entry); |
| 1256 | } |
| 1257 | |
| 1258 | static void ntb_async_tx(struct ntb_transport_qp *qp, |
| 1259 | struct ntb_queue_entry *entry) |
| 1260 | { |
| 1261 | struct ntb_payload_header __iomem *hdr; |
| 1262 | struct dma_async_tx_descriptor *txd; |
| 1263 | struct dma_chan *chan = qp->dma_chan; |
| 1264 | struct dma_device *device; |
| 1265 | size_t dest_off, buff_off; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1266 | struct dmaengine_unmap_data *unmap; |
| 1267 | dma_addr_t dest; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1268 | dma_cookie_t cookie; |
| 1269 | void __iomem *offset; |
| 1270 | size_t len = entry->len; |
| 1271 | void *buf = entry->buf; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1272 | |
| 1273 | offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; |
| 1274 | hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); |
| 1275 | entry->tx_hdr = hdr; |
| 1276 | |
| 1277 | iowrite32(entry->len, &hdr->len); |
| 1278 | iowrite32((u32) qp->tx_pkts, &hdr->ver); |
| 1279 | |
| 1280 | if (!chan) |
| 1281 | goto err; |
| 1282 | |
| 1283 | if (len < copy_bytes) |
| 1284 | goto err; |
| 1285 | |
| 1286 | device = chan->device; |
| 1287 | dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index; |
| 1288 | buff_off = (size_t) buf & ~PAGE_MASK; |
| 1289 | dest_off = (size_t) dest & ~PAGE_MASK; |
| 1290 | |
| 1291 | if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) |
| 1292 | goto err; |
| 1293 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1294 | unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); |
| 1295 | if (!unmap) |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1296 | goto err; |
| 1297 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1298 | unmap->len = len; |
| 1299 | unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf), |
| 1300 | buff_off, len, DMA_TO_DEVICE); |
| 1301 | if (dma_mapping_error(device->dev, unmap->addr[0])) |
| 1302 | goto err_get_unmap; |
| 1303 | |
| 1304 | unmap->to_cnt = 1; |
| 1305 | |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1306 | txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, |
Bartlomiej Zolnierkiewicz | 0776ae7 | 2013-10-18 19:35:33 +0200 | [diff] [blame] | 1307 | DMA_PREP_INTERRUPT); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1308 | if (!txd) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1309 | goto err_get_unmap; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1310 | |
| 1311 | txd->callback = ntb_tx_copy_callback; |
| 1312 | txd->callback_param = entry; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1313 | dma_set_unmap(txd, unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1314 | |
| 1315 | cookie = dmaengine_submit(txd); |
| 1316 | if (dma_submit_error(cookie)) |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1317 | goto err_set_unmap; |
| 1318 | |
| 1319 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1320 | |
| 1321 | dma_async_issue_pending(chan); |
| 1322 | qp->tx_async++; |
| 1323 | |
| 1324 | return; |
Bartlomiej Zolnierkiewicz | 6f57fd0 | 2013-10-18 19:35:31 +0200 | [diff] [blame] | 1325 | err_set_unmap: |
| 1326 | dmaengine_unmap_put(unmap); |
| 1327 | err_get_unmap: |
| 1328 | dmaengine_unmap_put(unmap); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1329 | err: |
| 1330 | ntb_memcpy_tx(entry, offset); |
| 1331 | qp->tx_memcpy++; |
| 1332 | } |
| 1333 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1334 | static int ntb_process_tx(struct ntb_transport_qp *qp, |
| 1335 | struct ntb_queue_entry *entry) |
| 1336 | { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1337 | dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%lld - tx %u, entry len %d flags %x buff %p\n", |
| 1338 | qp->tx_pkts, qp->tx_index, entry->len, entry->flags, |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1339 | entry->buf); |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1340 | if (qp->tx_index == qp->remote_rx_info->entry) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1341 | qp->tx_ring_full++; |
| 1342 | return -EAGAIN; |
| 1343 | } |
| 1344 | |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 1345 | if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1346 | if (qp->tx_handler) |
| 1347 | qp->tx_handler(qp->cb_data, qp, NULL, -EIO); |
| 1348 | |
| 1349 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, |
| 1350 | &qp->tx_free_q); |
| 1351 | return 0; |
| 1352 | } |
| 1353 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1354 | ntb_async_tx(qp, entry); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1355 | |
Jon Mason | 793c20e | 2013-01-19 02:02:26 -0700 | [diff] [blame] | 1356 | qp->tx_index++; |
| 1357 | qp->tx_index %= qp->tx_max_entry; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1358 | |
| 1359 | qp->tx_pkts++; |
| 1360 | |
| 1361 | return 0; |
| 1362 | } |
| 1363 | |
| 1364 | static void ntb_send_link_down(struct ntb_transport_qp *qp) |
| 1365 | { |
| 1366 | struct pci_dev *pdev = ntb_query_pdev(qp->ndev); |
| 1367 | struct ntb_queue_entry *entry; |
| 1368 | int i, rc; |
| 1369 | |
| 1370 | if (qp->qp_link == NTB_LINK_DOWN) |
| 1371 | return; |
| 1372 | |
| 1373 | qp->qp_link = NTB_LINK_DOWN; |
| 1374 | dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); |
| 1375 | |
| 1376 | for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1377 | entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1378 | if (entry) |
| 1379 | break; |
| 1380 | msleep(100); |
| 1381 | } |
| 1382 | |
| 1383 | if (!entry) |
| 1384 | return; |
| 1385 | |
| 1386 | entry->cb_data = NULL; |
| 1387 | entry->buf = NULL; |
| 1388 | entry->len = 0; |
| 1389 | entry->flags = LINK_DOWN_FLAG; |
| 1390 | |
| 1391 | rc = ntb_process_tx(qp, entry); |
| 1392 | if (rc) |
| 1393 | dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n", |
| 1394 | qp->qp_num); |
| 1395 | } |
| 1396 | |
| 1397 | /** |
| 1398 | * ntb_transport_create_queue - Create a new NTB transport layer queue |
| 1399 | * @rx_handler: receive callback function |
| 1400 | * @tx_handler: transmit callback function |
| 1401 | * @event_handler: event callback function |
| 1402 | * |
| 1403 | * Create a new NTB transport layer queue and provide the queue with a callback |
| 1404 | * routine for both transmit and receive. The receive callback routine will be |
| 1405 | * used to pass up data when the transport has received it on the queue. The |
| 1406 | * transmit callback routine will be called when the transport has completed the |
| 1407 | * transmission of the data on the queue and the data is ready to be freed. |
| 1408 | * |
| 1409 | * RETURNS: pointer to newly created ntb_queue, NULL on error. |
| 1410 | */ |
| 1411 | struct ntb_transport_qp * |
| 1412 | ntb_transport_create_queue(void *data, struct pci_dev *pdev, |
| 1413 | const struct ntb_queue_handlers *handlers) |
| 1414 | { |
| 1415 | struct ntb_queue_entry *entry; |
| 1416 | struct ntb_transport_qp *qp; |
| 1417 | struct ntb_transport *nt; |
| 1418 | unsigned int free_queue; |
| 1419 | int rc, i; |
| 1420 | |
| 1421 | nt = ntb_find_transport(pdev); |
| 1422 | if (!nt) |
| 1423 | goto err; |
| 1424 | |
| 1425 | free_queue = ffs(nt->qp_bitmap); |
| 1426 | if (!free_queue) |
| 1427 | goto err; |
| 1428 | |
| 1429 | /* decrement free_queue to make it zero based */ |
| 1430 | free_queue--; |
| 1431 | |
| 1432 | clear_bit(free_queue, &nt->qp_bitmap); |
| 1433 | |
| 1434 | qp = &nt->qps[free_queue]; |
| 1435 | qp->cb_data = data; |
| 1436 | qp->rx_handler = handlers->rx_handler; |
| 1437 | qp->tx_handler = handlers->tx_handler; |
| 1438 | qp->event_handler = handlers->event_handler; |
| 1439 | |
Jon Mason | 9468119 | 2013-11-19 09:34:47 -0700 | [diff] [blame] | 1440 | dmaengine_get(); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1441 | qp->dma_chan = dma_find_channel(DMA_MEMCPY); |
Jon Mason | 9468119 | 2013-11-19 09:34:47 -0700 | [diff] [blame] | 1442 | if (!qp->dma_chan) { |
| 1443 | dmaengine_put(); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1444 | dev_info(&pdev->dev, "Unable to allocate DMA channel, using CPU instead\n"); |
Jon Mason | 9468119 | 2013-11-19 09:34:47 -0700 | [diff] [blame] | 1445 | } |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1446 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1447 | for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { |
| 1448 | entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC); |
| 1449 | if (!entry) |
| 1450 | goto err1; |
| 1451 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1452 | entry->qp = qp; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1453 | ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1454 | &qp->rx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1455 | } |
| 1456 | |
| 1457 | for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { |
| 1458 | entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC); |
| 1459 | if (!entry) |
| 1460 | goto err2; |
| 1461 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1462 | entry->qp = qp; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1463 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1464 | &qp->tx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1465 | } |
| 1466 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1467 | rc = ntb_register_db_callback(qp->ndev, free_queue, qp, |
| 1468 | ntb_transport_rxc_db); |
| 1469 | if (rc) |
Jon Mason | e8aeb60 | 2013-04-18 17:59:44 -0700 | [diff] [blame] | 1470 | goto err2; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1471 | |
| 1472 | dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); |
| 1473 | |
| 1474 | return qp; |
| 1475 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1476 | err2: |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1477 | while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1478 | kfree(entry); |
| 1479 | err1: |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1480 | while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1481 | kfree(entry); |
Jon Mason | 9468119 | 2013-11-19 09:34:47 -0700 | [diff] [blame] | 1482 | if (qp->dma_chan) |
| 1483 | dmaengine_put(); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1484 | set_bit(free_queue, &nt->qp_bitmap); |
| 1485 | err: |
| 1486 | return NULL; |
| 1487 | } |
| 1488 | EXPORT_SYMBOL_GPL(ntb_transport_create_queue); |
| 1489 | |
| 1490 | /** |
| 1491 | * ntb_transport_free_queue - Frees NTB transport queue |
| 1492 | * @qp: NTB queue to be freed |
| 1493 | * |
| 1494 | * Frees NTB transport queue |
| 1495 | */ |
| 1496 | void ntb_transport_free_queue(struct ntb_transport_qp *qp) |
| 1497 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1498 | struct pci_dev *pdev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1499 | struct ntb_queue_entry *entry; |
| 1500 | |
| 1501 | if (!qp) |
| 1502 | return; |
| 1503 | |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1504 | pdev = ntb_query_pdev(qp->ndev); |
| 1505 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1506 | if (qp->dma_chan) { |
| 1507 | struct dma_chan *chan = qp->dma_chan; |
| 1508 | /* Putting the dma_chan to NULL will force any new traffic to be |
| 1509 | * processed by the CPU instead of the DAM engine |
| 1510 | */ |
| 1511 | qp->dma_chan = NULL; |
| 1512 | |
| 1513 | /* Try to be nice and wait for any queued DMA engine |
| 1514 | * transactions to process before smashing it with a rock |
| 1515 | */ |
| 1516 | dma_sync_wait(chan, qp->last_cookie); |
| 1517 | dmaengine_terminate_all(chan); |
| 1518 | dmaengine_put(); |
| 1519 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1520 | |
| 1521 | ntb_unregister_db_callback(qp->ndev, qp->qp_num); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1522 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1523 | cancel_delayed_work_sync(&qp->link_work); |
| 1524 | |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1525 | while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1526 | kfree(entry); |
| 1527 | |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1528 | while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) { |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1529 | dev_warn(&pdev->dev, "Freeing item from a non-empty queue\n"); |
| 1530 | kfree(entry); |
| 1531 | } |
| 1532 | |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1533 | while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1534 | kfree(entry); |
| 1535 | |
| 1536 | set_bit(qp->qp_num, &qp->transport->qp_bitmap); |
| 1537 | |
| 1538 | dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); |
| 1539 | } |
| 1540 | EXPORT_SYMBOL_GPL(ntb_transport_free_queue); |
| 1541 | |
| 1542 | /** |
| 1543 | * ntb_transport_rx_remove - Dequeues enqueued rx packet |
| 1544 | * @qp: NTB queue to be freed |
| 1545 | * @len: pointer to variable to write enqueued buffers length |
| 1546 | * |
| 1547 | * Dequeues unused buffers from receive queue. Should only be used during |
| 1548 | * shutdown of qp. |
| 1549 | * |
| 1550 | * RETURNS: NULL error value on error, or void* for success. |
| 1551 | */ |
| 1552 | void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len) |
| 1553 | { |
| 1554 | struct ntb_queue_entry *entry; |
| 1555 | void *buf; |
| 1556 | |
| 1557 | if (!qp || qp->client_ready == NTB_LINK_UP) |
| 1558 | return NULL; |
| 1559 | |
| 1560 | entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); |
| 1561 | if (!entry) |
| 1562 | return NULL; |
| 1563 | |
| 1564 | buf = entry->cb_data; |
| 1565 | *len = entry->len; |
| 1566 | |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1567 | ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1568 | |
| 1569 | return buf; |
| 1570 | } |
| 1571 | EXPORT_SYMBOL_GPL(ntb_transport_rx_remove); |
| 1572 | |
| 1573 | /** |
| 1574 | * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry |
| 1575 | * @qp: NTB transport layer queue the entry is to be enqueued on |
| 1576 | * @cb: per buffer pointer for callback function to use |
| 1577 | * @data: pointer to data buffer that incoming packets will be copied into |
| 1578 | * @len: length of the data buffer |
| 1579 | * |
| 1580 | * Enqueue a new receive buffer onto the transport queue into which a NTB |
| 1581 | * payload can be received into. |
| 1582 | * |
| 1583 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 1584 | */ |
| 1585 | int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, |
| 1586 | unsigned int len) |
| 1587 | { |
| 1588 | struct ntb_queue_entry *entry; |
| 1589 | |
| 1590 | if (!qp) |
| 1591 | return -EINVAL; |
| 1592 | |
| 1593 | entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q); |
| 1594 | if (!entry) |
| 1595 | return -ENOMEM; |
| 1596 | |
| 1597 | entry->cb_data = cb; |
| 1598 | entry->buf = data; |
| 1599 | entry->len = len; |
| 1600 | |
Jon Mason | f766755 | 2013-01-19 02:02:24 -0700 | [diff] [blame] | 1601 | ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, &qp->rx_pend_q); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1602 | |
| 1603 | return 0; |
| 1604 | } |
| 1605 | EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue); |
| 1606 | |
| 1607 | /** |
| 1608 | * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry |
| 1609 | * @qp: NTB transport layer queue the entry is to be enqueued on |
| 1610 | * @cb: per buffer pointer for callback function to use |
| 1611 | * @data: pointer to data buffer that will be sent |
| 1612 | * @len: length of the data buffer |
| 1613 | * |
| 1614 | * Enqueue a new transmit buffer onto the transport queue from which a NTB |
Jon Mason | f9a2cf8 | 2013-07-29 16:46:43 -0700 | [diff] [blame] | 1615 | * payload will be transmitted. This assumes that a lock is being held to |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1616 | * serialize access to the qp. |
| 1617 | * |
| 1618 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. |
| 1619 | */ |
| 1620 | int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, |
| 1621 | unsigned int len) |
| 1622 | { |
| 1623 | struct ntb_queue_entry *entry; |
| 1624 | int rc; |
| 1625 | |
| 1626 | if (!qp || qp->qp_link != NTB_LINK_UP || !len) |
| 1627 | return -EINVAL; |
| 1628 | |
| 1629 | entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1630 | if (!entry) { |
| 1631 | qp->tx_err_no_buf++; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1632 | return -ENOMEM; |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1633 | } |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1634 | |
| 1635 | entry->cb_data = cb; |
| 1636 | entry->buf = data; |
| 1637 | entry->len = len; |
| 1638 | entry->flags = 0; |
| 1639 | |
| 1640 | rc = ntb_process_tx(qp, entry); |
| 1641 | if (rc) |
| 1642 | ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, |
| 1643 | &qp->tx_free_q); |
| 1644 | |
| 1645 | return rc; |
| 1646 | } |
| 1647 | EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue); |
| 1648 | |
| 1649 | /** |
| 1650 | * ntb_transport_link_up - Notify NTB transport of client readiness to use queue |
| 1651 | * @qp: NTB transport layer queue to be enabled |
| 1652 | * |
| 1653 | * Notify NTB transport layer of client readiness to use queue |
| 1654 | */ |
| 1655 | void ntb_transport_link_up(struct ntb_transport_qp *qp) |
| 1656 | { |
| 1657 | if (!qp) |
| 1658 | return; |
| 1659 | |
| 1660 | qp->client_ready = NTB_LINK_UP; |
| 1661 | |
| 1662 | if (qp->transport->transport_link == NTB_LINK_UP) |
| 1663 | schedule_delayed_work(&qp->link_work, 0); |
| 1664 | } |
| 1665 | EXPORT_SYMBOL_GPL(ntb_transport_link_up); |
| 1666 | |
| 1667 | /** |
| 1668 | * ntb_transport_link_down - Notify NTB transport to no longer enqueue data |
| 1669 | * @qp: NTB transport layer queue to be disabled |
| 1670 | * |
| 1671 | * Notify NTB transport layer of client's desire to no longer receive data on |
| 1672 | * transport queue specified. It is the client's responsibility to ensure all |
Jon Mason | f9a2cf8 | 2013-07-29 16:46:43 -0700 | [diff] [blame] | 1673 | * entries on queue are purged or otherwise handled appropriately. |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1674 | */ |
| 1675 | void ntb_transport_link_down(struct ntb_transport_qp *qp) |
| 1676 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1677 | struct pci_dev *pdev; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1678 | int rc, val; |
| 1679 | |
| 1680 | if (!qp) |
| 1681 | return; |
| 1682 | |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1683 | pdev = ntb_query_pdev(qp->ndev); |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1684 | qp->client_ready = NTB_LINK_DOWN; |
| 1685 | |
| 1686 | rc = ntb_read_local_spad(qp->ndev, QP_LINKS, &val); |
| 1687 | if (rc) { |
| 1688 | dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS); |
| 1689 | return; |
| 1690 | } |
| 1691 | |
| 1692 | rc = ntb_write_remote_spad(qp->ndev, QP_LINKS, |
| 1693 | val & ~(1 << qp->qp_num)); |
| 1694 | if (rc) |
| 1695 | dev_err(&pdev->dev, "Error writing %x to remote spad %d\n", |
| 1696 | val & ~(1 << qp->qp_num), QP_LINKS); |
| 1697 | |
| 1698 | if (qp->qp_link == NTB_LINK_UP) |
| 1699 | ntb_send_link_down(qp); |
| 1700 | else |
| 1701 | cancel_delayed_work_sync(&qp->link_work); |
| 1702 | } |
| 1703 | EXPORT_SYMBOL_GPL(ntb_transport_link_down); |
| 1704 | |
| 1705 | /** |
| 1706 | * ntb_transport_link_query - Query transport link state |
| 1707 | * @qp: NTB transport layer queue to be queried |
| 1708 | * |
| 1709 | * Query connectivity to the remote system of the NTB transport queue |
| 1710 | * |
| 1711 | * RETURNS: true for link up or false for link down |
| 1712 | */ |
| 1713 | bool ntb_transport_link_query(struct ntb_transport_qp *qp) |
| 1714 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1715 | if (!qp) |
| 1716 | return false; |
| 1717 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1718 | return qp->qp_link == NTB_LINK_UP; |
| 1719 | } |
| 1720 | EXPORT_SYMBOL_GPL(ntb_transport_link_query); |
| 1721 | |
| 1722 | /** |
| 1723 | * ntb_transport_qp_num - Query the qp number |
| 1724 | * @qp: NTB transport layer queue to be queried |
| 1725 | * |
| 1726 | * Query qp number of the NTB transport queue |
| 1727 | * |
| 1728 | * RETURNS: a zero based number specifying the qp number |
| 1729 | */ |
| 1730 | unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp) |
| 1731 | { |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1732 | if (!qp) |
| 1733 | return 0; |
| 1734 | |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1735 | return qp->qp_num; |
| 1736 | } |
| 1737 | EXPORT_SYMBOL_GPL(ntb_transport_qp_num); |
| 1738 | |
| 1739 | /** |
| 1740 | * ntb_transport_max_size - Query the max payload size of a qp |
| 1741 | * @qp: NTB transport layer queue to be queried |
| 1742 | * |
| 1743 | * Query the maximum payload size permissible on the given qp |
| 1744 | * |
| 1745 | * RETURNS: the max payload size of a qp |
| 1746 | */ |
Jon Mason | ef114ed | 2013-01-19 02:02:18 -0700 | [diff] [blame] | 1747 | unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp) |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1748 | { |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1749 | unsigned int max; |
| 1750 | |
Jon Mason | 186f27f | 2013-01-22 11:35:40 -0700 | [diff] [blame] | 1751 | if (!qp) |
| 1752 | return 0; |
| 1753 | |
Jon Mason | 282a2fe | 2013-02-12 09:52:50 -0700 | [diff] [blame] | 1754 | if (!qp->dma_chan) |
| 1755 | return qp->tx_max_frame - sizeof(struct ntb_payload_header); |
| 1756 | |
| 1757 | /* If DMA engine usage is possible, try to find the max size for that */ |
| 1758 | max = qp->tx_max_frame - sizeof(struct ntb_payload_header); |
| 1759 | max -= max % (1 << qp->dma_chan->device->copy_align); |
| 1760 | |
| 1761 | return max; |
Jon Mason | fce8a7b | 2012-11-16 19:27:12 -0700 | [diff] [blame] | 1762 | } |
| 1763 | EXPORT_SYMBOL_GPL(ntb_transport_max_size); |