Leon Romanovsky | 6bf9d8f | 2020-07-19 10:25:21 +0300 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | /* |
| 3 | * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. |
| 4 | * Copyright (c) 2004 Infinicon Corporation. All rights reserved. |
Jianxin Xiong | 3bc489e | 2020-12-15 13:27:14 -0800 | [diff] [blame] | 5 | * Copyright (c) 2004, 2020 Intel Corporation. All rights reserved. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6 | * Copyright (c) 2004 Topspin Corporation. All rights reserved. |
| 7 | * Copyright (c) 2004 Voltaire Corporation. All rights reserved. |
Roland Dreier | 2a1d9b7 | 2005-08-10 23:03:10 -0700 | [diff] [blame] | 8 | * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. |
Roland Dreier | f7c6a7b | 2007-03-04 16:15:11 -0800 | [diff] [blame] | 9 | * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | */ |
| 11 | |
Leon Romanovsky | 6bf9d8f | 2020-07-19 10:25:21 +0300 | [diff] [blame] | 12 | #ifndef IB_VERBS_H |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #define IB_VERBS_H |
| 14 | |
Jakub Kicinski | cc69837 | 2020-11-20 14:50:52 -0800 | [diff] [blame] | 15 | #include <linux/ethtool.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <linux/types.h> |
| 17 | #include <linux/device.h> |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 18 | #include <linux/dma-mapping.h> |
Michael S. Tsirkin | 459d6e2 | 2007-02-04 14:11:55 -0800 | [diff] [blame] | 19 | #include <linux/kref.h> |
Dotan Barak | bfb3ea1 | 2007-07-31 16:49:15 +0300 | [diff] [blame] | 20 | #include <linux/list.h> |
| 21 | #include <linux/rwsem.h> |
Tejun Heo | f062671 | 2010-10-19 15:24:36 +0000 | [diff] [blame] | 22 | #include <linux/workqueue.h> |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 23 | #include <linux/irq_poll.h> |
Matan Barak | dd5f03b | 2013-12-12 18:03:11 +0200 | [diff] [blame] | 24 | #include <uapi/linux/if_ether.h> |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 25 | #include <net/ipv6.h> |
| 26 | #include <net/ip.h> |
Matan Barak | 301a721 | 2015-12-15 20:30:10 +0200 | [diff] [blame] | 27 | #include <linux/string.h> |
| 28 | #include <linux/slab.h> |
Vishwanathapura, Niranjana | 2fc7757 | 2017-04-12 20:29:20 -0700 | [diff] [blame] | 29 | #include <linux/netdevice.h> |
Parav Pandit | 01b6711 | 2018-11-16 03:50:57 +0200 | [diff] [blame] | 30 | #include <linux/refcount.h> |
Eli Cohen | 50174a7 | 2016-03-11 22:58:38 +0200 | [diff] [blame] | 31 | #include <linux/if_link.h> |
Arun Sharma | 60063497 | 2011-07-26 16:09:06 -0700 | [diff] [blame] | 32 | #include <linux/atomic.h> |
Haggai Eran | 882214e | 2014-12-11 17:04:18 +0200 | [diff] [blame] | 33 | #include <linux/mmu_notifier.h> |
Linus Torvalds | 7c0f6ba | 2016-12-24 11:46:01 -0800 | [diff] [blame] | 34 | #include <linux/uaccess.h> |
Parav Pandit | 43579b5 | 2017-01-10 00:02:14 +0000 | [diff] [blame] | 35 | #include <linux/cgroup_rdma.h> |
Leon Romanovsky | f631603 | 2019-03-28 15:12:58 +0200 | [diff] [blame] | 36 | #include <linux/irqflags.h> |
| 37 | #include <linux/preempt.h> |
Yamin Friedman | da66297 | 2019-07-08 13:59:03 +0300 | [diff] [blame] | 38 | #include <linux/dim.h> |
Nicolas Dichtel | ea6819e | 2017-03-27 14:20:14 +0200 | [diff] [blame] | 39 | #include <uapi/rdma/ib_user_verbs.h> |
Mark Zhang | 413d334 | 2019-07-02 13:02:34 +0300 | [diff] [blame] | 40 | #include <rdma/rdma_counter.h> |
Leon Romanovsky | 02d8883 | 2018-01-28 11:17:20 +0200 | [diff] [blame] | 41 | #include <rdma/restrack.h> |
Max Gurtovoy | 36b1e47 | 2019-06-11 18:52:37 +0300 | [diff] [blame] | 42 | #include <rdma/signature.h> |
Matan Barak | 0ede73b | 2018-03-19 15:02:34 +0200 | [diff] [blame] | 43 | #include <uapi/rdma/rdma_user_ioctl.h> |
Matan Barak | 2eb9bea | 2018-03-28 09:27:45 +0300 | [diff] [blame] | 44 | #include <uapi/rdma/ib_user_ioctl_verbs.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | |
Leon Romanovsky | 9abb0d1 | 2017-06-27 16:49:53 +0300 | [diff] [blame] | 46 | #define IB_FW_VERSION_NAME_MAX ETHTOOL_FWVERS_LEN |
| 47 | |
Jason Gunthorpe | b5231b0 | 2018-09-16 20:48:04 +0300 | [diff] [blame] | 48 | struct ib_umem_odp; |
Jason Gunthorpe | 620d3f8 | 2020-01-08 19:21:59 +0200 | [diff] [blame] | 49 | struct ib_uqp_object; |
Jason Gunthorpe | 9fbe334 | 2020-01-08 19:22:00 +0200 | [diff] [blame] | 50 | struct ib_usrq_object; |
Jason Gunthorpe | e04dd13 | 2020-01-08 19:22:01 +0200 | [diff] [blame] | 51 | struct ib_uwq_object; |
Maor Gottlieb | 211cd94 | 2020-06-23 14:30:39 +0300 | [diff] [blame] | 52 | struct rdma_cm_id; |
Jason Gunthorpe | d8a5883 | 2021-06-11 19:00:21 +0300 | [diff] [blame] | 53 | struct ib_port; |
Jason Gunthorpe | 467f432 | 2021-06-11 19:00:22 +0300 | [diff] [blame] | 54 | struct hw_stats_device_data; |
Jason Gunthorpe | b5231b0 | 2018-09-16 20:48:04 +0300 | [diff] [blame] | 55 | |
Tejun Heo | f062671 | 2010-10-19 15:24:36 +0000 | [diff] [blame] | 56 | extern struct workqueue_struct *ib_wq; |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 57 | extern struct workqueue_struct *ib_comp_wq; |
Jack Morgenstein | f794809 | 2018-08-27 08:35:55 +0300 | [diff] [blame] | 58 | extern struct workqueue_struct *ib_comp_unbound_wq; |
Tejun Heo | f062671 | 2010-10-19 15:24:36 +0000 | [diff] [blame] | 59 | |
Jason Gunthorpe | 5bd48c1 | 2020-01-08 19:21:58 +0200 | [diff] [blame] | 60 | struct ib_ucq_object; |
| 61 | |
Gal Pressman | 923abb9 | 2019-05-01 13:48:13 +0300 | [diff] [blame] | 62 | __printf(3, 4) __cold |
| 63 | void ibdev_printk(const char *level, const struct ib_device *ibdev, |
| 64 | const char *format, ...); |
| 65 | __printf(2, 3) __cold |
| 66 | void ibdev_emerg(const struct ib_device *ibdev, const char *format, ...); |
| 67 | __printf(2, 3) __cold |
| 68 | void ibdev_alert(const struct ib_device *ibdev, const char *format, ...); |
| 69 | __printf(2, 3) __cold |
| 70 | void ibdev_crit(const struct ib_device *ibdev, const char *format, ...); |
| 71 | __printf(2, 3) __cold |
| 72 | void ibdev_err(const struct ib_device *ibdev, const char *format, ...); |
| 73 | __printf(2, 3) __cold |
| 74 | void ibdev_warn(const struct ib_device *ibdev, const char *format, ...); |
| 75 | __printf(2, 3) __cold |
| 76 | void ibdev_notice(const struct ib_device *ibdev, const char *format, ...); |
| 77 | __printf(2, 3) __cold |
| 78 | void ibdev_info(const struct ib_device *ibdev, const char *format, ...); |
| 79 | |
Orson Zhai | ceabef7 | 2020-06-07 21:40:14 -0700 | [diff] [blame] | 80 | #if defined(CONFIG_DYNAMIC_DEBUG) || \ |
| 81 | (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) |
Gal Pressman | 923abb9 | 2019-05-01 13:48:13 +0300 | [diff] [blame] | 82 | #define ibdev_dbg(__dev, format, args...) \ |
| 83 | dynamic_ibdev_dbg(__dev, format, ##args) |
Gal Pressman | 923abb9 | 2019-05-01 13:48:13 +0300 | [diff] [blame] | 84 | #else |
| 85 | __printf(2, 3) __cold |
| 86 | static inline |
| 87 | void ibdev_dbg(const struct ib_device *ibdev, const char *format, ...) {} |
| 88 | #endif |
| 89 | |
Gal Pressman | 05bb411 | 2019-08-01 20:14:46 +0300 | [diff] [blame] | 90 | #define ibdev_level_ratelimited(ibdev_level, ibdev, fmt, ...) \ |
| 91 | do { \ |
| 92 | static DEFINE_RATELIMIT_STATE(_rs, \ |
| 93 | DEFAULT_RATELIMIT_INTERVAL, \ |
| 94 | DEFAULT_RATELIMIT_BURST); \ |
| 95 | if (__ratelimit(&_rs)) \ |
| 96 | ibdev_level(ibdev, fmt, ##__VA_ARGS__); \ |
| 97 | } while (0) |
| 98 | |
| 99 | #define ibdev_emerg_ratelimited(ibdev, fmt, ...) \ |
| 100 | ibdev_level_ratelimited(ibdev_emerg, ibdev, fmt, ##__VA_ARGS__) |
| 101 | #define ibdev_alert_ratelimited(ibdev, fmt, ...) \ |
| 102 | ibdev_level_ratelimited(ibdev_alert, ibdev, fmt, ##__VA_ARGS__) |
| 103 | #define ibdev_crit_ratelimited(ibdev, fmt, ...) \ |
| 104 | ibdev_level_ratelimited(ibdev_crit, ibdev, fmt, ##__VA_ARGS__) |
| 105 | #define ibdev_err_ratelimited(ibdev, fmt, ...) \ |
| 106 | ibdev_level_ratelimited(ibdev_err, ibdev, fmt, ##__VA_ARGS__) |
| 107 | #define ibdev_warn_ratelimited(ibdev, fmt, ...) \ |
| 108 | ibdev_level_ratelimited(ibdev_warn, ibdev, fmt, ##__VA_ARGS__) |
| 109 | #define ibdev_notice_ratelimited(ibdev, fmt, ...) \ |
| 110 | ibdev_level_ratelimited(ibdev_notice, ibdev, fmt, ##__VA_ARGS__) |
| 111 | #define ibdev_info_ratelimited(ibdev, fmt, ...) \ |
| 112 | ibdev_level_ratelimited(ibdev_info, ibdev, fmt, ##__VA_ARGS__) |
| 113 | |
Orson Zhai | ceabef7 | 2020-06-07 21:40:14 -0700 | [diff] [blame] | 114 | #if defined(CONFIG_DYNAMIC_DEBUG) || \ |
| 115 | (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE)) |
Gal Pressman | 05bb411 | 2019-08-01 20:14:46 +0300 | [diff] [blame] | 116 | /* descriptor check is first to prevent flooding with "callbacks suppressed" */ |
| 117 | #define ibdev_dbg_ratelimited(ibdev, fmt, ...) \ |
| 118 | do { \ |
| 119 | static DEFINE_RATELIMIT_STATE(_rs, \ |
| 120 | DEFAULT_RATELIMIT_INTERVAL, \ |
| 121 | DEFAULT_RATELIMIT_BURST); \ |
| 122 | DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, fmt); \ |
| 123 | if (DYNAMIC_DEBUG_BRANCH(descriptor) && __ratelimit(&_rs)) \ |
| 124 | __dynamic_ibdev_dbg(&descriptor, ibdev, fmt, \ |
| 125 | ##__VA_ARGS__); \ |
| 126 | } while (0) |
| 127 | #else |
| 128 | __printf(2, 3) __cold |
| 129 | static inline |
| 130 | void ibdev_dbg_ratelimited(const struct ib_device *ibdev, const char *format, ...) {} |
| 131 | #endif |
| 132 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 133 | union ib_gid { |
| 134 | u8 raw[16]; |
| 135 | struct { |
Sean Hefty | 97f52eb | 2005-08-13 21:05:57 -0700 | [diff] [blame] | 136 | __be64 subnet_prefix; |
| 137 | __be64 interface_id; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 138 | } global; |
| 139 | }; |
| 140 | |
Moni Shoua | e26be1b | 2015-07-30 18:33:29 +0300 | [diff] [blame] | 141 | extern union ib_gid zgid; |
| 142 | |
Matan Barak | b39ffa1 | 2015-12-23 14:56:47 +0200 | [diff] [blame] | 143 | enum ib_gid_type { |
Avihai Horon | 9f85cbe | 2020-09-23 19:50:15 +0300 | [diff] [blame] | 144 | IB_GID_TYPE_IB = IB_UVERBS_GID_TYPE_IB, |
| 145 | IB_GID_TYPE_ROCE = IB_UVERBS_GID_TYPE_ROCE_V1, |
| 146 | IB_GID_TYPE_ROCE_UDP_ENCAP = IB_UVERBS_GID_TYPE_ROCE_V2, |
Matan Barak | b39ffa1 | 2015-12-23 14:56:47 +0200 | [diff] [blame] | 147 | IB_GID_TYPE_SIZE |
| 148 | }; |
| 149 | |
Moni Shoua | 7ead4bc | 2016-01-14 17:50:38 +0200 | [diff] [blame] | 150 | #define ROCE_V2_UDP_DPORT 4791 |
Matan Barak | 03db3a2 | 2015-07-30 18:33:26 +0300 | [diff] [blame] | 151 | struct ib_gid_attr { |
Parav Pandit | 943bd98 | 2019-05-02 10:48:07 +0300 | [diff] [blame] | 152 | struct net_device __rcu *ndev; |
Parav Pandit | 598ff6b | 2018-04-01 15:08:21 +0300 | [diff] [blame] | 153 | struct ib_device *device; |
Parav Pandit | b150c38 | 2018-06-05 08:40:15 +0300 | [diff] [blame] | 154 | union ib_gid gid; |
Parav Pandit | 598ff6b | 2018-04-01 15:08:21 +0300 | [diff] [blame] | 155 | enum ib_gid_type gid_type; |
| 156 | u16 index; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 157 | u32 port_num; |
Matan Barak | 03db3a2 | 2015-07-30 18:33:26 +0300 | [diff] [blame] | 158 | }; |
| 159 | |
Eli Cohen | a0c1b2a | 2016-03-11 22:58:37 +0200 | [diff] [blame] | 160 | enum { |
| 161 | /* set the local administered indication */ |
| 162 | IB_SA_WELL_KNOWN_GUID = BIT_ULL(57) | 2, |
| 163 | }; |
| 164 | |
Tom Tucker | 07ebafb | 2006-08-03 16:02:42 -0500 | [diff] [blame] | 165 | enum rdma_transport_type { |
| 166 | RDMA_TRANSPORT_IB, |
Upinder Malhi \(umalhi\) | 180771a | 2013-09-10 03:36:59 +0000 | [diff] [blame] | 167 | RDMA_TRANSPORT_IWARP, |
Upinder Malhi | 248567f | 2014-01-09 14:48:19 -0800 | [diff] [blame] | 168 | RDMA_TRANSPORT_USNIC, |
Gal Pressman | f95be3d | 2019-05-05 20:59:21 +0300 | [diff] [blame] | 169 | RDMA_TRANSPORT_USNIC_UDP, |
| 170 | RDMA_TRANSPORT_UNSPECIFIED, |
Tom Tucker | 07ebafb | 2006-08-03 16:02:42 -0500 | [diff] [blame] | 171 | }; |
| 172 | |
Michael Wang | 6b90a6d | 2015-05-05 14:50:18 +0200 | [diff] [blame] | 173 | enum rdma_protocol_type { |
| 174 | RDMA_PROTOCOL_IB, |
| 175 | RDMA_PROTOCOL_IBOE, |
| 176 | RDMA_PROTOCOL_IWARP, |
| 177 | RDMA_PROTOCOL_USNIC_UDP |
| 178 | }; |
| 179 | |
Roland Dreier | 8385fd8 | 2014-06-04 10:00:16 -0700 | [diff] [blame] | 180 | __attribute_const__ enum rdma_transport_type |
Jason Gunthorpe | 5d60c11 | 2019-06-13 21:38:17 -0300 | [diff] [blame] | 181 | rdma_node_get_transport(unsigned int node_type); |
Tom Tucker | 07ebafb | 2006-08-03 16:02:42 -0500 | [diff] [blame] | 182 | |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 183 | enum rdma_network_type { |
| 184 | RDMA_NETWORK_IB, |
Avihai Horon | 1c15b4f | 2020-09-23 19:50:13 +0300 | [diff] [blame] | 185 | RDMA_NETWORK_ROCE_V1, |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 186 | RDMA_NETWORK_IPV4, |
| 187 | RDMA_NETWORK_IPV6 |
| 188 | }; |
| 189 | |
| 190 | static inline enum ib_gid_type ib_network_to_gid_type(enum rdma_network_type network_type) |
| 191 | { |
| 192 | if (network_type == RDMA_NETWORK_IPV4 || |
| 193 | network_type == RDMA_NETWORK_IPV6) |
| 194 | return IB_GID_TYPE_ROCE_UDP_ENCAP; |
Avihai Horon | 1c15b4f | 2020-09-23 19:50:13 +0300 | [diff] [blame] | 195 | else if (network_type == RDMA_NETWORK_ROCE_V1) |
| 196 | return IB_GID_TYPE_ROCE; |
| 197 | else |
| 198 | return IB_GID_TYPE_IB; |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 199 | } |
| 200 | |
Parav Pandit | 47ec386 | 2018-06-13 10:22:06 +0300 | [diff] [blame] | 201 | static inline enum rdma_network_type |
| 202 | rdma_gid_attr_network_type(const struct ib_gid_attr *attr) |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 203 | { |
Parav Pandit | 47ec386 | 2018-06-13 10:22:06 +0300 | [diff] [blame] | 204 | if (attr->gid_type == IB_GID_TYPE_IB) |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 205 | return RDMA_NETWORK_IB; |
| 206 | |
Avihai Horon | 1c15b4f | 2020-09-23 19:50:13 +0300 | [diff] [blame] | 207 | if (attr->gid_type == IB_GID_TYPE_ROCE) |
| 208 | return RDMA_NETWORK_ROCE_V1; |
| 209 | |
Parav Pandit | 47ec386 | 2018-06-13 10:22:06 +0300 | [diff] [blame] | 210 | if (ipv6_addr_v4mapped((struct in6_addr *)&attr->gid)) |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 211 | return RDMA_NETWORK_IPV4; |
| 212 | else |
| 213 | return RDMA_NETWORK_IPV6; |
| 214 | } |
| 215 | |
Eli Cohen | a3f5ada | 2010-09-27 17:51:10 -0700 | [diff] [blame] | 216 | enum rdma_link_layer { |
| 217 | IB_LINK_LAYER_UNSPECIFIED, |
| 218 | IB_LINK_LAYER_INFINIBAND, |
| 219 | IB_LINK_LAYER_ETHERNET, |
| 220 | }; |
| 221 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 222 | enum ib_device_cap_flags { |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 223 | IB_DEVICE_RESIZE_MAX_WR = IB_UVERBS_DEVICE_RESIZE_MAX_WR, |
| 224 | IB_DEVICE_BAD_PKEY_CNTR = IB_UVERBS_DEVICE_BAD_PKEY_CNTR, |
| 225 | IB_DEVICE_BAD_QKEY_CNTR = IB_UVERBS_DEVICE_BAD_QKEY_CNTR, |
| 226 | IB_DEVICE_RAW_MULTI = IB_UVERBS_DEVICE_RAW_MULTI, |
| 227 | IB_DEVICE_AUTO_PATH_MIG = IB_UVERBS_DEVICE_AUTO_PATH_MIG, |
| 228 | IB_DEVICE_CHANGE_PHY_PORT = IB_UVERBS_DEVICE_CHANGE_PHY_PORT, |
| 229 | IB_DEVICE_UD_AV_PORT_ENFORCE = IB_UVERBS_DEVICE_UD_AV_PORT_ENFORCE, |
| 230 | IB_DEVICE_CURR_QP_STATE_MOD = IB_UVERBS_DEVICE_CURR_QP_STATE_MOD, |
| 231 | IB_DEVICE_SHUTDOWN_PORT = IB_UVERBS_DEVICE_SHUTDOWN_PORT, |
| 232 | /* IB_DEVICE_INIT_TYPE = IB_UVERBS_DEVICE_INIT_TYPE, (not in use) */ |
| 233 | IB_DEVICE_PORT_ACTIVE_EVENT = IB_UVERBS_DEVICE_PORT_ACTIVE_EVENT, |
| 234 | IB_DEVICE_SYS_IMAGE_GUID = IB_UVERBS_DEVICE_SYS_IMAGE_GUID, |
| 235 | IB_DEVICE_RC_RNR_NAK_GEN = IB_UVERBS_DEVICE_RC_RNR_NAK_GEN, |
| 236 | IB_DEVICE_SRQ_RESIZE = IB_UVERBS_DEVICE_SRQ_RESIZE, |
| 237 | IB_DEVICE_N_NOTIFY_CQ = IB_UVERBS_DEVICE_N_NOTIFY_CQ, |
Christoph Hellwig | b1adc71 | 2015-12-23 19:12:45 +0100 | [diff] [blame] | 238 | |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 239 | /* Reserved, old SEND_W_INV = 1 << 16,*/ |
| 240 | IB_DEVICE_MEM_WINDOW = IB_UVERBS_DEVICE_MEM_WINDOW, |
Eli Cohen | e0605d9 | 2008-01-30 18:30:57 +0200 | [diff] [blame] | 241 | /* |
| 242 | * Devices should set IB_DEVICE_UD_IP_SUM if they support |
| 243 | * insertion of UDP and TCP checksum on outgoing UD IPoIB |
| 244 | * messages and can verify the validity of checksum for |
| 245 | * incoming messages. Setting this flag implies that the |
| 246 | * IPoIB driver may set NETIF_F_IP_CSUM for datagram mode. |
| 247 | */ |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 248 | IB_DEVICE_UD_IP_CSUM = IB_UVERBS_DEVICE_UD_IP_CSUM, |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 249 | IB_DEVICE_XRC = IB_UVERBS_DEVICE_XRC, |
Christoph Hellwig | b1adc71 | 2015-12-23 19:12:45 +0100 | [diff] [blame] | 250 | |
| 251 | /* |
| 252 | * This device supports the IB "base memory management extension", |
| 253 | * which includes support for fast registrations (IB_WR_REG_MR, |
| 254 | * IB_WR_LOCAL_INV and IB_WR_SEND_WITH_INV verbs). This flag should |
| 255 | * also be set by any iWarp device which must support FRs to comply |
| 256 | * to the iWarp verbs spec. iWarp devices also support the |
| 257 | * IB_WR_RDMA_READ_WITH_INV verb for RDMA READs that invalidate the |
| 258 | * stag. |
| 259 | */ |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 260 | IB_DEVICE_MEM_MGT_EXTENSIONS = IB_UVERBS_DEVICE_MEM_MGT_EXTENSIONS, |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 261 | IB_DEVICE_MEM_WINDOW_TYPE_2A = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2A, |
| 262 | IB_DEVICE_MEM_WINDOW_TYPE_2B = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2B, |
| 263 | IB_DEVICE_RC_IP_CSUM = IB_UVERBS_DEVICE_RC_IP_CSUM, |
Noa Osherovich | ebaaee2 | 2017-01-18 15:39:54 +0200 | [diff] [blame] | 264 | /* Deprecated. Please use IB_RAW_PACKET_CAP_IP_CSUM. */ |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 265 | IB_DEVICE_RAW_IP_CSUM = IB_UVERBS_DEVICE_RAW_IP_CSUM, |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 266 | IB_DEVICE_MANAGED_FLOW_STEERING = |
| 267 | IB_UVERBS_DEVICE_MANAGED_FLOW_STEERING, |
Noa Osherovich | ebaaee2 | 2017-01-18 15:39:54 +0200 | [diff] [blame] | 268 | /* Deprecated. Please use IB_RAW_PACKET_CAP_SCATTER_FCS. */ |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 269 | IB_DEVICE_RAW_SCATTER_FCS = IB_UVERBS_DEVICE_RAW_SCATTER_FCS, |
Noa Osherovich | e1d2e88 | 2017-10-29 13:59:44 +0200 | [diff] [blame] | 270 | /* The device supports padding incoming writes to cacheline. */ |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 271 | IB_DEVICE_PCI_WRITE_END_PADDING = |
| 272 | IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING, |
Li Zhijian | 208e3a1 | 2022-12-06 21:01:53 +0800 | [diff] [blame] | 273 | /* Placement type attributes */ |
| 274 | IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL, |
| 275 | IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT, |
Xiao Yang | 3ff81e8 | 2022-12-01 14:37:06 +0000 | [diff] [blame] | 276 | IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE, |
Sagi Grimberg | 1b01d33 | 2014-02-23 14:19:05 +0200 | [diff] [blame] | 277 | }; |
| 278 | |
Jason Gunthorpe | e945c65 | 2022-04-04 12:26:42 -0300 | [diff] [blame] | 279 | enum ib_kernel_cap_flags { |
| 280 | /* |
| 281 | * This device supports a per-device lkey or stag that can be |
| 282 | * used without performing a memory registration for the local |
| 283 | * memory. Note that ULPs should never check this flag, but |
| 284 | * instead of use the local_dma_lkey flag in the ib_pd structure, |
| 285 | * which will always contain a usable lkey. |
| 286 | */ |
| 287 | IBK_LOCAL_DMA_LKEY = 1 << 0, |
| 288 | /* IB_QP_CREATE_INTEGRITY_EN is supported to implement T10-PI */ |
| 289 | IBK_INTEGRITY_HANDOVER = 1 << 1, |
| 290 | /* IB_ACCESS_ON_DEMAND is supported during reg_user_mr() */ |
| 291 | IBK_ON_DEMAND_PAGING = 1 << 2, |
| 292 | /* IB_MR_TYPE_SG_GAPS is supported */ |
| 293 | IBK_SG_GAPS_REG = 1 << 3, |
| 294 | /* Driver supports RDMA_NLDEV_CMD_DELLINK */ |
| 295 | IBK_ALLOW_USER_UNREG = 1 << 4, |
| 296 | |
| 297 | /* ipoib will use IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK */ |
| 298 | IBK_BLOCK_MULTICAST_LOOPBACK = 1 << 5, |
| 299 | /* iopib will use IB_QP_CREATE_IPOIB_UD_LSO for its QPs */ |
| 300 | IBK_UD_TSO = 1 << 6, |
| 301 | /* iopib will use the device ops: |
| 302 | * get_vf_config |
| 303 | * get_vf_guid |
| 304 | * get_vf_stats |
| 305 | * set_vf_guid |
| 306 | * set_vf_link_state |
| 307 | */ |
| 308 | IBK_VIRTUAL_FUNCTION = 1 << 7, |
| 309 | /* ipoib will use IB_QP_CREATE_NETDEV_USE for its QPs */ |
| 310 | IBK_RDMA_NETDEV_OPA = 1 << 8, |
| 311 | }; |
Xiao Yang | f543a3e | 2022-03-31 11:24:19 +0800 | [diff] [blame] | 312 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 313 | enum ib_atomic_cap { |
| 314 | IB_ATOMIC_NONE, |
| 315 | IB_ATOMIC_HCA, |
| 316 | IB_ATOMIC_GLOB |
| 317 | }; |
| 318 | |
Sagi Grimberg | 860f10a | 2014-12-11 17:04:16 +0200 | [diff] [blame] | 319 | enum ib_odp_general_cap_bits { |
Artemy Kovalyov | 25bf14d | 2017-01-18 16:58:06 +0200 | [diff] [blame] | 320 | IB_ODP_SUPPORT = 1 << 0, |
| 321 | IB_ODP_SUPPORT_IMPLICIT = 1 << 1, |
Sagi Grimberg | 860f10a | 2014-12-11 17:04:16 +0200 | [diff] [blame] | 322 | }; |
| 323 | |
| 324 | enum ib_odp_transport_cap_bits { |
| 325 | IB_ODP_SUPPORT_SEND = 1 << 0, |
| 326 | IB_ODP_SUPPORT_RECV = 1 << 1, |
| 327 | IB_ODP_SUPPORT_WRITE = 1 << 2, |
| 328 | IB_ODP_SUPPORT_READ = 1 << 3, |
| 329 | IB_ODP_SUPPORT_ATOMIC = 1 << 4, |
Moni Shoua | da82334 | 2019-01-22 08:48:41 +0200 | [diff] [blame] | 330 | IB_ODP_SUPPORT_SRQ_RECV = 1 << 5, |
Sagi Grimberg | 860f10a | 2014-12-11 17:04:16 +0200 | [diff] [blame] | 331 | }; |
| 332 | |
| 333 | struct ib_odp_caps { |
| 334 | uint64_t general_caps; |
| 335 | struct { |
| 336 | uint32_t rc_odp_caps; |
| 337 | uint32_t uc_odp_caps; |
| 338 | uint32_t ud_odp_caps; |
Moni Shoua | 52a72e2 | 2019-01-22 08:48:42 +0200 | [diff] [blame] | 339 | uint32_t xrc_odp_caps; |
Sagi Grimberg | 860f10a | 2014-12-11 17:04:16 +0200 | [diff] [blame] | 340 | } per_transport_caps; |
| 341 | }; |
| 342 | |
Yishai Hadas | ccf2056 | 2016-08-28 11:28:43 +0300 | [diff] [blame] | 343 | struct ib_rss_caps { |
| 344 | /* Corresponding bit will be set if qp type from |
| 345 | * 'enum ib_qp_type' is supported, e.g. |
| 346 | * supported_qpts |= 1 << IB_QPT_UD |
| 347 | */ |
| 348 | u32 supported_qpts; |
| 349 | u32 max_rwq_indirection_tables; |
| 350 | u32 max_rwq_indirection_table_size; |
| 351 | }; |
| 352 | |
Artemy Kovalyov | 6938fc1e | 2017-08-17 15:52:03 +0300 | [diff] [blame] | 353 | enum ib_tm_cap_flags { |
Danit Goldberg | 89705e9 | 2019-07-05 19:21:57 +0300 | [diff] [blame] | 354 | /* Support tag matching with rendezvous offload for RC transport */ |
| 355 | IB_TM_CAP_RNDV_RC = 1 << 0, |
Artemy Kovalyov | 6938fc1e | 2017-08-17 15:52:03 +0300 | [diff] [blame] | 356 | }; |
| 357 | |
Leon Romanovsky | 78b1beb | 2017-09-24 21:46:29 +0300 | [diff] [blame] | 358 | struct ib_tm_caps { |
Artemy Kovalyov | 6938fc1e | 2017-08-17 15:52:03 +0300 | [diff] [blame] | 359 | /* Max size of RNDV header */ |
| 360 | u32 max_rndv_hdr_size; |
| 361 | /* Max number of entries in tag matching list */ |
| 362 | u32 max_num_tags; |
| 363 | /* From enum ib_tm_cap_flags */ |
| 364 | u32 flags; |
| 365 | /* Max number of outstanding list operations */ |
| 366 | u32 max_ops; |
| 367 | /* Max number of SGE in tag matching entry */ |
| 368 | u32 max_sge; |
| 369 | }; |
| 370 | |
Matan Barak | bcf4c1e | 2015-06-11 16:35:20 +0300 | [diff] [blame] | 371 | struct ib_cq_init_attr { |
| 372 | unsigned int cqe; |
Dan Carpenter | a9018ad | 2019-10-11 16:34:19 +0300 | [diff] [blame] | 373 | u32 comp_vector; |
Matan Barak | bcf4c1e | 2015-06-11 16:35:20 +0300 | [diff] [blame] | 374 | u32 flags; |
| 375 | }; |
| 376 | |
Yonatan Cohen | 869ddcf | 2017-11-13 10:51:13 +0200 | [diff] [blame] | 377 | enum ib_cq_attr_mask { |
| 378 | IB_CQ_MODERATE = 1 << 0, |
| 379 | }; |
| 380 | |
Yonatan Cohen | 18bd907 | 2017-11-13 10:51:16 +0200 | [diff] [blame] | 381 | struct ib_cq_caps { |
| 382 | u16 max_cq_moderation_count; |
| 383 | u16 max_cq_moderation_period; |
| 384 | }; |
| 385 | |
Ariel Levkovich | be934cc | 2018-04-05 18:53:25 +0300 | [diff] [blame] | 386 | struct ib_dm_mr_attr { |
| 387 | u64 length; |
| 388 | u64 offset; |
| 389 | u32 access_flags; |
| 390 | }; |
| 391 | |
Ariel Levkovich | bee76d7 | 2018-04-05 18:53:24 +0300 | [diff] [blame] | 392 | struct ib_dm_alloc_attr { |
| 393 | u64 length; |
| 394 | u32 alignment; |
| 395 | u32 flags; |
| 396 | }; |
| 397 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 398 | struct ib_device_attr { |
| 399 | u64 fw_ver; |
Sean Hefty | 97f52eb | 2005-08-13 21:05:57 -0700 | [diff] [blame] | 400 | __be64 sys_image_guid; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 401 | u64 max_mr_size; |
| 402 | u64 page_size_cap; |
| 403 | u32 vendor_id; |
| 404 | u32 vendor_part_id; |
| 405 | u32 hw_ver; |
| 406 | int max_qp; |
| 407 | int max_qp_wr; |
Leon Romanovsky | fb532d6 | 2016-02-23 10:25:25 +0200 | [diff] [blame] | 408 | u64 device_cap_flags; |
Jason Gunthorpe | e945c65 | 2022-04-04 12:26:42 -0300 | [diff] [blame] | 409 | u64 kernel_cap_flags; |
Steve Wise | 33023fb | 2018-06-18 08:05:26 -0700 | [diff] [blame] | 410 | int max_send_sge; |
| 411 | int max_recv_sge; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 412 | int max_sge_rd; |
| 413 | int max_cq; |
| 414 | int max_cqe; |
| 415 | int max_mr; |
| 416 | int max_pd; |
| 417 | int max_qp_rd_atom; |
| 418 | int max_ee_rd_atom; |
| 419 | int max_res_rd_atom; |
| 420 | int max_qp_init_rd_atom; |
| 421 | int max_ee_init_rd_atom; |
| 422 | enum ib_atomic_cap atomic_cap; |
Vladimir Sokolovsky | 5e80ba8 | 2010-04-14 17:23:01 +0300 | [diff] [blame] | 423 | enum ib_atomic_cap masked_atomic_cap; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 424 | int max_ee; |
| 425 | int max_rdd; |
| 426 | int max_mw; |
| 427 | int max_raw_ipv6_qp; |
| 428 | int max_raw_ethy_qp; |
| 429 | int max_mcast_grp; |
| 430 | int max_mcast_qp_attach; |
| 431 | int max_total_mcast_qp_attach; |
| 432 | int max_ah; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 433 | int max_srq; |
| 434 | int max_srq_wr; |
| 435 | int max_srq_sge; |
Steve Wise | 00f7ec3 | 2008-07-14 23:48:45 -0700 | [diff] [blame] | 436 | unsigned int max_fast_reg_page_list_len; |
Max Gurtovoy | 62e3c37 | 2019-06-11 18:52:43 +0300 | [diff] [blame] | 437 | unsigned int max_pi_fast_reg_page_list_len; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 438 | u16 max_pkeys; |
| 439 | u8 local_ca_ack_delay; |
Sagi Grimberg | 1b01d33 | 2014-02-23 14:19:05 +0200 | [diff] [blame] | 440 | int sig_prot_cap; |
| 441 | int sig_guard_cap; |
Sagi Grimberg | 860f10a | 2014-12-11 17:04:16 +0200 | [diff] [blame] | 442 | struct ib_odp_caps odp_caps; |
Matan Barak | 24306dc | 2015-06-11 16:35:24 +0300 | [diff] [blame] | 443 | uint64_t timestamp_mask; |
| 444 | uint64_t hca_core_clock; /* in KHZ */ |
Yishai Hadas | ccf2056 | 2016-08-28 11:28:43 +0300 | [diff] [blame] | 445 | struct ib_rss_caps rss_caps; |
| 446 | u32 max_wq_type_rq; |
Noa Osherovich | ebaaee2 | 2017-01-18 15:39:54 +0200 | [diff] [blame] | 447 | u32 raw_packet_caps; /* Use ib_raw_packet_caps enum */ |
Leon Romanovsky | 78b1beb | 2017-09-24 21:46:29 +0300 | [diff] [blame] | 448 | struct ib_tm_caps tm_caps; |
Yonatan Cohen | 18bd907 | 2017-11-13 10:51:16 +0200 | [diff] [blame] | 449 | struct ib_cq_caps cq_caps; |
Ariel Levkovich | 1d8eeb9 | 2018-04-05 18:53:23 +0300 | [diff] [blame] | 450 | u64 max_dm_size; |
Yamin Friedman | 00bd143 | 2019-10-07 16:59:32 +0300 | [diff] [blame] | 451 | /* Max entries for sgl for optimized performance per READ */ |
| 452 | u32 max_sgl_rd; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 453 | }; |
| 454 | |
| 455 | enum ib_mtu { |
| 456 | IB_MTU_256 = 1, |
| 457 | IB_MTU_512 = 2, |
| 458 | IB_MTU_1024 = 3, |
| 459 | IB_MTU_2048 = 4, |
| 460 | IB_MTU_4096 = 5 |
| 461 | }; |
| 462 | |
Kaike Wan | 6d72344 | 2020-05-11 12:06:18 -0400 | [diff] [blame] | 463 | enum opa_mtu { |
| 464 | OPA_MTU_8192 = 6, |
| 465 | OPA_MTU_10240 = 7 |
| 466 | }; |
| 467 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 468 | static inline int ib_mtu_enum_to_int(enum ib_mtu mtu) |
| 469 | { |
| 470 | switch (mtu) { |
| 471 | case IB_MTU_256: return 256; |
| 472 | case IB_MTU_512: return 512; |
| 473 | case IB_MTU_1024: return 1024; |
| 474 | case IB_MTU_2048: return 2048; |
| 475 | case IB_MTU_4096: return 4096; |
| 476 | default: return -1; |
| 477 | } |
| 478 | } |
| 479 | |
Amrani, Ram | d3f4aad | 2016-12-26 08:40:57 +0200 | [diff] [blame] | 480 | static inline enum ib_mtu ib_mtu_int_to_enum(int mtu) |
| 481 | { |
| 482 | if (mtu >= 4096) |
| 483 | return IB_MTU_4096; |
| 484 | else if (mtu >= 2048) |
| 485 | return IB_MTU_2048; |
| 486 | else if (mtu >= 1024) |
| 487 | return IB_MTU_1024; |
| 488 | else if (mtu >= 512) |
| 489 | return IB_MTU_512; |
| 490 | else |
| 491 | return IB_MTU_256; |
| 492 | } |
| 493 | |
Kaike Wan | 6d72344 | 2020-05-11 12:06:18 -0400 | [diff] [blame] | 494 | static inline int opa_mtu_enum_to_int(enum opa_mtu mtu) |
| 495 | { |
| 496 | switch (mtu) { |
| 497 | case OPA_MTU_8192: |
| 498 | return 8192; |
| 499 | case OPA_MTU_10240: |
| 500 | return 10240; |
| 501 | default: |
| 502 | return(ib_mtu_enum_to_int((enum ib_mtu)mtu)); |
| 503 | } |
| 504 | } |
| 505 | |
| 506 | static inline enum opa_mtu opa_mtu_int_to_enum(int mtu) |
| 507 | { |
| 508 | if (mtu >= 10240) |
| 509 | return OPA_MTU_10240; |
| 510 | else if (mtu >= 8192) |
| 511 | return OPA_MTU_8192; |
| 512 | else |
| 513 | return ((enum opa_mtu)ib_mtu_int_to_enum(mtu)); |
| 514 | } |
| 515 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 516 | enum ib_port_state { |
| 517 | IB_PORT_NOP = 0, |
| 518 | IB_PORT_DOWN = 1, |
| 519 | IB_PORT_INIT = 2, |
| 520 | IB_PORT_ARMED = 3, |
| 521 | IB_PORT_ACTIVE = 4, |
| 522 | IB_PORT_ACTIVE_DEFER = 5 |
| 523 | }; |
| 524 | |
Kamal Heib | 72a7720 | 2019-08-07 13:31:35 +0300 | [diff] [blame] | 525 | enum ib_port_phys_state { |
| 526 | IB_PORT_PHYS_STATE_SLEEP = 1, |
| 527 | IB_PORT_PHYS_STATE_POLLING = 2, |
| 528 | IB_PORT_PHYS_STATE_DISABLED = 3, |
| 529 | IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING = 4, |
| 530 | IB_PORT_PHYS_STATE_LINK_UP = 5, |
| 531 | IB_PORT_PHYS_STATE_LINK_ERROR_RECOVERY = 6, |
| 532 | IB_PORT_PHYS_STATE_PHY_TEST = 7, |
| 533 | }; |
| 534 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 535 | enum ib_port_width { |
| 536 | IB_WIDTH_1X = 1, |
Michael Guralnik | dbabf68 | 2018-12-09 11:49:49 +0200 | [diff] [blame] | 537 | IB_WIDTH_2X = 16, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 538 | IB_WIDTH_4X = 2, |
| 539 | IB_WIDTH_8X = 4, |
| 540 | IB_WIDTH_12X = 8 |
| 541 | }; |
| 542 | |
| 543 | static inline int ib_width_enum_to_int(enum ib_port_width width) |
| 544 | { |
| 545 | switch (width) { |
| 546 | case IB_WIDTH_1X: return 1; |
Michael Guralnik | dbabf68 | 2018-12-09 11:49:49 +0200 | [diff] [blame] | 547 | case IB_WIDTH_2X: return 2; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 548 | case IB_WIDTH_4X: return 4; |
| 549 | case IB_WIDTH_8X: return 8; |
| 550 | case IB_WIDTH_12X: return 12; |
| 551 | default: return -1; |
| 552 | } |
| 553 | } |
| 554 | |
Or Gerlitz | 2e96691 | 2012-02-28 18:49:50 +0200 | [diff] [blame] | 555 | enum ib_port_speed { |
| 556 | IB_SPEED_SDR = 1, |
| 557 | IB_SPEED_DDR = 2, |
| 558 | IB_SPEED_QDR = 4, |
| 559 | IB_SPEED_FDR10 = 8, |
| 560 | IB_SPEED_FDR = 16, |
Noa Osherovich | 12113a3 | 2017-04-20 20:53:31 +0300 | [diff] [blame] | 561 | IB_SPEED_EDR = 32, |
Aharon Landau | 376ceb3 | 2020-09-17 12:02:23 +0300 | [diff] [blame] | 562 | IB_SPEED_HDR = 64, |
| 563 | IB_SPEED_NDR = 128, |
Or Gerlitz | 2e96691 | 2012-02-28 18:49:50 +0200 | [diff] [blame] | 564 | }; |
| 565 | |
Aharon Landau | 5e2ddd1 | 2021-10-08 15:24:32 +0300 | [diff] [blame] | 566 | enum ib_stat_flag { |
| 567 | IB_STAT_FLAG_OPTIONAL = 1 << 0, |
| 568 | }; |
| 569 | |
Christoph Lameter | b40f475 | 2016-05-16 12:49:33 -0500 | [diff] [blame] | 570 | /** |
Aharon Landau | 13f30b0 | 2021-10-08 15:24:29 +0300 | [diff] [blame] | 571 | * struct rdma_stat_desc |
| 572 | * @name - The name of the counter |
Aharon Landau | 5e2ddd1 | 2021-10-08 15:24:32 +0300 | [diff] [blame] | 573 | * @flags - Flags of the counter; For example, IB_STAT_FLAG_OPTIONAL |
Aharon Landau | a29b934 | 2021-10-08 15:24:38 +0300 | [diff] [blame] | 574 | * @priv - Driver private information; Core code should not use |
Aharon Landau | 13f30b0 | 2021-10-08 15:24:29 +0300 | [diff] [blame] | 575 | */ |
| 576 | struct rdma_stat_desc { |
| 577 | const char *name; |
Aharon Landau | 5e2ddd1 | 2021-10-08 15:24:32 +0300 | [diff] [blame] | 578 | unsigned int flags; |
Aharon Landau | a29b934 | 2021-10-08 15:24:38 +0300 | [diff] [blame] | 579 | const void *priv; |
Aharon Landau | 13f30b0 | 2021-10-08 15:24:29 +0300 | [diff] [blame] | 580 | }; |
| 581 | |
| 582 | /** |
Christoph Lameter | b40f475 | 2016-05-16 12:49:33 -0500 | [diff] [blame] | 583 | * struct rdma_hw_stats |
Mark Bloch | e945130 | 2018-03-27 15:51:05 +0300 | [diff] [blame] | 584 | * @lock - Mutex to protect parallel write access to lifespan and values |
Julia Lawall | 83567ce | 2022-05-21 13:11:36 +0200 | [diff] [blame] | 585 | * of counters, which are 64bits and not guaranteed to be written |
Mark Bloch | e945130 | 2018-03-27 15:51:05 +0300 | [diff] [blame] | 586 | * atomicaly on 32bits systems. |
Christoph Lameter | b40f475 | 2016-05-16 12:49:33 -0500 | [diff] [blame] | 587 | * @timestamp - Used by the core code to track when the last update was |
| 588 | * @lifespan - Used by the core code to determine how old the counters |
| 589 | * should be before being updated again. Stored in jiffies, defaults |
| 590 | * to 10 milliseconds, drivers can override the default be specifying |
| 591 | * their own value during their allocation routine. |
Aharon Landau | 13f30b0 | 2021-10-08 15:24:29 +0300 | [diff] [blame] | 592 | * @descs - Array of pointers to static descriptors used for the counters |
| 593 | * in directory. |
Aharon Landau | 0dc8968 | 2021-10-08 15:24:31 +0300 | [diff] [blame] | 594 | * @is_disabled - A bitmap to indicate each counter is currently disabled |
| 595 | * or not. |
Christoph Lameter | b40f475 | 2016-05-16 12:49:33 -0500 | [diff] [blame] | 596 | * @num_counters - How many hardware counters there are. If name is |
| 597 | * shorter than this number, a kernel oops will result. Driver authors |
| 598 | * are encouraged to leave BUILD_BUG_ON(ARRAY_SIZE(@name) < num_counters) |
| 599 | * in their code to prevent this. |
| 600 | * @value - Array of u64 counters that are accessed by the sysfs code and |
| 601 | * filled in by the drivers get_stats routine |
| 602 | */ |
| 603 | struct rdma_hw_stats { |
Mark Bloch | e945130 | 2018-03-27 15:51:05 +0300 | [diff] [blame] | 604 | struct mutex lock; /* Protect lifespan and values[] */ |
Christoph Lameter | b40f475 | 2016-05-16 12:49:33 -0500 | [diff] [blame] | 605 | unsigned long timestamp; |
| 606 | unsigned long lifespan; |
Aharon Landau | 13f30b0 | 2021-10-08 15:24:29 +0300 | [diff] [blame] | 607 | const struct rdma_stat_desc *descs; |
Aharon Landau | 0dc8968 | 2021-10-08 15:24:31 +0300 | [diff] [blame] | 608 | unsigned long *is_disabled; |
Christoph Lameter | b40f475 | 2016-05-16 12:49:33 -0500 | [diff] [blame] | 609 | int num_counters; |
| 610 | u64 value[]; |
Steve Wise | 7f624d0 | 2008-07-14 23:48:48 -0700 | [diff] [blame] | 611 | }; |
| 612 | |
Christoph Lameter | b40f475 | 2016-05-16 12:49:33 -0500 | [diff] [blame] | 613 | #define RDMA_HW_STATS_DEFAULT_LIFESPAN 10 |
Steve Wise | 7f624d0 | 2008-07-14 23:48:48 -0700 | [diff] [blame] | 614 | |
Mark Zhang | 0a0800c | 2021-10-08 15:24:30 +0300 | [diff] [blame] | 615 | struct rdma_hw_stats *rdma_alloc_hw_stats_struct( |
| 616 | const struct rdma_stat_desc *descs, int num_counters, |
| 617 | unsigned long lifespan); |
Aharon Landau | 13f30b0 | 2021-10-08 15:24:29 +0300 | [diff] [blame] | 618 | |
Mark Zhang | 0a0800c | 2021-10-08 15:24:30 +0300 | [diff] [blame] | 619 | void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats); |
Steve Wise | 7f624d0 | 2008-07-14 23:48:48 -0700 | [diff] [blame] | 620 | |
Ira Weiny | f9b22e3 | 2015-05-13 20:02:59 -0400 | [diff] [blame] | 621 | /* Define bits for the various functionality this port needs to be supported by |
| 622 | * the core. |
| 623 | */ |
| 624 | /* Management 0x00000FFF */ |
| 625 | #define RDMA_CORE_CAP_IB_MAD 0x00000001 |
| 626 | #define RDMA_CORE_CAP_IB_SMI 0x00000002 |
| 627 | #define RDMA_CORE_CAP_IB_CM 0x00000004 |
| 628 | #define RDMA_CORE_CAP_IW_CM 0x00000008 |
| 629 | #define RDMA_CORE_CAP_IB_SA 0x00000010 |
Ira Weiny | 65995fe | 2015-06-06 14:38:32 -0400 | [diff] [blame] | 630 | #define RDMA_CORE_CAP_OPA_MAD 0x00000020 |
Ira Weiny | f9b22e3 | 2015-05-13 20:02:59 -0400 | [diff] [blame] | 631 | |
| 632 | /* Address format 0x000FF000 */ |
| 633 | #define RDMA_CORE_CAP_AF_IB 0x00001000 |
| 634 | #define RDMA_CORE_CAP_ETH_AH 0x00002000 |
Dasaratharaman Chandramouli | 94d595c | 2017-03-20 19:38:09 -0400 | [diff] [blame] | 635 | #define RDMA_CORE_CAP_OPA_AH 0x00004000 |
Artemy Kovalyov | b02289b | 2018-07-04 15:57:50 +0300 | [diff] [blame] | 636 | #define RDMA_CORE_CAP_IB_GRH_REQUIRED 0x00008000 |
Ira Weiny | f9b22e3 | 2015-05-13 20:02:59 -0400 | [diff] [blame] | 637 | |
| 638 | /* Protocol 0xFFF00000 */ |
| 639 | #define RDMA_CORE_CAP_PROT_IB 0x00100000 |
| 640 | #define RDMA_CORE_CAP_PROT_ROCE 0x00200000 |
| 641 | #define RDMA_CORE_CAP_PROT_IWARP 0x00400000 |
Matan Barak | 7766a99 | 2015-12-23 14:56:50 +0200 | [diff] [blame] | 642 | #define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000 |
Or Gerlitz | aa773bd | 2017-01-24 13:02:35 +0200 | [diff] [blame] | 643 | #define RDMA_CORE_CAP_PROT_RAW_PACKET 0x01000000 |
Or Gerlitz | ce1e055 | 2017-01-24 13:02:38 +0200 | [diff] [blame] | 644 | #define RDMA_CORE_CAP_PROT_USNIC 0x02000000 |
Ira Weiny | f9b22e3 | 2015-05-13 20:02:59 -0400 | [diff] [blame] | 645 | |
Artemy Kovalyov | b02289b | 2018-07-04 15:57:50 +0300 | [diff] [blame] | 646 | #define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \ |
| 647 | | RDMA_CORE_CAP_PROT_ROCE \ |
| 648 | | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP) |
| 649 | |
Ira Weiny | f9b22e3 | 2015-05-13 20:02:59 -0400 | [diff] [blame] | 650 | #define RDMA_CORE_PORT_IBA_IB (RDMA_CORE_CAP_PROT_IB \ |
| 651 | | RDMA_CORE_CAP_IB_MAD \ |
| 652 | | RDMA_CORE_CAP_IB_SMI \ |
| 653 | | RDMA_CORE_CAP_IB_CM \ |
| 654 | | RDMA_CORE_CAP_IB_SA \ |
| 655 | | RDMA_CORE_CAP_AF_IB) |
| 656 | #define RDMA_CORE_PORT_IBA_ROCE (RDMA_CORE_CAP_PROT_ROCE \ |
| 657 | | RDMA_CORE_CAP_IB_MAD \ |
| 658 | | RDMA_CORE_CAP_IB_CM \ |
Ira Weiny | f9b22e3 | 2015-05-13 20:02:59 -0400 | [diff] [blame] | 659 | | RDMA_CORE_CAP_AF_IB \ |
| 660 | | RDMA_CORE_CAP_ETH_AH) |
Matan Barak | 7766a99 | 2015-12-23 14:56:50 +0200 | [diff] [blame] | 661 | #define RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP \ |
| 662 | (RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP \ |
| 663 | | RDMA_CORE_CAP_IB_MAD \ |
| 664 | | RDMA_CORE_CAP_IB_CM \ |
| 665 | | RDMA_CORE_CAP_AF_IB \ |
| 666 | | RDMA_CORE_CAP_ETH_AH) |
Ira Weiny | f9b22e3 | 2015-05-13 20:02:59 -0400 | [diff] [blame] | 667 | #define RDMA_CORE_PORT_IWARP (RDMA_CORE_CAP_PROT_IWARP \ |
| 668 | | RDMA_CORE_CAP_IW_CM) |
Ira Weiny | 65995fe | 2015-06-06 14:38:32 -0400 | [diff] [blame] | 669 | #define RDMA_CORE_PORT_INTEL_OPA (RDMA_CORE_PORT_IBA_IB \ |
| 670 | | RDMA_CORE_CAP_OPA_MAD) |
Ira Weiny | f9b22e3 | 2015-05-13 20:02:59 -0400 | [diff] [blame] | 671 | |
Or Gerlitz | aa773bd | 2017-01-24 13:02:35 +0200 | [diff] [blame] | 672 | #define RDMA_CORE_PORT_RAW_PACKET (RDMA_CORE_CAP_PROT_RAW_PACKET) |
| 673 | |
Or Gerlitz | ce1e055 | 2017-01-24 13:02:38 +0200 | [diff] [blame] | 674 | #define RDMA_CORE_PORT_USNIC (RDMA_CORE_CAP_PROT_USNIC) |
| 675 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 676 | struct ib_port_attr { |
Eli Cohen | fad61ad | 2016-03-11 22:58:36 +0200 | [diff] [blame] | 677 | u64 subnet_prefix; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 678 | enum ib_port_state state; |
| 679 | enum ib_mtu max_mtu; |
| 680 | enum ib_mtu active_mtu; |
Kaike Wan | 6d72344 | 2020-05-11 12:06:18 -0400 | [diff] [blame] | 681 | u32 phys_mtu; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 682 | int gid_tbl_len; |
Jason Gunthorpe | 2f944c0 | 2018-07-04 15:57:48 +0300 | [diff] [blame] | 683 | unsigned int ip_gids:1; |
| 684 | /* This is the value from PortInfo CapabilityMask, defined by IBA */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 685 | u32 port_cap_flags; |
| 686 | u32 max_msg_sz; |
| 687 | u32 bad_pkey_cntr; |
| 688 | u32 qkey_viol_cntr; |
| 689 | u16 pkey_tbl_len; |
Dasaratharaman Chandramouli | db58540 | 2017-06-08 13:37:48 -0400 | [diff] [blame] | 690 | u32 sm_lid; |
Dasaratharaman Chandramouli | 582faf3 | 2017-06-08 13:37:47 -0400 | [diff] [blame] | 691 | u32 lid; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 692 | u8 lmc; |
| 693 | u8 max_vl_num; |
| 694 | u8 sm_sl; |
| 695 | u8 subnet_timeout; |
| 696 | u8 init_type_reply; |
| 697 | u8 active_width; |
Aharon Landau | 376ceb3 | 2020-09-17 12:02:23 +0300 | [diff] [blame] | 698 | u16 active_speed; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 699 | u8 phys_state; |
Michael Guralnik | 1e8f43b | 2018-12-09 11:49:48 +0200 | [diff] [blame] | 700 | u16 port_cap_flags2; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 701 | }; |
| 702 | |
| 703 | enum ib_device_modify_flags { |
Roland Dreier | c5bcbbb | 2006-02-02 09:47:14 -0800 | [diff] [blame] | 704 | IB_DEVICE_MODIFY_SYS_IMAGE_GUID = 1 << 0, |
| 705 | IB_DEVICE_MODIFY_NODE_DESC = 1 << 1 |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 706 | }; |
| 707 | |
Yuval Shaia | bd99fde | 2016-08-25 10:57:07 -0700 | [diff] [blame] | 708 | #define IB_DEVICE_NODE_DESC_MAX 64 |
| 709 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 710 | struct ib_device_modify { |
| 711 | u64 sys_image_guid; |
Yuval Shaia | bd99fde | 2016-08-25 10:57:07 -0700 | [diff] [blame] | 712 | char node_desc[IB_DEVICE_NODE_DESC_MAX]; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 713 | }; |
| 714 | |
| 715 | enum ib_port_modify_flags { |
| 716 | IB_PORT_SHUTDOWN = 1, |
| 717 | IB_PORT_INIT_TYPE = (1<<2), |
Vishwanathapura, Niranjana | cb49366 | 2017-06-01 17:04:02 -0700 | [diff] [blame] | 718 | IB_PORT_RESET_QKEY_CNTR = (1<<3), |
| 719 | IB_PORT_OPA_MASK_CHG = (1<<4) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 720 | }; |
| 721 | |
| 722 | struct ib_port_modify { |
| 723 | u32 set_port_cap_mask; |
| 724 | u32 clr_port_cap_mask; |
| 725 | u8 init_type; |
| 726 | }; |
| 727 | |
| 728 | enum ib_event_type { |
| 729 | IB_EVENT_CQ_ERR, |
| 730 | IB_EVENT_QP_FATAL, |
| 731 | IB_EVENT_QP_REQ_ERR, |
| 732 | IB_EVENT_QP_ACCESS_ERR, |
| 733 | IB_EVENT_COMM_EST, |
| 734 | IB_EVENT_SQ_DRAINED, |
| 735 | IB_EVENT_PATH_MIG, |
| 736 | IB_EVENT_PATH_MIG_ERR, |
| 737 | IB_EVENT_DEVICE_FATAL, |
| 738 | IB_EVENT_PORT_ACTIVE, |
| 739 | IB_EVENT_PORT_ERR, |
| 740 | IB_EVENT_LID_CHANGE, |
| 741 | IB_EVENT_PKEY_CHANGE, |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 742 | IB_EVENT_SM_CHANGE, |
| 743 | IB_EVENT_SRQ_ERR, |
| 744 | IB_EVENT_SRQ_LIMIT_REACHED, |
Leonid Arsh | 63942c9 | 2006-06-17 20:37:35 -0700 | [diff] [blame] | 745 | IB_EVENT_QP_LAST_WQE_REACHED, |
Or Gerlitz | 761d90e | 2011-06-15 14:39:29 +0000 | [diff] [blame] | 746 | IB_EVENT_CLIENT_REREGISTER, |
| 747 | IB_EVENT_GID_CHANGE, |
Yishai Hadas | f213c05 | 2016-05-23 15:20:49 +0300 | [diff] [blame] | 748 | IB_EVENT_WQ_FATAL, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 749 | }; |
| 750 | |
Bart Van Assche | db7489e | 2015-08-03 10:01:52 -0700 | [diff] [blame] | 751 | const char *__attribute_const__ ib_event_msg(enum ib_event_type event); |
Sagi Grimberg | 2b1b5b6 | 2015-05-18 13:40:28 +0300 | [diff] [blame] | 752 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 753 | struct ib_event { |
| 754 | struct ib_device *device; |
| 755 | union { |
| 756 | struct ib_cq *cq; |
| 757 | struct ib_qp *qp; |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 758 | struct ib_srq *srq; |
Yishai Hadas | f213c05 | 2016-05-23 15:20:49 +0300 | [diff] [blame] | 759 | struct ib_wq *wq; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 760 | u32 port_num; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 761 | } element; |
| 762 | enum ib_event_type event; |
| 763 | }; |
| 764 | |
| 765 | struct ib_event_handler { |
| 766 | struct ib_device *device; |
| 767 | void (*handler)(struct ib_event_handler *, struct ib_event *); |
| 768 | struct list_head list; |
| 769 | }; |
| 770 | |
| 771 | #define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler) \ |
| 772 | do { \ |
| 773 | (_ptr)->device = _device; \ |
| 774 | (_ptr)->handler = _handler; \ |
| 775 | INIT_LIST_HEAD(&(_ptr)->list); \ |
| 776 | } while (0) |
| 777 | |
| 778 | struct ib_global_route { |
Jason Gunthorpe | 8d9ec9a | 2018-06-13 10:22:03 +0300 | [diff] [blame] | 779 | const struct ib_gid_attr *sgid_attr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 780 | union ib_gid dgid; |
| 781 | u32 flow_label; |
| 782 | u8 sgid_index; |
| 783 | u8 hop_limit; |
| 784 | u8 traffic_class; |
| 785 | }; |
| 786 | |
Hal Rosenstock | 513789e | 2005-07-27 11:45:34 -0700 | [diff] [blame] | 787 | struct ib_grh { |
Sean Hefty | 97f52eb | 2005-08-13 21:05:57 -0700 | [diff] [blame] | 788 | __be32 version_tclass_flow; |
| 789 | __be16 paylen; |
Hal Rosenstock | 513789e | 2005-07-27 11:45:34 -0700 | [diff] [blame] | 790 | u8 next_hdr; |
| 791 | u8 hop_limit; |
| 792 | union ib_gid sgid; |
| 793 | union ib_gid dgid; |
| 794 | }; |
| 795 | |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 796 | union rdma_network_hdr { |
| 797 | struct ib_grh ibgrh; |
| 798 | struct { |
| 799 | /* The IB spec states that if it's IPv4, the header |
| 800 | * is located in the last 20 bytes of the header. |
| 801 | */ |
| 802 | u8 reserved[20]; |
| 803 | struct iphdr roce4grh; |
| 804 | }; |
| 805 | }; |
| 806 | |
Don Hiatt | 7dafbab | 2017-05-12 09:19:55 -0700 | [diff] [blame] | 807 | #define IB_QPN_MASK 0xFFFFFF |
| 808 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 809 | enum { |
| 810 | IB_MULTICAST_QPN = 0xffffff |
| 811 | }; |
| 812 | |
Harvey Harrison | f3a7c66 | 2009-02-14 22:58:35 -0800 | [diff] [blame] | 813 | #define IB_LID_PERMISSIVE cpu_to_be16(0xFFFF) |
Dennis Dalessandro | b4e6439 | 2016-01-06 10:04:31 -0800 | [diff] [blame] | 814 | #define IB_MULTICAST_LID_BASE cpu_to_be16(0xC000) |
Sean Hefty | 97f52eb | 2005-08-13 21:05:57 -0700 | [diff] [blame] | 815 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 816 | enum ib_ah_flags { |
| 817 | IB_AH_GRH = 1 |
| 818 | }; |
| 819 | |
Jack Morgenstein | bf6a9e3 | 2006-04-10 09:43:47 -0700 | [diff] [blame] | 820 | enum ib_rate { |
| 821 | IB_RATE_PORT_CURRENT = 0, |
| 822 | IB_RATE_2_5_GBPS = 2, |
| 823 | IB_RATE_5_GBPS = 5, |
| 824 | IB_RATE_10_GBPS = 3, |
| 825 | IB_RATE_20_GBPS = 6, |
| 826 | IB_RATE_30_GBPS = 4, |
| 827 | IB_RATE_40_GBPS = 7, |
| 828 | IB_RATE_60_GBPS = 8, |
| 829 | IB_RATE_80_GBPS = 9, |
Marcel Apfelbaum | 71eeba1 | 2011-10-05 14:21:47 +0300 | [diff] [blame] | 830 | IB_RATE_120_GBPS = 10, |
| 831 | IB_RATE_14_GBPS = 11, |
| 832 | IB_RATE_56_GBPS = 12, |
| 833 | IB_RATE_112_GBPS = 13, |
| 834 | IB_RATE_168_GBPS = 14, |
| 835 | IB_RATE_25_GBPS = 15, |
| 836 | IB_RATE_100_GBPS = 16, |
| 837 | IB_RATE_200_GBPS = 17, |
Michael Guralnik | a5a5d19 | 2018-12-09 11:49:50 +0200 | [diff] [blame] | 838 | IB_RATE_300_GBPS = 18, |
| 839 | IB_RATE_28_GBPS = 19, |
| 840 | IB_RATE_50_GBPS = 20, |
| 841 | IB_RATE_400_GBPS = 21, |
| 842 | IB_RATE_600_GBPS = 22, |
Jack Morgenstein | bf6a9e3 | 2006-04-10 09:43:47 -0700 | [diff] [blame] | 843 | }; |
| 844 | |
| 845 | /** |
| 846 | * ib_rate_to_mult - Convert the IB rate enum to a multiple of the |
| 847 | * base rate of 2.5 Gbit/sec. For example, IB_RATE_5_GBPS will be |
| 848 | * converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec. |
| 849 | * @rate: rate to convert. |
| 850 | */ |
Roland Dreier | 8385fd8 | 2014-06-04 10:00:16 -0700 | [diff] [blame] | 851 | __attribute_const__ int ib_rate_to_mult(enum ib_rate rate); |
Jack Morgenstein | bf6a9e3 | 2006-04-10 09:43:47 -0700 | [diff] [blame] | 852 | |
| 853 | /** |
Marcel Apfelbaum | 71eeba1 | 2011-10-05 14:21:47 +0300 | [diff] [blame] | 854 | * ib_rate_to_mbps - Convert the IB rate enum to Mbps. |
| 855 | * For example, IB_RATE_2_5_GBPS will be converted to 2500. |
| 856 | * @rate: rate to convert. |
| 857 | */ |
Roland Dreier | 8385fd8 | 2014-06-04 10:00:16 -0700 | [diff] [blame] | 858 | __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate); |
Marcel Apfelbaum | 71eeba1 | 2011-10-05 14:21:47 +0300 | [diff] [blame] | 859 | |
Sagi Grimberg | 17cd3a2 | 2014-02-23 14:19:04 +0200 | [diff] [blame] | 860 | |
| 861 | /** |
Sagi Grimberg | 9bee178 | 2015-07-30 10:32:35 +0300 | [diff] [blame] | 862 | * enum ib_mr_type - memory region type |
| 863 | * @IB_MR_TYPE_MEM_REG: memory region that is used for |
| 864 | * normal registration |
Sagi Grimberg | f5aa915 | 2016-02-29 19:07:32 +0200 | [diff] [blame] | 865 | * @IB_MR_TYPE_SG_GAPS: memory region that is capable to |
| 866 | * register any arbitrary sg lists (without |
| 867 | * the normal mr constraints - see |
| 868 | * ib_map_mr_sg) |
Max Gurtovoy | a0bc099 | 2019-06-11 18:52:38 +0300 | [diff] [blame] | 869 | * @IB_MR_TYPE_DM: memory region that is used for device |
| 870 | * memory registration |
| 871 | * @IB_MR_TYPE_USER: memory region that is used for the user-space |
| 872 | * application |
| 873 | * @IB_MR_TYPE_DMA: memory region that is used for DMA operations |
| 874 | * without address translations (VA=PA) |
Israel Rukshin | 26bc7ea | 2019-06-11 18:52:39 +0300 | [diff] [blame] | 875 | * @IB_MR_TYPE_INTEGRITY: memory region that is used for |
| 876 | * data integrity operations |
Sagi Grimberg | 17cd3a2 | 2014-02-23 14:19:04 +0200 | [diff] [blame] | 877 | */ |
Sagi Grimberg | 9bee178 | 2015-07-30 10:32:35 +0300 | [diff] [blame] | 878 | enum ib_mr_type { |
| 879 | IB_MR_TYPE_MEM_REG, |
Sagi Grimberg | f5aa915 | 2016-02-29 19:07:32 +0200 | [diff] [blame] | 880 | IB_MR_TYPE_SG_GAPS, |
Max Gurtovoy | a0bc099 | 2019-06-11 18:52:38 +0300 | [diff] [blame] | 881 | IB_MR_TYPE_DM, |
| 882 | IB_MR_TYPE_USER, |
| 883 | IB_MR_TYPE_DMA, |
Israel Rukshin | 26bc7ea | 2019-06-11 18:52:39 +0300 | [diff] [blame] | 884 | IB_MR_TYPE_INTEGRITY, |
Sagi Grimberg | 17cd3a2 | 2014-02-23 14:19:04 +0200 | [diff] [blame] | 885 | }; |
| 886 | |
Sagi Grimberg | 1b01d33 | 2014-02-23 14:19:05 +0200 | [diff] [blame] | 887 | enum ib_mr_status_check { |
| 888 | IB_MR_CHECK_SIG_STATUS = 1, |
| 889 | }; |
| 890 | |
| 891 | /** |
| 892 | * struct ib_mr_status - Memory region status container |
| 893 | * |
| 894 | * @fail_status: Bitmask of MR checks status. For each |
| 895 | * failed check a corresponding status bit is set. |
| 896 | * @sig_err: Additional info for IB_MR_CEHCK_SIG_STATUS |
| 897 | * failure. |
| 898 | */ |
| 899 | struct ib_mr_status { |
| 900 | u32 fail_status; |
| 901 | struct ib_sig_err sig_err; |
| 902 | }; |
| 903 | |
Marcel Apfelbaum | 71eeba1 | 2011-10-05 14:21:47 +0300 | [diff] [blame] | 904 | /** |
Jack Morgenstein | bf6a9e3 | 2006-04-10 09:43:47 -0700 | [diff] [blame] | 905 | * mult_to_ib_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate |
| 906 | * enum. |
| 907 | * @mult: multiple to convert. |
| 908 | */ |
Roland Dreier | 8385fd8 | 2014-06-04 10:00:16 -0700 | [diff] [blame] | 909 | __attribute_const__ enum ib_rate mult_to_ib_rate(int mult); |
Jack Morgenstein | bf6a9e3 | 2006-04-10 09:43:47 -0700 | [diff] [blame] | 910 | |
Maor Gottlieb | fa5d010 | 2020-04-30 22:21:42 +0300 | [diff] [blame] | 911 | struct rdma_ah_init_attr { |
| 912 | struct rdma_ah_attr *ah_attr; |
| 913 | u32 flags; |
Maor Gottlieb | 51aab12 | 2020-04-30 22:21:44 +0300 | [diff] [blame] | 914 | struct net_device *xmit_slave; |
Maor Gottlieb | fa5d010 | 2020-04-30 22:21:42 +0300 | [diff] [blame] | 915 | }; |
| 916 | |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 917 | enum rdma_ah_attr_type { |
Don Hiatt | 87daac6 | 2018-02-01 10:57:03 -0800 | [diff] [blame] | 918 | RDMA_AH_ATTR_TYPE_UNDEFINED, |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 919 | RDMA_AH_ATTR_TYPE_IB, |
| 920 | RDMA_AH_ATTR_TYPE_ROCE, |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 921 | RDMA_AH_ATTR_TYPE_OPA, |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 922 | }; |
| 923 | |
| 924 | struct ib_ah_attr { |
| 925 | u16 dlid; |
| 926 | u8 src_path_bits; |
| 927 | }; |
| 928 | |
| 929 | struct roce_ah_attr { |
| 930 | u8 dmac[ETH_ALEN]; |
| 931 | }; |
| 932 | |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 933 | struct opa_ah_attr { |
| 934 | u32 dlid; |
| 935 | u8 src_path_bits; |
Don Hiatt | d98bb7f | 2017-08-04 13:54:16 -0700 | [diff] [blame] | 936 | bool make_grd; |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 937 | }; |
| 938 | |
Dasaratharaman Chandramouli | 9089885 | 2017-04-29 14:41:18 -0400 | [diff] [blame] | 939 | struct rdma_ah_attr { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 940 | struct ib_global_route grh; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 941 | u8 sl; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 942 | u8 static_rate; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 943 | u32 port_num; |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 944 | u8 ah_flags; |
| 945 | enum rdma_ah_attr_type type; |
| 946 | union { |
| 947 | struct ib_ah_attr ib; |
| 948 | struct roce_ah_attr roce; |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 949 | struct opa_ah_attr opa; |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 950 | }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 951 | }; |
| 952 | |
| 953 | enum ib_wc_status { |
| 954 | IB_WC_SUCCESS, |
| 955 | IB_WC_LOC_LEN_ERR, |
| 956 | IB_WC_LOC_QP_OP_ERR, |
| 957 | IB_WC_LOC_EEC_OP_ERR, |
| 958 | IB_WC_LOC_PROT_ERR, |
| 959 | IB_WC_WR_FLUSH_ERR, |
| 960 | IB_WC_MW_BIND_ERR, |
| 961 | IB_WC_BAD_RESP_ERR, |
| 962 | IB_WC_LOC_ACCESS_ERR, |
| 963 | IB_WC_REM_INV_REQ_ERR, |
| 964 | IB_WC_REM_ACCESS_ERR, |
| 965 | IB_WC_REM_OP_ERR, |
| 966 | IB_WC_RETRY_EXC_ERR, |
| 967 | IB_WC_RNR_RETRY_EXC_ERR, |
| 968 | IB_WC_LOC_RDD_VIOL_ERR, |
| 969 | IB_WC_REM_INV_RD_REQ_ERR, |
| 970 | IB_WC_REM_ABORT_ERR, |
| 971 | IB_WC_INV_EECN_ERR, |
| 972 | IB_WC_INV_EEC_STATE_ERR, |
| 973 | IB_WC_FATAL_ERR, |
| 974 | IB_WC_RESP_TIMEOUT_ERR, |
| 975 | IB_WC_GENERAL_ERR |
| 976 | }; |
| 977 | |
Bart Van Assche | db7489e | 2015-08-03 10:01:52 -0700 | [diff] [blame] | 978 | const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status); |
Sagi Grimberg | 2b1b5b6 | 2015-05-18 13:40:28 +0300 | [diff] [blame] | 979 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 980 | enum ib_wc_opcode { |
Bob Pearson | b60b9c0 | 2020-09-03 17:40:34 -0500 | [diff] [blame] | 981 | IB_WC_SEND = IB_UVERBS_WC_SEND, |
| 982 | IB_WC_RDMA_WRITE = IB_UVERBS_WC_RDMA_WRITE, |
| 983 | IB_WC_RDMA_READ = IB_UVERBS_WC_RDMA_READ, |
| 984 | IB_WC_COMP_SWAP = IB_UVERBS_WC_COMP_SWAP, |
| 985 | IB_WC_FETCH_ADD = IB_UVERBS_WC_FETCH_ADD, |
| 986 | IB_WC_BIND_MW = IB_UVERBS_WC_BIND_MW, |
| 987 | IB_WC_LOCAL_INV = IB_UVERBS_WC_LOCAL_INV, |
| 988 | IB_WC_LSO = IB_UVERBS_WC_TSO, |
Xiao Yang | 3ff81e8 | 2022-12-01 14:37:06 +0000 | [diff] [blame] | 989 | IB_WC_ATOMIC_WRITE = IB_UVERBS_WC_ATOMIC_WRITE, |
Sagi Grimberg | 4c67e2b | 2015-10-13 19:11:24 +0300 | [diff] [blame] | 990 | IB_WC_REG_MR, |
Vladimir Sokolovsky | 5e80ba8 | 2010-04-14 17:23:01 +0300 | [diff] [blame] | 991 | IB_WC_MASKED_COMP_SWAP, |
| 992 | IB_WC_MASKED_FETCH_ADD, |
Li Zhijian | 208e3a1 | 2022-12-06 21:01:53 +0800 | [diff] [blame] | 993 | IB_WC_FLUSH = IB_UVERBS_WC_FLUSH, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 994 | /* |
| 995 | * Set value of IB_WC_RECV so consumers can test if a completion is a |
| 996 | * receive by testing (opcode & IB_WC_RECV). |
| 997 | */ |
| 998 | IB_WC_RECV = 1 << 7, |
| 999 | IB_WC_RECV_RDMA_WITH_IMM |
| 1000 | }; |
| 1001 | |
| 1002 | enum ib_wc_flags { |
| 1003 | IB_WC_GRH = 1, |
Steve Wise | 00f7ec3 | 2008-07-14 23:48:45 -0700 | [diff] [blame] | 1004 | IB_WC_WITH_IMM = (1<<1), |
| 1005 | IB_WC_WITH_INVALIDATE = (1<<2), |
Or Gerlitz | d927d50 | 2012-01-11 19:03:51 +0200 | [diff] [blame] | 1006 | IB_WC_IP_CSUM_OK = (1<<3), |
Matan Barak | dd5f03b | 2013-12-12 18:03:11 +0200 | [diff] [blame] | 1007 | IB_WC_WITH_SMAC = (1<<4), |
| 1008 | IB_WC_WITH_VLAN = (1<<5), |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 1009 | IB_WC_WITH_NETWORK_HDR_TYPE = (1<<6), |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1010 | }; |
| 1011 | |
| 1012 | struct ib_wc { |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 1013 | union { |
| 1014 | u64 wr_id; |
| 1015 | struct ib_cqe *wr_cqe; |
| 1016 | }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1017 | enum ib_wc_status status; |
| 1018 | enum ib_wc_opcode opcode; |
| 1019 | u32 vendor_err; |
| 1020 | u32 byte_len; |
Michael S. Tsirkin | 062dbb6 | 2006-12-31 21:09:42 +0200 | [diff] [blame] | 1021 | struct ib_qp *qp; |
Steve Wise | 00f7ec3 | 2008-07-14 23:48:45 -0700 | [diff] [blame] | 1022 | union { |
| 1023 | __be32 imm_data; |
| 1024 | u32 invalidate_rkey; |
| 1025 | } ex; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1026 | u32 src_qp; |
Bodong Wang | cd2a6e7 | 2018-01-12 07:58:41 +0200 | [diff] [blame] | 1027 | u32 slid; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1028 | int wc_flags; |
| 1029 | u16 pkey_index; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1030 | u8 sl; |
| 1031 | u8 dlid_path_bits; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 1032 | u32 port_num; /* valid only for DR SMPs on switches */ |
Matan Barak | dd5f03b | 2013-12-12 18:03:11 +0200 | [diff] [blame] | 1033 | u8 smac[ETH_ALEN]; |
| 1034 | u16 vlan_id; |
Somnath Kotur | c865f24 | 2015-12-23 14:56:51 +0200 | [diff] [blame] | 1035 | u8 network_hdr_type; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1036 | }; |
| 1037 | |
Roland Dreier | ed23a72 | 2007-05-06 21:02:48 -0700 | [diff] [blame] | 1038 | enum ib_cq_notify_flags { |
| 1039 | IB_CQ_SOLICITED = 1 << 0, |
| 1040 | IB_CQ_NEXT_COMP = 1 << 1, |
| 1041 | IB_CQ_SOLICITED_MASK = IB_CQ_SOLICITED | IB_CQ_NEXT_COMP, |
| 1042 | IB_CQ_REPORT_MISSED_EVENTS = 1 << 2, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1043 | }; |
| 1044 | |
Sean Hefty | 96104ed | 2011-05-23 16:31:36 -0700 | [diff] [blame] | 1045 | enum ib_srq_type { |
Yishai Hadas | 175ba58 | 2020-05-19 10:27:08 +0300 | [diff] [blame] | 1046 | IB_SRQT_BASIC = IB_UVERBS_SRQT_BASIC, |
| 1047 | IB_SRQT_XRC = IB_UVERBS_SRQT_XRC, |
| 1048 | IB_SRQT_TM = IB_UVERBS_SRQT_TM, |
Sean Hefty | 96104ed | 2011-05-23 16:31:36 -0700 | [diff] [blame] | 1049 | }; |
| 1050 | |
Artemy Kovalyov | 1a56ff6 | 2017-08-17 15:52:04 +0300 | [diff] [blame] | 1051 | static inline bool ib_srq_has_cq(enum ib_srq_type srq_type) |
| 1052 | { |
Artemy Kovalyov | 9c2c849 | 2017-08-17 15:52:05 +0300 | [diff] [blame] | 1053 | return srq_type == IB_SRQT_XRC || |
| 1054 | srq_type == IB_SRQT_TM; |
Artemy Kovalyov | 1a56ff6 | 2017-08-17 15:52:04 +0300 | [diff] [blame] | 1055 | } |
| 1056 | |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 1057 | enum ib_srq_attr_mask { |
| 1058 | IB_SRQ_MAX_WR = 1 << 0, |
| 1059 | IB_SRQ_LIMIT = 1 << 1, |
| 1060 | }; |
| 1061 | |
| 1062 | struct ib_srq_attr { |
| 1063 | u32 max_wr; |
| 1064 | u32 max_sge; |
| 1065 | u32 srq_limit; |
| 1066 | }; |
| 1067 | |
| 1068 | struct ib_srq_init_attr { |
| 1069 | void (*event_handler)(struct ib_event *, void *); |
| 1070 | void *srq_context; |
| 1071 | struct ib_srq_attr attr; |
Sean Hefty | 96104ed | 2011-05-23 16:31:36 -0700 | [diff] [blame] | 1072 | enum ib_srq_type srq_type; |
Sean Hefty | 418d513 | 2011-05-23 19:42:29 -0700 | [diff] [blame] | 1073 | |
Artemy Kovalyov | 1a56ff6 | 2017-08-17 15:52:04 +0300 | [diff] [blame] | 1074 | struct { |
| 1075 | struct ib_cq *cq; |
| 1076 | union { |
| 1077 | struct { |
| 1078 | struct ib_xrcd *xrcd; |
| 1079 | } xrc; |
Artemy Kovalyov | 9c2c849 | 2017-08-17 15:52:05 +0300 | [diff] [blame] | 1080 | |
| 1081 | struct { |
| 1082 | u32 max_num_tags; |
| 1083 | } tag_matching; |
Artemy Kovalyov | 1a56ff6 | 2017-08-17 15:52:04 +0300 | [diff] [blame] | 1084 | }; |
Sean Hefty | 418d513 | 2011-05-23 19:42:29 -0700 | [diff] [blame] | 1085 | } ext; |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 1086 | }; |
| 1087 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1088 | struct ib_qp_cap { |
| 1089 | u32 max_send_wr; |
| 1090 | u32 max_recv_wr; |
| 1091 | u32 max_send_sge; |
| 1092 | u32 max_recv_sge; |
| 1093 | u32 max_inline_data; |
Christoph Hellwig | a060b56 | 2016-05-03 18:01:09 +0200 | [diff] [blame] | 1094 | |
| 1095 | /* |
| 1096 | * Maximum number of rdma_rw_ctx structures in flight at a time. |
| 1097 | * ib_create_qp() will calculate the right amount of neededed WRs |
| 1098 | * and MRs based on this. |
| 1099 | */ |
| 1100 | u32 max_rdma_ctxs; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1101 | }; |
| 1102 | |
| 1103 | enum ib_sig_type { |
| 1104 | IB_SIGNAL_ALL_WR, |
| 1105 | IB_SIGNAL_REQ_WR |
| 1106 | }; |
| 1107 | |
| 1108 | enum ib_qp_type { |
| 1109 | /* |
| 1110 | * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries |
| 1111 | * here (and in that order) since the MAD layer uses them as |
| 1112 | * indices into a 2-entry table. |
| 1113 | */ |
| 1114 | IB_QPT_SMI, |
| 1115 | IB_QPT_GSI, |
| 1116 | |
Yishai Hadas | 175ba58 | 2020-05-19 10:27:08 +0300 | [diff] [blame] | 1117 | IB_QPT_RC = IB_UVERBS_QPT_RC, |
| 1118 | IB_QPT_UC = IB_UVERBS_QPT_UC, |
| 1119 | IB_QPT_UD = IB_UVERBS_QPT_UD, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1120 | IB_QPT_RAW_IPV6, |
Sean Hefty | b42b63c | 2011-05-23 19:59:25 -0700 | [diff] [blame] | 1121 | IB_QPT_RAW_ETHERTYPE, |
Yishai Hadas | 175ba58 | 2020-05-19 10:27:08 +0300 | [diff] [blame] | 1122 | IB_QPT_RAW_PACKET = IB_UVERBS_QPT_RAW_PACKET, |
| 1123 | IB_QPT_XRC_INI = IB_UVERBS_QPT_XRC_INI, |
| 1124 | IB_QPT_XRC_TGT = IB_UVERBS_QPT_XRC_TGT, |
Jack Morgenstein | 0134f16 | 2013-07-07 17:25:52 +0300 | [diff] [blame] | 1125 | IB_QPT_MAX, |
Yishai Hadas | 175ba58 | 2020-05-19 10:27:08 +0300 | [diff] [blame] | 1126 | IB_QPT_DRIVER = IB_UVERBS_QPT_DRIVER, |
Jack Morgenstein | 0134f16 | 2013-07-07 17:25:52 +0300 | [diff] [blame] | 1127 | /* Reserve a range for qp types internal to the low level driver. |
| 1128 | * These qp types will not be visible at the IB core layer, so the |
| 1129 | * IB_QPT_MAX usages should not be affected in the core layer |
| 1130 | */ |
| 1131 | IB_QPT_RESERVED1 = 0x1000, |
| 1132 | IB_QPT_RESERVED2, |
| 1133 | IB_QPT_RESERVED3, |
| 1134 | IB_QPT_RESERVED4, |
| 1135 | IB_QPT_RESERVED5, |
| 1136 | IB_QPT_RESERVED6, |
| 1137 | IB_QPT_RESERVED7, |
| 1138 | IB_QPT_RESERVED8, |
| 1139 | IB_QPT_RESERVED9, |
| 1140 | IB_QPT_RESERVED10, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1141 | }; |
| 1142 | |
Eli Cohen | b846f25 | 2008-04-16 21:09:27 -0700 | [diff] [blame] | 1143 | enum ib_qp_create_flags { |
Ron Livne | 47ee1b9 | 2008-07-14 23:48:48 -0700 | [diff] [blame] | 1144 | IB_QP_CREATE_IPOIB_UD_LSO = 1 << 0, |
Yishai Hadas | 175ba58 | 2020-05-19 10:27:08 +0300 | [diff] [blame] | 1145 | IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK = |
| 1146 | IB_UVERBS_QP_CREATE_BLOCK_MULTICAST_LOOPBACK, |
Leon Romanovsky | 8a06ce5 | 2015-12-20 12:16:10 +0200 | [diff] [blame] | 1147 | IB_QP_CREATE_CROSS_CHANNEL = 1 << 2, |
| 1148 | IB_QP_CREATE_MANAGED_SEND = 1 << 3, |
| 1149 | IB_QP_CREATE_MANAGED_RECV = 1 << 4, |
Matan Barak | 90f1d1b | 2013-11-07 15:25:12 +0200 | [diff] [blame] | 1150 | IB_QP_CREATE_NETIF_QP = 1 << 5, |
Israel Rukshin | c0a6cbb | 2019-06-11 18:52:50 +0300 | [diff] [blame] | 1151 | IB_QP_CREATE_INTEGRITY_EN = 1 << 6, |
Gary Leshner | 7f90a5a | 2020-05-11 12:06:07 -0400 | [diff] [blame] | 1152 | IB_QP_CREATE_NETDEV_USE = 1 << 7, |
Yishai Hadas | 175ba58 | 2020-05-19 10:27:08 +0300 | [diff] [blame] | 1153 | IB_QP_CREATE_SCATTER_FCS = |
| 1154 | IB_UVERBS_QP_CREATE_SCATTER_FCS, |
| 1155 | IB_QP_CREATE_CVLAN_STRIPPING = |
| 1156 | IB_UVERBS_QP_CREATE_CVLAN_STRIPPING, |
Yishai Hadas | 02984cc | 2017-06-08 16:15:06 +0300 | [diff] [blame] | 1157 | IB_QP_CREATE_SOURCE_QPN = 1 << 10, |
Yishai Hadas | 175ba58 | 2020-05-19 10:27:08 +0300 | [diff] [blame] | 1158 | IB_QP_CREATE_PCI_WRITE_END_PADDING = |
| 1159 | IB_UVERBS_QP_CREATE_PCI_WRITE_END_PADDING, |
Jack Morgenstein | d2b5706 | 2012-08-03 08:40:37 +0000 | [diff] [blame] | 1160 | /* reserve bits 26-31 for low level drivers' internal use */ |
| 1161 | IB_QP_CREATE_RESERVED_START = 1 << 26, |
| 1162 | IB_QP_CREATE_RESERVED_END = 1 << 31, |
Eli Cohen | b846f25 | 2008-04-16 21:09:27 -0700 | [diff] [blame] | 1163 | }; |
| 1164 | |
Yishai Hadas | 73c40c6 | 2013-08-01 18:49:53 +0300 | [diff] [blame] | 1165 | /* |
| 1166 | * Note: users may not call ib_close_qp or ib_destroy_qp from the event_handler |
| 1167 | * callback to destroy the passed in QP. |
| 1168 | */ |
| 1169 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1170 | struct ib_qp_init_attr { |
Mark Zhang | 312b8f7 | 2023-01-04 11:43:34 +0200 | [diff] [blame] | 1171 | /* This callback occurs in workqueue context */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1172 | void (*event_handler)(struct ib_event *, void *); |
Chuck Lever | eb93c82e | 2018-09-04 11:45:20 -0400 | [diff] [blame] | 1173 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1174 | void *qp_context; |
| 1175 | struct ib_cq *send_cq; |
| 1176 | struct ib_cq *recv_cq; |
| 1177 | struct ib_srq *srq; |
Sean Hefty | b42b63c | 2011-05-23 19:59:25 -0700 | [diff] [blame] | 1178 | struct ib_xrcd *xrcd; /* XRC TGT QPs only */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1179 | struct ib_qp_cap cap; |
| 1180 | enum ib_sig_type sq_sig_type; |
| 1181 | enum ib_qp_type qp_type; |
Nathan Chancellor | b56511c | 2018-09-24 12:57:16 -0700 | [diff] [blame] | 1182 | u32 create_flags; |
Christoph Hellwig | a060b56 | 2016-05-03 18:01:09 +0200 | [diff] [blame] | 1183 | |
| 1184 | /* |
| 1185 | * Only needed for special QP types, or when using the RW API. |
| 1186 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 1187 | u32 port_num; |
Yishai Hadas | a9017e2 | 2016-05-23 15:20:54 +0300 | [diff] [blame] | 1188 | struct ib_rwq_ind_table *rwq_ind_tbl; |
Yishai Hadas | 02984cc | 2017-06-08 16:15:06 +0300 | [diff] [blame] | 1189 | u32 source_qpn; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1190 | }; |
| 1191 | |
Sean Hefty | 0e0ec7e | 2011-08-08 15:31:51 -0700 | [diff] [blame] | 1192 | struct ib_qp_open_attr { |
| 1193 | void (*event_handler)(struct ib_event *, void *); |
| 1194 | void *qp_context; |
| 1195 | u32 qp_num; |
| 1196 | enum ib_qp_type qp_type; |
| 1197 | }; |
| 1198 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1199 | enum ib_rnr_timeout { |
| 1200 | IB_RNR_TIMER_655_36 = 0, |
| 1201 | IB_RNR_TIMER_000_01 = 1, |
| 1202 | IB_RNR_TIMER_000_02 = 2, |
| 1203 | IB_RNR_TIMER_000_03 = 3, |
| 1204 | IB_RNR_TIMER_000_04 = 4, |
| 1205 | IB_RNR_TIMER_000_06 = 5, |
| 1206 | IB_RNR_TIMER_000_08 = 6, |
| 1207 | IB_RNR_TIMER_000_12 = 7, |
| 1208 | IB_RNR_TIMER_000_16 = 8, |
| 1209 | IB_RNR_TIMER_000_24 = 9, |
| 1210 | IB_RNR_TIMER_000_32 = 10, |
| 1211 | IB_RNR_TIMER_000_48 = 11, |
| 1212 | IB_RNR_TIMER_000_64 = 12, |
| 1213 | IB_RNR_TIMER_000_96 = 13, |
| 1214 | IB_RNR_TIMER_001_28 = 14, |
| 1215 | IB_RNR_TIMER_001_92 = 15, |
| 1216 | IB_RNR_TIMER_002_56 = 16, |
| 1217 | IB_RNR_TIMER_003_84 = 17, |
| 1218 | IB_RNR_TIMER_005_12 = 18, |
| 1219 | IB_RNR_TIMER_007_68 = 19, |
| 1220 | IB_RNR_TIMER_010_24 = 20, |
| 1221 | IB_RNR_TIMER_015_36 = 21, |
| 1222 | IB_RNR_TIMER_020_48 = 22, |
| 1223 | IB_RNR_TIMER_030_72 = 23, |
| 1224 | IB_RNR_TIMER_040_96 = 24, |
| 1225 | IB_RNR_TIMER_061_44 = 25, |
| 1226 | IB_RNR_TIMER_081_92 = 26, |
| 1227 | IB_RNR_TIMER_122_88 = 27, |
| 1228 | IB_RNR_TIMER_163_84 = 28, |
| 1229 | IB_RNR_TIMER_245_76 = 29, |
| 1230 | IB_RNR_TIMER_327_68 = 30, |
| 1231 | IB_RNR_TIMER_491_52 = 31 |
| 1232 | }; |
| 1233 | |
| 1234 | enum ib_qp_attr_mask { |
| 1235 | IB_QP_STATE = 1, |
| 1236 | IB_QP_CUR_STATE = (1<<1), |
| 1237 | IB_QP_EN_SQD_ASYNC_NOTIFY = (1<<2), |
| 1238 | IB_QP_ACCESS_FLAGS = (1<<3), |
| 1239 | IB_QP_PKEY_INDEX = (1<<4), |
| 1240 | IB_QP_PORT = (1<<5), |
| 1241 | IB_QP_QKEY = (1<<6), |
| 1242 | IB_QP_AV = (1<<7), |
| 1243 | IB_QP_PATH_MTU = (1<<8), |
| 1244 | IB_QP_TIMEOUT = (1<<9), |
| 1245 | IB_QP_RETRY_CNT = (1<<10), |
| 1246 | IB_QP_RNR_RETRY = (1<<11), |
| 1247 | IB_QP_RQ_PSN = (1<<12), |
| 1248 | IB_QP_MAX_QP_RD_ATOMIC = (1<<13), |
| 1249 | IB_QP_ALT_PATH = (1<<14), |
| 1250 | IB_QP_MIN_RNR_TIMER = (1<<15), |
| 1251 | IB_QP_SQ_PSN = (1<<16), |
| 1252 | IB_QP_MAX_DEST_RD_ATOMIC = (1<<17), |
| 1253 | IB_QP_PATH_MIG_STATE = (1<<18), |
| 1254 | IB_QP_CAP = (1<<19), |
Matan Barak | dd5f03b | 2013-12-12 18:03:11 +0200 | [diff] [blame] | 1255 | IB_QP_DEST_QPN = (1<<20), |
Matan Barak | aa744cc | 2015-10-15 18:38:53 +0300 | [diff] [blame] | 1256 | IB_QP_RESERVED1 = (1<<21), |
| 1257 | IB_QP_RESERVED2 = (1<<22), |
| 1258 | IB_QP_RESERVED3 = (1<<23), |
| 1259 | IB_QP_RESERVED4 = (1<<24), |
Bodong Wang | 528e5a1 | 2016-12-01 13:43:14 +0200 | [diff] [blame] | 1260 | IB_QP_RATE_LIMIT = (1<<25), |
Jason Gunthorpe | 26e990b | 2020-10-03 20:20:06 -0300 | [diff] [blame] | 1261 | |
| 1262 | IB_QP_ATTR_STANDARD_BITS = GENMASK(20, 0), |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1263 | }; |
| 1264 | |
| 1265 | enum ib_qp_state { |
| 1266 | IB_QPS_RESET, |
| 1267 | IB_QPS_INIT, |
| 1268 | IB_QPS_RTR, |
| 1269 | IB_QPS_RTS, |
| 1270 | IB_QPS_SQD, |
| 1271 | IB_QPS_SQE, |
| 1272 | IB_QPS_ERR |
| 1273 | }; |
| 1274 | |
| 1275 | enum ib_mig_state { |
| 1276 | IB_MIG_MIGRATED, |
| 1277 | IB_MIG_REARM, |
| 1278 | IB_MIG_ARMED |
| 1279 | }; |
| 1280 | |
Shani Michaeli | 7083e42 | 2013-02-06 16:19:12 +0000 | [diff] [blame] | 1281 | enum ib_mw_type { |
| 1282 | IB_MW_TYPE_1 = 1, |
| 1283 | IB_MW_TYPE_2 = 2 |
| 1284 | }; |
| 1285 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1286 | struct ib_qp_attr { |
| 1287 | enum ib_qp_state qp_state; |
| 1288 | enum ib_qp_state cur_qp_state; |
| 1289 | enum ib_mtu path_mtu; |
| 1290 | enum ib_mig_state path_mig_state; |
| 1291 | u32 qkey; |
| 1292 | u32 rq_psn; |
| 1293 | u32 sq_psn; |
| 1294 | u32 dest_qp_num; |
| 1295 | int qp_access_flags; |
| 1296 | struct ib_qp_cap cap; |
Dasaratharaman Chandramouli | 9089885 | 2017-04-29 14:41:18 -0400 | [diff] [blame] | 1297 | struct rdma_ah_attr ah_attr; |
| 1298 | struct rdma_ah_attr alt_ah_attr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1299 | u16 pkey_index; |
| 1300 | u16 alt_pkey_index; |
| 1301 | u8 en_sqd_async_notify; |
| 1302 | u8 sq_draining; |
| 1303 | u8 max_rd_atomic; |
| 1304 | u8 max_dest_rd_atomic; |
| 1305 | u8 min_rnr_timer; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 1306 | u32 port_num; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1307 | u8 timeout; |
| 1308 | u8 retry_cnt; |
| 1309 | u8 rnr_retry; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 1310 | u32 alt_port_num; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1311 | u8 alt_timeout; |
Bodong Wang | 528e5a1 | 2016-12-01 13:43:14 +0200 | [diff] [blame] | 1312 | u32 rate_limit; |
Maor Gottlieb | 51aab12 | 2020-04-30 22:21:44 +0300 | [diff] [blame] | 1313 | struct net_device *xmit_slave; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1314 | }; |
| 1315 | |
| 1316 | enum ib_wr_opcode { |
Jason Gunthorpe | 9a59739 | 2018-08-14 15:33:02 -0700 | [diff] [blame] | 1317 | /* These are shared with userspace */ |
| 1318 | IB_WR_RDMA_WRITE = IB_UVERBS_WR_RDMA_WRITE, |
| 1319 | IB_WR_RDMA_WRITE_WITH_IMM = IB_UVERBS_WR_RDMA_WRITE_WITH_IMM, |
| 1320 | IB_WR_SEND = IB_UVERBS_WR_SEND, |
| 1321 | IB_WR_SEND_WITH_IMM = IB_UVERBS_WR_SEND_WITH_IMM, |
| 1322 | IB_WR_RDMA_READ = IB_UVERBS_WR_RDMA_READ, |
| 1323 | IB_WR_ATOMIC_CMP_AND_SWP = IB_UVERBS_WR_ATOMIC_CMP_AND_SWP, |
| 1324 | IB_WR_ATOMIC_FETCH_AND_ADD = IB_UVERBS_WR_ATOMIC_FETCH_AND_ADD, |
Bob Pearson | b60b9c0 | 2020-09-03 17:40:34 -0500 | [diff] [blame] | 1325 | IB_WR_BIND_MW = IB_UVERBS_WR_BIND_MW, |
Jason Gunthorpe | 9a59739 | 2018-08-14 15:33:02 -0700 | [diff] [blame] | 1326 | IB_WR_LSO = IB_UVERBS_WR_TSO, |
| 1327 | IB_WR_SEND_WITH_INV = IB_UVERBS_WR_SEND_WITH_INV, |
| 1328 | IB_WR_RDMA_READ_WITH_INV = IB_UVERBS_WR_RDMA_READ_WITH_INV, |
| 1329 | IB_WR_LOCAL_INV = IB_UVERBS_WR_LOCAL_INV, |
| 1330 | IB_WR_MASKED_ATOMIC_CMP_AND_SWP = |
| 1331 | IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP, |
| 1332 | IB_WR_MASKED_ATOMIC_FETCH_AND_ADD = |
| 1333 | IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD, |
Li Zhijian | 208e3a1 | 2022-12-06 21:01:53 +0800 | [diff] [blame] | 1334 | IB_WR_FLUSH = IB_UVERBS_WR_FLUSH, |
Xiao Yang | 3ff81e8 | 2022-12-01 14:37:06 +0000 | [diff] [blame] | 1335 | IB_WR_ATOMIC_WRITE = IB_UVERBS_WR_ATOMIC_WRITE, |
Jason Gunthorpe | 9a59739 | 2018-08-14 15:33:02 -0700 | [diff] [blame] | 1336 | |
| 1337 | /* These are kernel only and can not be issued by userspace */ |
| 1338 | IB_WR_REG_MR = 0x20, |
Max Gurtovoy | 38ca87c | 2019-06-11 18:52:46 +0300 | [diff] [blame] | 1339 | IB_WR_REG_MR_INTEGRITY, |
Jason Gunthorpe | 9a59739 | 2018-08-14 15:33:02 -0700 | [diff] [blame] | 1340 | |
Jack Morgenstein | 0134f16 | 2013-07-07 17:25:52 +0300 | [diff] [blame] | 1341 | /* reserve values for low level drivers' internal use. |
| 1342 | * These values will not be used at all in the ib core layer. |
| 1343 | */ |
| 1344 | IB_WR_RESERVED1 = 0xf0, |
| 1345 | IB_WR_RESERVED2, |
| 1346 | IB_WR_RESERVED3, |
| 1347 | IB_WR_RESERVED4, |
| 1348 | IB_WR_RESERVED5, |
| 1349 | IB_WR_RESERVED6, |
| 1350 | IB_WR_RESERVED7, |
| 1351 | IB_WR_RESERVED8, |
| 1352 | IB_WR_RESERVED9, |
| 1353 | IB_WR_RESERVED10, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1354 | }; |
| 1355 | |
| 1356 | enum ib_send_flags { |
| 1357 | IB_SEND_FENCE = 1, |
| 1358 | IB_SEND_SIGNALED = (1<<1), |
| 1359 | IB_SEND_SOLICITED = (1<<2), |
Eli Cohen | e0605d9 | 2008-01-30 18:30:57 +0200 | [diff] [blame] | 1360 | IB_SEND_INLINE = (1<<3), |
Jack Morgenstein | 0134f16 | 2013-07-07 17:25:52 +0300 | [diff] [blame] | 1361 | IB_SEND_IP_CSUM = (1<<4), |
| 1362 | |
| 1363 | /* reserve bits 26-31 for low level drivers' internal use */ |
| 1364 | IB_SEND_RESERVED_START = (1 << 26), |
| 1365 | IB_SEND_RESERVED_END = (1 << 31), |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1366 | }; |
| 1367 | |
| 1368 | struct ib_sge { |
| 1369 | u64 addr; |
| 1370 | u32 length; |
| 1371 | u32 lkey; |
| 1372 | }; |
| 1373 | |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 1374 | struct ib_cqe { |
| 1375 | void (*done)(struct ib_cq *cq, struct ib_wc *wc); |
| 1376 | }; |
| 1377 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1378 | struct ib_send_wr { |
| 1379 | struct ib_send_wr *next; |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 1380 | union { |
| 1381 | u64 wr_id; |
| 1382 | struct ib_cqe *wr_cqe; |
| 1383 | }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1384 | struct ib_sge *sg_list; |
| 1385 | int num_sge; |
| 1386 | enum ib_wr_opcode opcode; |
| 1387 | int send_flags; |
Roland Dreier | 0f39cf3 | 2008-04-16 21:09:32 -0700 | [diff] [blame] | 1388 | union { |
| 1389 | __be32 imm_data; |
| 1390 | u32 invalidate_rkey; |
| 1391 | } ex; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1392 | }; |
| 1393 | |
Christoph Hellwig | e622f2f | 2015-10-08 09:16:33 +0100 | [diff] [blame] | 1394 | struct ib_rdma_wr { |
| 1395 | struct ib_send_wr wr; |
| 1396 | u64 remote_addr; |
| 1397 | u32 rkey; |
| 1398 | }; |
| 1399 | |
Bart Van Assche | f696bf6 | 2018-07-18 09:25:14 -0700 | [diff] [blame] | 1400 | static inline const struct ib_rdma_wr *rdma_wr(const struct ib_send_wr *wr) |
Christoph Hellwig | e622f2f | 2015-10-08 09:16:33 +0100 | [diff] [blame] | 1401 | { |
| 1402 | return container_of(wr, struct ib_rdma_wr, wr); |
| 1403 | } |
| 1404 | |
| 1405 | struct ib_atomic_wr { |
| 1406 | struct ib_send_wr wr; |
| 1407 | u64 remote_addr; |
| 1408 | u64 compare_add; |
| 1409 | u64 swap; |
| 1410 | u64 compare_add_mask; |
| 1411 | u64 swap_mask; |
| 1412 | u32 rkey; |
| 1413 | }; |
| 1414 | |
Bart Van Assche | f696bf6 | 2018-07-18 09:25:14 -0700 | [diff] [blame] | 1415 | static inline const struct ib_atomic_wr *atomic_wr(const struct ib_send_wr *wr) |
Christoph Hellwig | e622f2f | 2015-10-08 09:16:33 +0100 | [diff] [blame] | 1416 | { |
| 1417 | return container_of(wr, struct ib_atomic_wr, wr); |
| 1418 | } |
| 1419 | |
| 1420 | struct ib_ud_wr { |
| 1421 | struct ib_send_wr wr; |
| 1422 | struct ib_ah *ah; |
| 1423 | void *header; |
| 1424 | int hlen; |
| 1425 | int mss; |
| 1426 | u32 remote_qpn; |
| 1427 | u32 remote_qkey; |
| 1428 | u16 pkey_index; /* valid for GSI only */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 1429 | u32 port_num; /* valid for DR SMPs on switch only */ |
Christoph Hellwig | e622f2f | 2015-10-08 09:16:33 +0100 | [diff] [blame] | 1430 | }; |
| 1431 | |
Bart Van Assche | f696bf6 | 2018-07-18 09:25:14 -0700 | [diff] [blame] | 1432 | static inline const struct ib_ud_wr *ud_wr(const struct ib_send_wr *wr) |
Christoph Hellwig | e622f2f | 2015-10-08 09:16:33 +0100 | [diff] [blame] | 1433 | { |
| 1434 | return container_of(wr, struct ib_ud_wr, wr); |
| 1435 | } |
| 1436 | |
Sagi Grimberg | 4c67e2b | 2015-10-13 19:11:24 +0300 | [diff] [blame] | 1437 | struct ib_reg_wr { |
| 1438 | struct ib_send_wr wr; |
| 1439 | struct ib_mr *mr; |
| 1440 | u32 key; |
| 1441 | int access; |
| 1442 | }; |
| 1443 | |
Bart Van Assche | f696bf6 | 2018-07-18 09:25:14 -0700 | [diff] [blame] | 1444 | static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr) |
Sagi Grimberg | 4c67e2b | 2015-10-13 19:11:24 +0300 | [diff] [blame] | 1445 | { |
| 1446 | return container_of(wr, struct ib_reg_wr, wr); |
| 1447 | } |
| 1448 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1449 | struct ib_recv_wr { |
| 1450 | struct ib_recv_wr *next; |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 1451 | union { |
| 1452 | u64 wr_id; |
| 1453 | struct ib_cqe *wr_cqe; |
| 1454 | }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1455 | struct ib_sge *sg_list; |
| 1456 | int num_sge; |
| 1457 | }; |
| 1458 | |
| 1459 | enum ib_access_flags { |
Jason Gunthorpe | 4fca037 | 2018-07-11 16:20:44 -0600 | [diff] [blame] | 1460 | IB_ACCESS_LOCAL_WRITE = IB_UVERBS_ACCESS_LOCAL_WRITE, |
| 1461 | IB_ACCESS_REMOTE_WRITE = IB_UVERBS_ACCESS_REMOTE_WRITE, |
| 1462 | IB_ACCESS_REMOTE_READ = IB_UVERBS_ACCESS_REMOTE_READ, |
| 1463 | IB_ACCESS_REMOTE_ATOMIC = IB_UVERBS_ACCESS_REMOTE_ATOMIC, |
| 1464 | IB_ACCESS_MW_BIND = IB_UVERBS_ACCESS_MW_BIND, |
| 1465 | IB_ZERO_BASED = IB_UVERBS_ACCESS_ZERO_BASED, |
| 1466 | IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND, |
| 1467 | IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB, |
Michael Guralnik | 2233c66 | 2020-01-08 20:05:38 +0200 | [diff] [blame] | 1468 | IB_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_RELAXED_ORDERING, |
Li Zhijian | 208e3a1 | 2022-12-06 21:01:53 +0800 | [diff] [blame] | 1469 | IB_ACCESS_FLUSH_GLOBAL = IB_UVERBS_ACCESS_FLUSH_GLOBAL, |
| 1470 | IB_ACCESS_FLUSH_PERSISTENT = IB_UVERBS_ACCESS_FLUSH_PERSISTENT, |
Jason Gunthorpe | 4fca037 | 2018-07-11 16:20:44 -0600 | [diff] [blame] | 1471 | |
Michael Guralnik | 68d384b | 2020-01-08 20:05:36 +0200 | [diff] [blame] | 1472 | IB_ACCESS_OPTIONAL = IB_UVERBS_ACCESS_OPTIONAL_RANGE, |
| 1473 | IB_ACCESS_SUPPORTED = |
Li Zhijian | 208e3a1 | 2022-12-06 21:01:53 +0800 | [diff] [blame] | 1474 | ((IB_ACCESS_FLUSH_PERSISTENT << 1) - 1) | IB_ACCESS_OPTIONAL, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1475 | }; |
| 1476 | |
Christoph Hellwig | b7d3e0a | 2015-12-23 19:12:47 +0100 | [diff] [blame] | 1477 | /* |
| 1478 | * XXX: these are apparently used for ->rereg_user_mr, no idea why they |
| 1479 | * are hidden here instead of a uapi header! |
| 1480 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1481 | enum ib_mr_rereg_flags { |
| 1482 | IB_MR_REREG_TRANS = 1, |
| 1483 | IB_MR_REREG_PD = (1<<1), |
Matan Barak | 7e6edb9 | 2014-07-31 11:01:28 +0300 | [diff] [blame] | 1484 | IB_MR_REREG_ACCESS = (1<<2), |
| 1485 | IB_MR_REREG_SUPPORTED = ((IB_MR_REREG_ACCESS << 1) - 1) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1486 | }; |
| 1487 | |
Haggai Eran | 882214e | 2014-12-11 17:04:18 +0200 | [diff] [blame] | 1488 | struct ib_umem; |
| 1489 | |
Matan Barak | 3832125 | 2017-04-04 13:31:42 +0300 | [diff] [blame] | 1490 | enum rdma_remove_reason { |
Yishai Hadas | 1c77483 | 2018-06-20 17:11:39 +0300 | [diff] [blame] | 1491 | /* |
| 1492 | * Userspace requested uobject deletion or initial try |
| 1493 | * to remove uobject via cleanup. Call could fail |
| 1494 | */ |
Matan Barak | 3832125 | 2017-04-04 13:31:42 +0300 | [diff] [blame] | 1495 | RDMA_REMOVE_DESTROY, |
| 1496 | /* Context deletion. This call should delete the actual object itself */ |
| 1497 | RDMA_REMOVE_CLOSE, |
| 1498 | /* Driver is being hot-unplugged. This call should delete the actual object itself */ |
| 1499 | RDMA_REMOVE_DRIVER_REMOVE, |
Jason Gunthorpe | 87ad80a | 2018-07-25 21:40:12 -0600 | [diff] [blame] | 1500 | /* uobj is being cleaned-up before being committed */ |
| 1501 | RDMA_REMOVE_ABORT, |
Leon Romanovsky | efa968e | 2020-11-04 16:45:55 +0200 | [diff] [blame] | 1502 | /* The driver failed to destroy the uobject and is being disconnected */ |
| 1503 | RDMA_REMOVE_DRIVER_FAILURE, |
Matan Barak | 3832125 | 2017-04-04 13:31:42 +0300 | [diff] [blame] | 1504 | }; |
| 1505 | |
Parav Pandit | 43579b5 | 2017-01-10 00:02:14 +0000 | [diff] [blame] | 1506 | struct ib_rdmacg_object { |
| 1507 | #ifdef CONFIG_CGROUP_RDMA |
| 1508 | struct rdma_cgroup *cg; /* owner rdma cgroup */ |
| 1509 | #endif |
| 1510 | }; |
| 1511 | |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1512 | struct ib_ucontext { |
| 1513 | struct ib_device *device; |
Matan Barak | 771addf | 2017-04-04 13:31:41 +0300 | [diff] [blame] | 1514 | struct ib_uverbs_file *ufile; |
Shachar Raindel | 8ada2c1 | 2014-12-11 17:04:17 +0200 | [diff] [blame] | 1515 | |
Parav Pandit | 43579b5 | 2017-01-10 00:02:14 +0000 | [diff] [blame] | 1516 | struct ib_rdmacg_object cg_obj; |
Leon Romanovsky | 6061521 | 2018-11-28 13:16:43 +0200 | [diff] [blame] | 1517 | /* |
| 1518 | * Implementation details of the RDMA core, don't use in drivers: |
| 1519 | */ |
| 1520 | struct rdma_restrack_entry res; |
Michal Kalderon | 3411f9f | 2019-10-30 11:44:11 +0200 | [diff] [blame] | 1521 | struct xarray mmap_xa; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1522 | }; |
| 1523 | |
| 1524 | struct ib_uobject { |
| 1525 | u64 user_handle; /* handle given to us by userspace */ |
Jason Gunthorpe | 6a5e9c8 | 2018-07-04 11:32:07 +0300 | [diff] [blame] | 1526 | /* ufile & ucontext owning this object */ |
| 1527 | struct ib_uverbs_file *ufile; |
| 1528 | /* FIXME, save memory: ufile->context == context */ |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1529 | struct ib_ucontext *context; /* associated user context */ |
Roland Dreier | 9ead190 | 2006-06-17 20:44:49 -0700 | [diff] [blame] | 1530 | void *object; /* containing object */ |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1531 | struct list_head list; /* link to context's list */ |
Parav Pandit | 43579b5 | 2017-01-10 00:02:14 +0000 | [diff] [blame] | 1532 | struct ib_rdmacg_object cg_obj; /* rdmacg object */ |
Roland Dreier | b3d636b | 2008-04-16 21:01:06 -0700 | [diff] [blame] | 1533 | int id; /* index into kernel idr */ |
Roland Dreier | 9ead190 | 2006-06-17 20:44:49 -0700 | [diff] [blame] | 1534 | struct kref ref; |
Matan Barak | 3832125 | 2017-04-04 13:31:42 +0300 | [diff] [blame] | 1535 | atomic_t usecnt; /* protects exclusive access */ |
Mike Marciniszyn | d144da8 | 2015-11-02 12:13:25 -0500 | [diff] [blame] | 1536 | struct rcu_head rcu; /* kfree_rcu() overhead */ |
Matan Barak | 3832125 | 2017-04-04 13:31:42 +0300 | [diff] [blame] | 1537 | |
Jason Gunthorpe | 6b0d08f | 2018-08-09 20:14:37 -0600 | [diff] [blame] | 1538 | const struct uverbs_api_object *uapi_object; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1539 | }; |
| 1540 | |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1541 | struct ib_udata { |
Yann Droneaud | 309243e | 2013-12-11 23:01:44 +0100 | [diff] [blame] | 1542 | const void __user *inbuf; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1543 | void __user *outbuf; |
| 1544 | size_t inlen; |
| 1545 | size_t outlen; |
| 1546 | }; |
| 1547 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1548 | struct ib_pd { |
Jason Gunthorpe | 96249d7 | 2015-08-05 14:14:45 -0600 | [diff] [blame] | 1549 | u32 local_dma_lkey; |
Christoph Hellwig | ed082d3 | 2016-09-05 12:56:17 +0200 | [diff] [blame] | 1550 | u32 flags; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1551 | struct ib_device *device; |
| 1552 | struct ib_uobject *uobject; |
| 1553 | atomic_t usecnt; /* count all resources */ |
Christoph Hellwig | 50d4633 | 2016-09-05 12:56:16 +0200 | [diff] [blame] | 1554 | |
Christoph Hellwig | ed082d3 | 2016-09-05 12:56:17 +0200 | [diff] [blame] | 1555 | u32 unsafe_global_rkey; |
| 1556 | |
Christoph Hellwig | 50d4633 | 2016-09-05 12:56:16 +0200 | [diff] [blame] | 1557 | /* |
| 1558 | * Implementation details of the RDMA core, don't use in drivers: |
| 1559 | */ |
| 1560 | struct ib_mr *__internal_mr; |
Leon Romanovsky | 02d8883 | 2018-01-28 11:17:20 +0200 | [diff] [blame] | 1561 | struct rdma_restrack_entry res; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1562 | }; |
| 1563 | |
Sean Hefty | 59991f9 | 2011-05-23 17:52:46 -0700 | [diff] [blame] | 1564 | struct ib_xrcd { |
| 1565 | struct ib_device *device; |
Sean Hefty | d3d72d9 | 2011-05-26 23:06:44 -0700 | [diff] [blame] | 1566 | atomic_t usecnt; /* count all exposed resources */ |
Sean Hefty | 53d0bd1 | 2011-05-24 08:33:46 -0700 | [diff] [blame] | 1567 | struct inode *inode; |
Maor Gottlieb | 6f3ca6f | 2020-07-06 15:27:16 +0300 | [diff] [blame] | 1568 | struct rw_semaphore tgt_qps_rwsem; |
| 1569 | struct xarray tgt_qps; |
Sean Hefty | 59991f9 | 2011-05-23 17:52:46 -0700 | [diff] [blame] | 1570 | }; |
| 1571 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1572 | struct ib_ah { |
| 1573 | struct ib_device *device; |
| 1574 | struct ib_pd *pd; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1575 | struct ib_uobject *uobject; |
Jason Gunthorpe | 1a1f460 | 2018-06-13 10:22:08 +0300 | [diff] [blame] | 1576 | const struct ib_gid_attr *sgid_attr; |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 1577 | enum rdma_ah_attr_type type; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1578 | }; |
| 1579 | |
| 1580 | typedef void (*ib_comp_handler)(struct ib_cq *cq, void *cq_context); |
| 1581 | |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 1582 | enum ib_poll_context { |
Jack Morgenstein | f794809 | 2018-08-27 08:35:55 +0300 | [diff] [blame] | 1583 | IB_POLL_SOFTIRQ, /* poll from softirq context */ |
| 1584 | IB_POLL_WORKQUEUE, /* poll from workqueue */ |
| 1585 | IB_POLL_UNBOUND_WORKQUEUE, /* poll from unbound workqueue */ |
Yamin Friedman | c7ff819 | 2020-05-27 11:34:53 +0300 | [diff] [blame] | 1586 | IB_POLL_LAST_POOL_TYPE = IB_POLL_UNBOUND_WORKQUEUE, |
| 1587 | |
| 1588 | IB_POLL_DIRECT, /* caller context, no hw completions */ |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 1589 | }; |
| 1590 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1591 | struct ib_cq { |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1592 | struct ib_device *device; |
Jason Gunthorpe | 5bd48c1 | 2020-01-08 19:21:58 +0200 | [diff] [blame] | 1593 | struct ib_ucq_object *uobject; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1594 | ib_comp_handler comp_handler; |
| 1595 | void (*event_handler)(struct ib_event *, void *); |
Dotan Barak | 4deccd6 | 2008-07-14 23:48:44 -0700 | [diff] [blame] | 1596 | void *cq_context; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1597 | int cqe; |
Yamin Friedman | c7ff819 | 2020-05-27 11:34:53 +0300 | [diff] [blame] | 1598 | unsigned int cqe_used; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1599 | atomic_t usecnt; /* count number of work queues */ |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 1600 | enum ib_poll_context poll_ctx; |
| 1601 | struct ib_wc *wc; |
Yamin Friedman | c7ff819 | 2020-05-27 11:34:53 +0300 | [diff] [blame] | 1602 | struct list_head pool_entry; |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 1603 | union { |
| 1604 | struct irq_poll iop; |
| 1605 | struct work_struct work; |
| 1606 | }; |
Jack Morgenstein | f794809 | 2018-08-27 08:35:55 +0300 | [diff] [blame] | 1607 | struct workqueue_struct *comp_wq; |
Yamin Friedman | da66297 | 2019-07-08 13:59:03 +0300 | [diff] [blame] | 1608 | struct dim *dim; |
Chuck Lever | 3e5901c | 2019-12-18 15:18:15 -0500 | [diff] [blame] | 1609 | |
| 1610 | /* updated only by trace points */ |
| 1611 | ktime_t timestamp; |
Yamin Friedman | 3446cbd | 2020-05-27 11:34:52 +0300 | [diff] [blame] | 1612 | u8 interrupt:1; |
| 1613 | u8 shared:1; |
Yamin Friedman | c7ff819 | 2020-05-27 11:34:53 +0300 | [diff] [blame] | 1614 | unsigned int comp_vector; |
Chuck Lever | 3e5901c | 2019-12-18 15:18:15 -0500 | [diff] [blame] | 1615 | |
Leon Romanovsky | 02d8883 | 2018-01-28 11:17:20 +0200 | [diff] [blame] | 1616 | /* |
| 1617 | * Implementation details of the RDMA core, don't use in drivers: |
| 1618 | */ |
| 1619 | struct rdma_restrack_entry res; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1620 | }; |
| 1621 | |
| 1622 | struct ib_srq { |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 1623 | struct ib_device *device; |
| 1624 | struct ib_pd *pd; |
Jason Gunthorpe | 9fbe334 | 2020-01-08 19:22:00 +0200 | [diff] [blame] | 1625 | struct ib_usrq_object *uobject; |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 1626 | void (*event_handler)(struct ib_event *, void *); |
| 1627 | void *srq_context; |
Sean Hefty | 96104ed | 2011-05-23 16:31:36 -0700 | [diff] [blame] | 1628 | enum ib_srq_type srq_type; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1629 | atomic_t usecnt; |
Sean Hefty | 418d513 | 2011-05-23 19:42:29 -0700 | [diff] [blame] | 1630 | |
Artemy Kovalyov | 1a56ff6 | 2017-08-17 15:52:04 +0300 | [diff] [blame] | 1631 | struct { |
| 1632 | struct ib_cq *cq; |
| 1633 | union { |
| 1634 | struct { |
| 1635 | struct ib_xrcd *xrcd; |
| 1636 | u32 srq_num; |
| 1637 | } xrc; |
| 1638 | }; |
Sean Hefty | 418d513 | 2011-05-23 19:42:29 -0700 | [diff] [blame] | 1639 | } ext; |
Neta Ostrovsky | 48f8a70 | 2021-04-18 16:41:24 +0300 | [diff] [blame] | 1640 | |
| 1641 | /* |
| 1642 | * Implementation details of the RDMA core, don't use in drivers: |
| 1643 | */ |
| 1644 | struct rdma_restrack_entry res; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1645 | }; |
| 1646 | |
Noa Osherovich | ebaaee2 | 2017-01-18 15:39:54 +0200 | [diff] [blame] | 1647 | enum ib_raw_packet_caps { |
Xiao Yang | 30ad63e | 2022-03-31 11:24:18 +0800 | [diff] [blame] | 1648 | /* |
| 1649 | * Strip cvlan from incoming packet and report it in the matching work |
Noa Osherovich | ebaaee2 | 2017-01-18 15:39:54 +0200 | [diff] [blame] | 1650 | * completion is supported. |
| 1651 | */ |
Xiao Yang | 30ad63e | 2022-03-31 11:24:18 +0800 | [diff] [blame] | 1652 | IB_RAW_PACKET_CAP_CVLAN_STRIPPING = |
| 1653 | IB_UVERBS_RAW_PACKET_CAP_CVLAN_STRIPPING, |
| 1654 | /* |
| 1655 | * Scatter FCS field of an incoming packet to host memory is supported. |
Noa Osherovich | ebaaee2 | 2017-01-18 15:39:54 +0200 | [diff] [blame] | 1656 | */ |
Xiao Yang | 30ad63e | 2022-03-31 11:24:18 +0800 | [diff] [blame] | 1657 | IB_RAW_PACKET_CAP_SCATTER_FCS = IB_UVERBS_RAW_PACKET_CAP_SCATTER_FCS, |
Noa Osherovich | ebaaee2 | 2017-01-18 15:39:54 +0200 | [diff] [blame] | 1658 | /* Checksum offloads are supported (for both send and receive). */ |
Xiao Yang | 30ad63e | 2022-03-31 11:24:18 +0800 | [diff] [blame] | 1659 | IB_RAW_PACKET_CAP_IP_CSUM = IB_UVERBS_RAW_PACKET_CAP_IP_CSUM, |
| 1660 | /* |
| 1661 | * When a packet is received for an RQ with no receive WQEs, the |
Maor Gottlieb | 7d9336d | 2017-05-30 10:29:10 +0300 | [diff] [blame] | 1662 | * packet processing is delayed. |
| 1663 | */ |
Xiao Yang | 30ad63e | 2022-03-31 11:24:18 +0800 | [diff] [blame] | 1664 | IB_RAW_PACKET_CAP_DELAY_DROP = IB_UVERBS_RAW_PACKET_CAP_DELAY_DROP, |
Noa Osherovich | ebaaee2 | 2017-01-18 15:39:54 +0200 | [diff] [blame] | 1665 | }; |
| 1666 | |
Yishai Hadas | 5fd251c | 2016-05-23 15:20:48 +0300 | [diff] [blame] | 1667 | enum ib_wq_type { |
Yishai Hadas | 175ba58 | 2020-05-19 10:27:08 +0300 | [diff] [blame] | 1668 | IB_WQT_RQ = IB_UVERBS_WQT_RQ, |
Yishai Hadas | 5fd251c | 2016-05-23 15:20:48 +0300 | [diff] [blame] | 1669 | }; |
| 1670 | |
| 1671 | enum ib_wq_state { |
| 1672 | IB_WQS_RESET, |
| 1673 | IB_WQS_RDY, |
| 1674 | IB_WQS_ERR |
| 1675 | }; |
| 1676 | |
| 1677 | struct ib_wq { |
| 1678 | struct ib_device *device; |
Jason Gunthorpe | e04dd13 | 2020-01-08 19:22:01 +0200 | [diff] [blame] | 1679 | struct ib_uwq_object *uobject; |
Yishai Hadas | 5fd251c | 2016-05-23 15:20:48 +0300 | [diff] [blame] | 1680 | void *wq_context; |
| 1681 | void (*event_handler)(struct ib_event *, void *); |
| 1682 | struct ib_pd *pd; |
| 1683 | struct ib_cq *cq; |
| 1684 | u32 wq_num; |
| 1685 | enum ib_wq_state state; |
| 1686 | enum ib_wq_type wq_type; |
| 1687 | atomic_t usecnt; |
| 1688 | }; |
| 1689 | |
Noa Osherovich | 10bac72 | 2017-01-18 15:39:55 +0200 | [diff] [blame] | 1690 | enum ib_wq_flags { |
Yishai Hadas | 175ba58 | 2020-05-19 10:27:08 +0300 | [diff] [blame] | 1691 | IB_WQ_FLAGS_CVLAN_STRIPPING = IB_UVERBS_WQ_FLAGS_CVLAN_STRIPPING, |
| 1692 | IB_WQ_FLAGS_SCATTER_FCS = IB_UVERBS_WQ_FLAGS_SCATTER_FCS, |
| 1693 | IB_WQ_FLAGS_DELAY_DROP = IB_UVERBS_WQ_FLAGS_DELAY_DROP, |
| 1694 | IB_WQ_FLAGS_PCI_WRITE_END_PADDING = |
| 1695 | IB_UVERBS_WQ_FLAGS_PCI_WRITE_END_PADDING, |
Noa Osherovich | 10bac72 | 2017-01-18 15:39:55 +0200 | [diff] [blame] | 1696 | }; |
| 1697 | |
Yishai Hadas | 5fd251c | 2016-05-23 15:20:48 +0300 | [diff] [blame] | 1698 | struct ib_wq_init_attr { |
| 1699 | void *wq_context; |
| 1700 | enum ib_wq_type wq_type; |
| 1701 | u32 max_wr; |
| 1702 | u32 max_sge; |
| 1703 | struct ib_cq *cq; |
| 1704 | void (*event_handler)(struct ib_event *, void *); |
Noa Osherovich | 10bac72 | 2017-01-18 15:39:55 +0200 | [diff] [blame] | 1705 | u32 create_flags; /* Use enum ib_wq_flags */ |
Yishai Hadas | 5fd251c | 2016-05-23 15:20:48 +0300 | [diff] [blame] | 1706 | }; |
| 1707 | |
| 1708 | enum ib_wq_attr_mask { |
Noa Osherovich | 10bac72 | 2017-01-18 15:39:55 +0200 | [diff] [blame] | 1709 | IB_WQ_STATE = 1 << 0, |
| 1710 | IB_WQ_CUR_STATE = 1 << 1, |
| 1711 | IB_WQ_FLAGS = 1 << 2, |
Yishai Hadas | 5fd251c | 2016-05-23 15:20:48 +0300 | [diff] [blame] | 1712 | }; |
| 1713 | |
| 1714 | struct ib_wq_attr { |
| 1715 | enum ib_wq_state wq_state; |
| 1716 | enum ib_wq_state curr_wq_state; |
Noa Osherovich | 10bac72 | 2017-01-18 15:39:55 +0200 | [diff] [blame] | 1717 | u32 flags; /* Use enum ib_wq_flags */ |
| 1718 | u32 flags_mask; /* Use enum ib_wq_flags */ |
Yishai Hadas | 5fd251c | 2016-05-23 15:20:48 +0300 | [diff] [blame] | 1719 | }; |
| 1720 | |
Yishai Hadas | 6d39786b | 2016-05-23 15:20:51 +0300 | [diff] [blame] | 1721 | struct ib_rwq_ind_table { |
| 1722 | struct ib_device *device; |
| 1723 | struct ib_uobject *uobject; |
| 1724 | atomic_t usecnt; |
| 1725 | u32 ind_tbl_num; |
| 1726 | u32 log_ind_tbl_size; |
| 1727 | struct ib_wq **ind_tbl; |
| 1728 | }; |
| 1729 | |
| 1730 | struct ib_rwq_ind_table_init_attr { |
| 1731 | u32 log_ind_tbl_size; |
| 1732 | /* Each entry is a pointer to Receive Work Queue */ |
| 1733 | struct ib_wq **ind_tbl; |
| 1734 | }; |
| 1735 | |
Daniel Jurgens | d291f1a | 2017-05-19 15:48:52 +0300 | [diff] [blame] | 1736 | enum port_pkey_state { |
| 1737 | IB_PORT_PKEY_NOT_VALID = 0, |
| 1738 | IB_PORT_PKEY_VALID = 1, |
| 1739 | IB_PORT_PKEY_LISTED = 2, |
| 1740 | }; |
| 1741 | |
| 1742 | struct ib_qp_security; |
| 1743 | |
| 1744 | struct ib_port_pkey { |
| 1745 | enum port_pkey_state state; |
| 1746 | u16 pkey_index; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 1747 | u32 port_num; |
Daniel Jurgens | d291f1a | 2017-05-19 15:48:52 +0300 | [diff] [blame] | 1748 | struct list_head qp_list; |
| 1749 | struct list_head to_error_list; |
| 1750 | struct ib_qp_security *sec; |
| 1751 | }; |
| 1752 | |
| 1753 | struct ib_ports_pkeys { |
| 1754 | struct ib_port_pkey main; |
| 1755 | struct ib_port_pkey alt; |
| 1756 | }; |
| 1757 | |
| 1758 | struct ib_qp_security { |
| 1759 | struct ib_qp *qp; |
| 1760 | struct ib_device *dev; |
| 1761 | /* Hold this mutex when changing port and pkey settings. */ |
| 1762 | struct mutex mutex; |
| 1763 | struct ib_ports_pkeys *ports_pkeys; |
| 1764 | /* A list of all open shared QP handles. Required to enforce security |
| 1765 | * properly for all users of a shared QP. |
| 1766 | */ |
| 1767 | struct list_head shared_qp_list; |
| 1768 | void *security; |
| 1769 | bool destroying; |
| 1770 | atomic_t error_list_count; |
| 1771 | struct completion error_complete; |
| 1772 | int error_comps_pending; |
| 1773 | }; |
| 1774 | |
Bart Van Assche | 632bc3f | 2016-07-21 13:03:30 -0700 | [diff] [blame] | 1775 | /* |
| 1776 | * @max_write_sge: Maximum SGE elements per RDMA WRITE request. |
| 1777 | * @max_read_sge: Maximum SGE elements per RDMA READ request. |
| 1778 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1779 | struct ib_qp { |
| 1780 | struct ib_device *device; |
| 1781 | struct ib_pd *pd; |
| 1782 | struct ib_cq *send_cq; |
| 1783 | struct ib_cq *recv_cq; |
Christoph Hellwig | fffb038 | 2016-05-03 18:01:07 +0200 | [diff] [blame] | 1784 | spinlock_t mr_lock; |
| 1785 | int mrs_used; |
Christoph Hellwig | a060b56 | 2016-05-03 18:01:09 +0200 | [diff] [blame] | 1786 | struct list_head rdma_mrs; |
Christoph Hellwig | 0e353e3 | 2016-05-03 18:01:12 +0200 | [diff] [blame] | 1787 | struct list_head sig_mrs; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1788 | struct ib_srq *srq; |
Sean Hefty | b42b63c | 2011-05-23 19:59:25 -0700 | [diff] [blame] | 1789 | struct ib_xrcd *xrcd; /* XRC TGT QPs only */ |
Sean Hefty | d3d72d9 | 2011-05-26 23:06:44 -0700 | [diff] [blame] | 1790 | struct list_head xrcd_list; |
Christoph Hellwig | fffb038 | 2016-05-03 18:01:07 +0200 | [diff] [blame] | 1791 | |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1792 | /* count times opened, mcast attaches, flow attaches */ |
| 1793 | atomic_t usecnt; |
Sean Hefty | 0e0ec7e | 2011-08-08 15:31:51 -0700 | [diff] [blame] | 1794 | struct list_head open_list; |
| 1795 | struct ib_qp *real_qp; |
Jason Gunthorpe | 620d3f8 | 2020-01-08 19:21:59 +0200 | [diff] [blame] | 1796 | struct ib_uqp_object *uobject; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1797 | void (*event_handler)(struct ib_event *, void *); |
| 1798 | void *qp_context; |
Jason Gunthorpe | 1a1f460 | 2018-06-13 10:22:08 +0300 | [diff] [blame] | 1799 | /* sgid_attrs associated with the AV's */ |
| 1800 | const struct ib_gid_attr *av_sgid_attr; |
| 1801 | const struct ib_gid_attr *alt_path_sgid_attr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1802 | u32 qp_num; |
Bart Van Assche | 632bc3f | 2016-07-21 13:03:30 -0700 | [diff] [blame] | 1803 | u32 max_write_sge; |
| 1804 | u32 max_read_sge; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1805 | enum ib_qp_type qp_type; |
Yishai Hadas | a9017e2 | 2016-05-23 15:20:54 +0300 | [diff] [blame] | 1806 | struct ib_rwq_ind_table *rwq_ind_tbl; |
Daniel Jurgens | d291f1a | 2017-05-19 15:48:52 +0300 | [diff] [blame] | 1807 | struct ib_qp_security *qp_sec; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 1808 | u32 port; |
Leon Romanovsky | 02d8883 | 2018-01-28 11:17:20 +0200 | [diff] [blame] | 1809 | |
Max Gurtovoy | 185eddc | 2019-06-11 18:52:51 +0300 | [diff] [blame] | 1810 | bool integrity_en; |
Leon Romanovsky | 02d8883 | 2018-01-28 11:17:20 +0200 | [diff] [blame] | 1811 | /* |
| 1812 | * Implementation details of the RDMA core, don't use in drivers: |
| 1813 | */ |
| 1814 | struct rdma_restrack_entry res; |
Mark Zhang | 99fa331 | 2019-07-02 13:02:35 +0300 | [diff] [blame] | 1815 | |
| 1816 | /* The counter the qp is bind to */ |
| 1817 | struct rdma_counter *counter; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1818 | }; |
| 1819 | |
Ariel Levkovich | bee76d7 | 2018-04-05 18:53:24 +0300 | [diff] [blame] | 1820 | struct ib_dm { |
| 1821 | struct ib_device *device; |
| 1822 | u32 length; |
| 1823 | u32 flags; |
| 1824 | struct ib_uobject *uobject; |
| 1825 | atomic_t usecnt; |
| 1826 | }; |
| 1827 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1828 | struct ib_mr { |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1829 | struct ib_device *device; |
| 1830 | struct ib_pd *pd; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1831 | u32 lkey; |
| 1832 | u32 rkey; |
Sagi Grimberg | 4c67e2b | 2015-10-13 19:11:24 +0300 | [diff] [blame] | 1833 | u64 iova; |
Parav Pandit | edd3155 | 2017-09-24 21:46:31 +0300 | [diff] [blame] | 1834 | u64 length; |
Sagi Grimberg | 4c67e2b | 2015-10-13 19:11:24 +0300 | [diff] [blame] | 1835 | unsigned int page_size; |
Max Gurtovoy | a0bc099 | 2019-06-11 18:52:38 +0300 | [diff] [blame] | 1836 | enum ib_mr_type type; |
Steve Wise | d4a85c3 | 2016-05-03 18:01:08 +0200 | [diff] [blame] | 1837 | bool need_inval; |
Christoph Hellwig | fffb038 | 2016-05-03 18:01:07 +0200 | [diff] [blame] | 1838 | union { |
| 1839 | struct ib_uobject *uobject; /* user */ |
| 1840 | struct list_head qp_entry; /* FR */ |
| 1841 | }; |
Steve Wise | fccec5b | 2018-03-01 13:58:13 -0800 | [diff] [blame] | 1842 | |
Ariel Levkovich | be934cc | 2018-04-05 18:53:25 +0300 | [diff] [blame] | 1843 | struct ib_dm *dm; |
Max Gurtovoy | 7c717d3 | 2019-06-11 18:52:41 +0300 | [diff] [blame] | 1844 | struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */ |
Steve Wise | fccec5b | 2018-03-01 13:58:13 -0800 | [diff] [blame] | 1845 | /* |
| 1846 | * Implementation details of the RDMA core, don't use in drivers: |
| 1847 | */ |
| 1848 | struct rdma_restrack_entry res; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1849 | }; |
| 1850 | |
| 1851 | struct ib_mw { |
| 1852 | struct ib_device *device; |
| 1853 | struct ib_pd *pd; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 1854 | struct ib_uobject *uobject; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1855 | u32 rkey; |
Shani Michaeli | 7083e42 | 2013-02-06 16:19:12 +0000 | [diff] [blame] | 1856 | enum ib_mw_type type; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1857 | }; |
| 1858 | |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1859 | /* Supported steering options */ |
| 1860 | enum ib_flow_attr_type { |
| 1861 | /* steering according to rule specifications */ |
| 1862 | IB_FLOW_ATTR_NORMAL = 0x0, |
| 1863 | /* default unicast and multicast rule - |
| 1864 | * receive all Eth traffic which isn't steered to any QP |
| 1865 | */ |
| 1866 | IB_FLOW_ATTR_ALL_DEFAULT = 0x1, |
| 1867 | /* default multicast rule - |
| 1868 | * receive all Eth multicast traffic which isn't steered to any QP |
| 1869 | */ |
| 1870 | IB_FLOW_ATTR_MC_DEFAULT = 0x2, |
| 1871 | /* sniffer rule - receive all port traffic */ |
| 1872 | IB_FLOW_ATTR_SNIFFER = 0x3 |
| 1873 | }; |
| 1874 | |
| 1875 | /* Supported steering header types */ |
| 1876 | enum ib_flow_spec_type { |
| 1877 | /* L2 headers*/ |
Moses Reuben | 76bd23b | 2016-11-14 19:04:48 +0200 | [diff] [blame] | 1878 | IB_FLOW_SPEC_ETH = 0x20, |
| 1879 | IB_FLOW_SPEC_IB = 0x22, |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1880 | /* L3 header*/ |
Moses Reuben | 76bd23b | 2016-11-14 19:04:48 +0200 | [diff] [blame] | 1881 | IB_FLOW_SPEC_IPV4 = 0x30, |
| 1882 | IB_FLOW_SPEC_IPV6 = 0x31, |
Matan Barak | 56ab0b3 | 2018-03-28 09:27:49 +0300 | [diff] [blame] | 1883 | IB_FLOW_SPEC_ESP = 0x34, |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1884 | /* L4 headers*/ |
Moses Reuben | 76bd23b | 2016-11-14 19:04:48 +0200 | [diff] [blame] | 1885 | IB_FLOW_SPEC_TCP = 0x40, |
| 1886 | IB_FLOW_SPEC_UDP = 0x41, |
Moses Reuben | 0dbf333 | 2016-11-14 19:04:47 +0200 | [diff] [blame] | 1887 | IB_FLOW_SPEC_VXLAN_TUNNEL = 0x50, |
Ariel Levkovich | d90e5e5 | 2018-05-13 14:33:30 +0300 | [diff] [blame] | 1888 | IB_FLOW_SPEC_GRE = 0x51, |
Ariel Levkovich | b04f0f0 | 2018-05-13 14:33:32 +0300 | [diff] [blame] | 1889 | IB_FLOW_SPEC_MPLS = 0x60, |
Moses Reuben | fbf4686 | 2016-11-14 19:04:51 +0200 | [diff] [blame] | 1890 | IB_FLOW_SPEC_INNER = 0x100, |
Moses Reuben | 460d019 | 2017-01-18 14:59:48 +0200 | [diff] [blame] | 1891 | /* Actions */ |
| 1892 | IB_FLOW_SPEC_ACTION_TAG = 0x1000, |
Slava Shwartsman | 483a396 | 2017-04-03 13:13:51 +0300 | [diff] [blame] | 1893 | IB_FLOW_SPEC_ACTION_DROP = 0x1001, |
Matan Barak | 9b82844 | 2018-03-28 09:27:46 +0300 | [diff] [blame] | 1894 | IB_FLOW_SPEC_ACTION_HANDLE = 0x1002, |
Raed Salem | 7eea23a | 2018-05-31 16:43:36 +0300 | [diff] [blame] | 1895 | IB_FLOW_SPEC_ACTION_COUNT = 0x1003, |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1896 | }; |
Matan Barak | 240ae00 | 2013-11-07 15:25:13 +0200 | [diff] [blame] | 1897 | #define IB_FLOW_SPEC_LAYER_MASK 0xF0 |
Raed Salem | 7eea23a | 2018-05-31 16:43:36 +0300 | [diff] [blame] | 1898 | #define IB_FLOW_SPEC_SUPPORT_LAYERS 10 |
Matan Barak | 22878db | 2013-09-01 18:39:52 +0300 | [diff] [blame] | 1899 | |
Marina Varshaver | a3100a7 | 2016-02-18 18:31:05 +0200 | [diff] [blame] | 1900 | enum ib_flow_flags { |
| 1901 | IB_FLOW_ATTR_FLAGS_DONT_TRAP = 1UL << 1, /* Continue match, no steal */ |
Boris Pismenny | 21e82d3 | 2018-03-28 09:27:47 +0300 | [diff] [blame] | 1902 | IB_FLOW_ATTR_FLAGS_EGRESS = 1UL << 2, /* Egress flow */ |
| 1903 | IB_FLOW_ATTR_FLAGS_RESERVED = 1UL << 3 /* Must be last */ |
Marina Varshaver | a3100a7 | 2016-02-18 18:31:05 +0200 | [diff] [blame] | 1904 | }; |
| 1905 | |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1906 | struct ib_flow_eth_filter { |
| 1907 | u8 dst_mac[6]; |
| 1908 | u8 src_mac[6]; |
| 1909 | __be16 ether_type; |
| 1910 | __be16 vlan_tag; |
Maor Gottlieb | 15dfbd6 | 2016-08-30 16:58:32 +0300 | [diff] [blame] | 1911 | /* Must be last */ |
Gustavo A. R. Silva | 5b36132 | 2020-02-12 19:04:25 -0600 | [diff] [blame] | 1912 | u8 real_sz[]; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1913 | }; |
| 1914 | |
| 1915 | struct ib_flow_spec_eth { |
Moses Reuben | fbf4686 | 2016-11-14 19:04:51 +0200 | [diff] [blame] | 1916 | u32 type; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1917 | u16 size; |
| 1918 | struct ib_flow_eth_filter val; |
| 1919 | struct ib_flow_eth_filter mask; |
| 1920 | }; |
| 1921 | |
Matan Barak | 240ae00 | 2013-11-07 15:25:13 +0200 | [diff] [blame] | 1922 | struct ib_flow_ib_filter { |
| 1923 | __be16 dlid; |
| 1924 | __u8 sl; |
Maor Gottlieb | 15dfbd6 | 2016-08-30 16:58:32 +0300 | [diff] [blame] | 1925 | /* Must be last */ |
Gustavo A. R. Silva | 5b36132 | 2020-02-12 19:04:25 -0600 | [diff] [blame] | 1926 | u8 real_sz[]; |
Matan Barak | 240ae00 | 2013-11-07 15:25:13 +0200 | [diff] [blame] | 1927 | }; |
| 1928 | |
| 1929 | struct ib_flow_spec_ib { |
Moses Reuben | fbf4686 | 2016-11-14 19:04:51 +0200 | [diff] [blame] | 1930 | u32 type; |
Matan Barak | 240ae00 | 2013-11-07 15:25:13 +0200 | [diff] [blame] | 1931 | u16 size; |
| 1932 | struct ib_flow_ib_filter val; |
| 1933 | struct ib_flow_ib_filter mask; |
| 1934 | }; |
| 1935 | |
Maor Gottlieb | 989a3a8 | 2016-08-30 16:58:33 +0300 | [diff] [blame] | 1936 | /* IPv4 header flags */ |
| 1937 | enum ib_ipv4_flags { |
| 1938 | IB_IPV4_DONT_FRAG = 0x2, /* Don't enable packet fragmentation */ |
| 1939 | IB_IPV4_MORE_FRAG = 0X4 /* For All fragmented packets except the |
| 1940 | last have this flag set */ |
| 1941 | }; |
| 1942 | |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1943 | struct ib_flow_ipv4_filter { |
| 1944 | __be32 src_ip; |
| 1945 | __be32 dst_ip; |
Maor Gottlieb | 989a3a8 | 2016-08-30 16:58:33 +0300 | [diff] [blame] | 1946 | u8 proto; |
| 1947 | u8 tos; |
| 1948 | u8 ttl; |
| 1949 | u8 flags; |
Maor Gottlieb | 15dfbd6 | 2016-08-30 16:58:32 +0300 | [diff] [blame] | 1950 | /* Must be last */ |
Gustavo A. R. Silva | 5b36132 | 2020-02-12 19:04:25 -0600 | [diff] [blame] | 1951 | u8 real_sz[]; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1952 | }; |
| 1953 | |
| 1954 | struct ib_flow_spec_ipv4 { |
Moses Reuben | fbf4686 | 2016-11-14 19:04:51 +0200 | [diff] [blame] | 1955 | u32 type; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1956 | u16 size; |
| 1957 | struct ib_flow_ipv4_filter val; |
| 1958 | struct ib_flow_ipv4_filter mask; |
| 1959 | }; |
| 1960 | |
Maor Gottlieb | 4c2aae7 | 2016-06-17 15:14:50 +0300 | [diff] [blame] | 1961 | struct ib_flow_ipv6_filter { |
| 1962 | u8 src_ip[16]; |
| 1963 | u8 dst_ip[16]; |
Maor Gottlieb | a72c6a2 | 2016-08-30 16:58:34 +0300 | [diff] [blame] | 1964 | __be32 flow_label; |
| 1965 | u8 next_hdr; |
| 1966 | u8 traffic_class; |
| 1967 | u8 hop_limit; |
Maor Gottlieb | 15dfbd6 | 2016-08-30 16:58:32 +0300 | [diff] [blame] | 1968 | /* Must be last */ |
Gustavo A. R. Silva | 5b36132 | 2020-02-12 19:04:25 -0600 | [diff] [blame] | 1969 | u8 real_sz[]; |
Maor Gottlieb | 4c2aae7 | 2016-06-17 15:14:50 +0300 | [diff] [blame] | 1970 | }; |
| 1971 | |
| 1972 | struct ib_flow_spec_ipv6 { |
Moses Reuben | fbf4686 | 2016-11-14 19:04:51 +0200 | [diff] [blame] | 1973 | u32 type; |
Maor Gottlieb | 4c2aae7 | 2016-06-17 15:14:50 +0300 | [diff] [blame] | 1974 | u16 size; |
| 1975 | struct ib_flow_ipv6_filter val; |
| 1976 | struct ib_flow_ipv6_filter mask; |
| 1977 | }; |
| 1978 | |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1979 | struct ib_flow_tcp_udp_filter { |
| 1980 | __be16 dst_port; |
| 1981 | __be16 src_port; |
Maor Gottlieb | 15dfbd6 | 2016-08-30 16:58:32 +0300 | [diff] [blame] | 1982 | /* Must be last */ |
Gustavo A. R. Silva | 5b36132 | 2020-02-12 19:04:25 -0600 | [diff] [blame] | 1983 | u8 real_sz[]; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1984 | }; |
| 1985 | |
| 1986 | struct ib_flow_spec_tcp_udp { |
Moses Reuben | fbf4686 | 2016-11-14 19:04:51 +0200 | [diff] [blame] | 1987 | u32 type; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 1988 | u16 size; |
| 1989 | struct ib_flow_tcp_udp_filter val; |
| 1990 | struct ib_flow_tcp_udp_filter mask; |
| 1991 | }; |
| 1992 | |
Moses Reuben | 0dbf333 | 2016-11-14 19:04:47 +0200 | [diff] [blame] | 1993 | struct ib_flow_tunnel_filter { |
| 1994 | __be32 tunnel_id; |
Gustavo A. R. Silva | 5b36132 | 2020-02-12 19:04:25 -0600 | [diff] [blame] | 1995 | u8 real_sz[]; |
Moses Reuben | 0dbf333 | 2016-11-14 19:04:47 +0200 | [diff] [blame] | 1996 | }; |
| 1997 | |
| 1998 | /* ib_flow_spec_tunnel describes the Vxlan tunnel |
| 1999 | * the tunnel_id from val has the vni value |
| 2000 | */ |
| 2001 | struct ib_flow_spec_tunnel { |
Moses Reuben | fbf4686 | 2016-11-14 19:04:51 +0200 | [diff] [blame] | 2002 | u32 type; |
Moses Reuben | 0dbf333 | 2016-11-14 19:04:47 +0200 | [diff] [blame] | 2003 | u16 size; |
| 2004 | struct ib_flow_tunnel_filter val; |
| 2005 | struct ib_flow_tunnel_filter mask; |
| 2006 | }; |
| 2007 | |
Matan Barak | 56ab0b3 | 2018-03-28 09:27:49 +0300 | [diff] [blame] | 2008 | struct ib_flow_esp_filter { |
| 2009 | __be32 spi; |
| 2010 | __be32 seq; |
| 2011 | /* Must be last */ |
Gustavo A. R. Silva | 5b36132 | 2020-02-12 19:04:25 -0600 | [diff] [blame] | 2012 | u8 real_sz[]; |
Matan Barak | 56ab0b3 | 2018-03-28 09:27:49 +0300 | [diff] [blame] | 2013 | }; |
| 2014 | |
| 2015 | struct ib_flow_spec_esp { |
| 2016 | u32 type; |
| 2017 | u16 size; |
| 2018 | struct ib_flow_esp_filter val; |
| 2019 | struct ib_flow_esp_filter mask; |
| 2020 | }; |
| 2021 | |
Ariel Levkovich | d90e5e5 | 2018-05-13 14:33:30 +0300 | [diff] [blame] | 2022 | struct ib_flow_gre_filter { |
| 2023 | __be16 c_ks_res0_ver; |
| 2024 | __be16 protocol; |
| 2025 | __be32 key; |
| 2026 | /* Must be last */ |
Gustavo A. R. Silva | 5b36132 | 2020-02-12 19:04:25 -0600 | [diff] [blame] | 2027 | u8 real_sz[]; |
Ariel Levkovich | d90e5e5 | 2018-05-13 14:33:30 +0300 | [diff] [blame] | 2028 | }; |
| 2029 | |
| 2030 | struct ib_flow_spec_gre { |
| 2031 | u32 type; |
| 2032 | u16 size; |
| 2033 | struct ib_flow_gre_filter val; |
| 2034 | struct ib_flow_gre_filter mask; |
| 2035 | }; |
| 2036 | |
Ariel Levkovich | b04f0f0 | 2018-05-13 14:33:32 +0300 | [diff] [blame] | 2037 | struct ib_flow_mpls_filter { |
| 2038 | __be32 tag; |
| 2039 | /* Must be last */ |
Gustavo A. R. Silva | 5b36132 | 2020-02-12 19:04:25 -0600 | [diff] [blame] | 2040 | u8 real_sz[]; |
Ariel Levkovich | b04f0f0 | 2018-05-13 14:33:32 +0300 | [diff] [blame] | 2041 | }; |
| 2042 | |
| 2043 | struct ib_flow_spec_mpls { |
| 2044 | u32 type; |
| 2045 | u16 size; |
| 2046 | struct ib_flow_mpls_filter val; |
| 2047 | struct ib_flow_mpls_filter mask; |
| 2048 | }; |
| 2049 | |
Moses Reuben | 460d019 | 2017-01-18 14:59:48 +0200 | [diff] [blame] | 2050 | struct ib_flow_spec_action_tag { |
| 2051 | enum ib_flow_spec_type type; |
| 2052 | u16 size; |
| 2053 | u32 tag_id; |
| 2054 | }; |
| 2055 | |
Slava Shwartsman | 483a396 | 2017-04-03 13:13:51 +0300 | [diff] [blame] | 2056 | struct ib_flow_spec_action_drop { |
| 2057 | enum ib_flow_spec_type type; |
| 2058 | u16 size; |
| 2059 | }; |
| 2060 | |
Matan Barak | 9b82844 | 2018-03-28 09:27:46 +0300 | [diff] [blame] | 2061 | struct ib_flow_spec_action_handle { |
| 2062 | enum ib_flow_spec_type type; |
| 2063 | u16 size; |
| 2064 | struct ib_flow_action *act; |
| 2065 | }; |
| 2066 | |
Raed Salem | 7eea23a | 2018-05-31 16:43:36 +0300 | [diff] [blame] | 2067 | enum ib_counters_description { |
| 2068 | IB_COUNTER_PACKETS, |
| 2069 | IB_COUNTER_BYTES, |
| 2070 | }; |
| 2071 | |
| 2072 | struct ib_flow_spec_action_count { |
| 2073 | enum ib_flow_spec_type type; |
| 2074 | u16 size; |
| 2075 | struct ib_counters *counters; |
| 2076 | }; |
| 2077 | |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 2078 | union ib_flow_spec { |
| 2079 | struct { |
Moses Reuben | fbf4686 | 2016-11-14 19:04:51 +0200 | [diff] [blame] | 2080 | u32 type; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 2081 | u16 size; |
| 2082 | }; |
| 2083 | struct ib_flow_spec_eth eth; |
Matan Barak | 240ae00 | 2013-11-07 15:25:13 +0200 | [diff] [blame] | 2084 | struct ib_flow_spec_ib ib; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 2085 | struct ib_flow_spec_ipv4 ipv4; |
| 2086 | struct ib_flow_spec_tcp_udp tcp_udp; |
Maor Gottlieb | 4c2aae7 | 2016-06-17 15:14:50 +0300 | [diff] [blame] | 2087 | struct ib_flow_spec_ipv6 ipv6; |
Moses Reuben | 0dbf333 | 2016-11-14 19:04:47 +0200 | [diff] [blame] | 2088 | struct ib_flow_spec_tunnel tunnel; |
Matan Barak | 56ab0b3 | 2018-03-28 09:27:49 +0300 | [diff] [blame] | 2089 | struct ib_flow_spec_esp esp; |
Ariel Levkovich | d90e5e5 | 2018-05-13 14:33:30 +0300 | [diff] [blame] | 2090 | struct ib_flow_spec_gre gre; |
Ariel Levkovich | b04f0f0 | 2018-05-13 14:33:32 +0300 | [diff] [blame] | 2091 | struct ib_flow_spec_mpls mpls; |
Moses Reuben | 460d019 | 2017-01-18 14:59:48 +0200 | [diff] [blame] | 2092 | struct ib_flow_spec_action_tag flow_tag; |
Slava Shwartsman | 483a396 | 2017-04-03 13:13:51 +0300 | [diff] [blame] | 2093 | struct ib_flow_spec_action_drop drop; |
Matan Barak | 9b82844 | 2018-03-28 09:27:46 +0300 | [diff] [blame] | 2094 | struct ib_flow_spec_action_handle action; |
Raed Salem | 7eea23a | 2018-05-31 16:43:36 +0300 | [diff] [blame] | 2095 | struct ib_flow_spec_action_count flow_count; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 2096 | }; |
| 2097 | |
| 2098 | struct ib_flow_attr { |
| 2099 | enum ib_flow_attr_type type; |
| 2100 | u16 size; |
| 2101 | u16 priority; |
| 2102 | u32 flags; |
| 2103 | u8 num_of_specs; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2104 | u32 port; |
Matthew Wilcox | 7654cb1 | 2018-06-07 07:57:16 -0700 | [diff] [blame] | 2105 | union ib_flow_spec flows[]; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 2106 | }; |
| 2107 | |
| 2108 | struct ib_flow { |
| 2109 | struct ib_qp *qp; |
Yishai Hadas | 6cd080a | 2018-07-23 15:25:08 +0300 | [diff] [blame] | 2110 | struct ib_device *device; |
Hadar Hen Zion | 319a441 | 2013-08-07 14:01:59 +0300 | [diff] [blame] | 2111 | struct ib_uobject *uobject; |
| 2112 | }; |
| 2113 | |
Matan Barak | 2eb9bea | 2018-03-28 09:27:45 +0300 | [diff] [blame] | 2114 | enum ib_flow_action_type { |
| 2115 | IB_FLOW_ACTION_UNSPECIFIED, |
| 2116 | IB_FLOW_ACTION_ESP = 1, |
| 2117 | }; |
| 2118 | |
| 2119 | struct ib_flow_action_attrs_esp_keymats { |
| 2120 | enum ib_uverbs_flow_action_esp_keymat protocol; |
| 2121 | union { |
| 2122 | struct ib_uverbs_flow_action_esp_keymat_aes_gcm aes_gcm; |
| 2123 | } keymat; |
| 2124 | }; |
| 2125 | |
| 2126 | struct ib_flow_action_attrs_esp_replays { |
| 2127 | enum ib_uverbs_flow_action_esp_replay protocol; |
| 2128 | union { |
| 2129 | struct ib_uverbs_flow_action_esp_replay_bmp bmp; |
| 2130 | } replay; |
| 2131 | }; |
| 2132 | |
| 2133 | enum ib_flow_action_attrs_esp_flags { |
| 2134 | /* All user-space flags at the top: Use enum ib_uverbs_flow_action_esp_flags |
| 2135 | * This is done in order to share the same flags between user-space and |
| 2136 | * kernel and spare an unnecessary translation. |
| 2137 | */ |
| 2138 | |
| 2139 | /* Kernel flags */ |
| 2140 | IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED = 1ULL << 32, |
Matan Barak | 7d12f8d | 2018-03-28 09:27:48 +0300 | [diff] [blame] | 2141 | IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS = 1ULL << 33, |
Matan Barak | 2eb9bea | 2018-03-28 09:27:45 +0300 | [diff] [blame] | 2142 | }; |
| 2143 | |
| 2144 | struct ib_flow_spec_list { |
| 2145 | struct ib_flow_spec_list *next; |
| 2146 | union ib_flow_spec spec; |
| 2147 | }; |
| 2148 | |
| 2149 | struct ib_flow_action_attrs_esp { |
| 2150 | struct ib_flow_action_attrs_esp_keymats *keymat; |
| 2151 | struct ib_flow_action_attrs_esp_replays *replay; |
| 2152 | struct ib_flow_spec_list *encap; |
| 2153 | /* Used only if IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED is enabled. |
| 2154 | * Value of 0 is a valid value. |
| 2155 | */ |
| 2156 | u32 esn; |
| 2157 | u32 spi; |
| 2158 | u32 seq; |
| 2159 | u32 tfc_pad; |
| 2160 | /* Use enum ib_flow_action_attrs_esp_flags */ |
| 2161 | u64 flags; |
| 2162 | u64 hard_limit_pkts; |
| 2163 | }; |
| 2164 | |
| 2165 | struct ib_flow_action { |
| 2166 | struct ib_device *device; |
| 2167 | struct ib_uobject *uobject; |
| 2168 | enum ib_flow_action_type type; |
| 2169 | atomic_t usecnt; |
| 2170 | }; |
| 2171 | |
Leon Romanovsky | e26e7b8 | 2019-10-29 08:27:45 +0200 | [diff] [blame] | 2172 | struct ib_mad; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2173 | |
| 2174 | enum ib_process_mad_flags { |
| 2175 | IB_MAD_IGNORE_MKEY = 1, |
| 2176 | IB_MAD_IGNORE_BKEY = 2, |
| 2177 | IB_MAD_IGNORE_ALL = IB_MAD_IGNORE_MKEY | IB_MAD_IGNORE_BKEY |
| 2178 | }; |
| 2179 | |
| 2180 | enum ib_mad_result { |
| 2181 | IB_MAD_RESULT_FAILURE = 0, /* (!SUCCESS is the important flag) */ |
| 2182 | IB_MAD_RESULT_SUCCESS = 1 << 0, /* MAD was successfully processed */ |
| 2183 | IB_MAD_RESULT_REPLY = 1 << 1, /* Reply packet needs to be sent */ |
| 2184 | IB_MAD_RESULT_CONSUMED = 1 << 2 /* Packet consumed: stop processing */ |
| 2185 | }; |
| 2186 | |
Jack Wang | 21d6454 | 2017-01-17 10:11:12 +0100 | [diff] [blame] | 2187 | struct ib_port_cache { |
Daniel Jurgens | 883c71f | 2017-05-19 15:48:51 +0300 | [diff] [blame] | 2188 | u64 subnet_prefix; |
Jack Wang | 21d6454 | 2017-01-17 10:11:12 +0100 | [diff] [blame] | 2189 | struct ib_pkey_cache *pkey; |
| 2190 | struct ib_gid_table *gid; |
| 2191 | u8 lmc; |
| 2192 | enum ib_port_state port_state; |
| 2193 | }; |
| 2194 | |
Ira Weiny | 7738613 | 2015-05-13 20:02:58 -0400 | [diff] [blame] | 2195 | struct ib_port_immutable { |
| 2196 | int pkey_tbl_len; |
| 2197 | int gid_tbl_len; |
Ira Weiny | f9b22e3 | 2015-05-13 20:02:59 -0400 | [diff] [blame] | 2198 | u32 core_cap_flags; |
Ira Weiny | 337877a | 2015-06-06 14:38:29 -0400 | [diff] [blame] | 2199 | u32 max_mad_size; |
Ira Weiny | 7738613 | 2015-05-13 20:02:58 -0400 | [diff] [blame] | 2200 | }; |
| 2201 | |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 2202 | struct ib_port_data { |
Jason Gunthorpe | 324e227 | 2019-02-12 21:12:51 -0700 | [diff] [blame] | 2203 | struct ib_device *ib_dev; |
| 2204 | |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 2205 | struct ib_port_immutable immutable; |
| 2206 | |
| 2207 | spinlock_t pkey_list_lock; |
Anand Khoje | 84dcd8c | 2021-06-16 21:15:08 +0530 | [diff] [blame] | 2208 | |
| 2209 | spinlock_t netdev_lock; |
| 2210 | |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 2211 | struct list_head pkey_list; |
Jason Gunthorpe | 8faea9f | 2019-02-12 21:12:49 -0700 | [diff] [blame] | 2212 | |
| 2213 | struct ib_port_cache cache; |
Jason Gunthorpe | c2261dd | 2019-02-12 21:12:50 -0700 | [diff] [blame] | 2214 | |
Jason Gunthorpe | 324e227 | 2019-02-12 21:12:51 -0700 | [diff] [blame] | 2215 | struct net_device __rcu *netdev; |
Jason Gunthorpe | 09f530f0 | 2022-11-23 20:27:14 -0400 | [diff] [blame] | 2216 | netdevice_tracker netdev_tracker; |
Jason Gunthorpe | 324e227 | 2019-02-12 21:12:51 -0700 | [diff] [blame] | 2217 | struct hlist_node ndev_hash_link; |
Mark Zhang | 413d334 | 2019-07-02 13:02:34 +0300 | [diff] [blame] | 2218 | struct rdma_port_counter port_counter; |
Jason Gunthorpe | d8a5883 | 2021-06-11 19:00:21 +0300 | [diff] [blame] | 2219 | struct ib_port *sysfs; |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 2220 | }; |
| 2221 | |
Vishwanathapura, Niranjana | 2fc7757 | 2017-04-12 20:29:20 -0700 | [diff] [blame] | 2222 | /* rdma netdev type - specifies protocol type */ |
| 2223 | enum rdma_netdev_t { |
Niranjana Vishwanathapura | f0ad83ac | 2017-04-10 11:22:25 +0300 | [diff] [blame] | 2224 | RDMA_NETDEV_OPA_VNIC, |
| 2225 | RDMA_NETDEV_IPOIB, |
Vishwanathapura, Niranjana | 2fc7757 | 2017-04-12 20:29:20 -0700 | [diff] [blame] | 2226 | }; |
| 2227 | |
| 2228 | /** |
| 2229 | * struct rdma_netdev - rdma netdev |
| 2230 | * For cases where netstack interfacing is required. |
| 2231 | */ |
| 2232 | struct rdma_netdev { |
| 2233 | void *clnt_priv; |
| 2234 | struct ib_device *hca; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2235 | u32 port_num; |
Gary Leshner | d99dc60 | 2020-05-11 12:05:48 -0400 | [diff] [blame] | 2236 | int mtu; |
Vishwanathapura, Niranjana | 2fc7757 | 2017-04-12 20:29:20 -0700 | [diff] [blame] | 2237 | |
Jason Gunthorpe | 9f49a5b | 2018-07-29 11:34:56 +0300 | [diff] [blame] | 2238 | /* |
| 2239 | * cleanup function must be specified. |
| 2240 | * FIXME: This is only used for OPA_VNIC and that usage should be |
| 2241 | * removed too. |
| 2242 | */ |
Niranjana Vishwanathapura | 8e95960 | 2017-06-30 13:14:46 -0700 | [diff] [blame] | 2243 | void (*free_rdma_netdev)(struct net_device *netdev); |
| 2244 | |
Vishwanathapura, Niranjana | 2fc7757 | 2017-04-12 20:29:20 -0700 | [diff] [blame] | 2245 | /* control functions */ |
| 2246 | void (*set_id)(struct net_device *netdev, int id); |
Niranjana Vishwanathapura | f0ad83ac | 2017-04-10 11:22:25 +0300 | [diff] [blame] | 2247 | /* send packet */ |
| 2248 | int (*send)(struct net_device *dev, struct sk_buff *skb, |
| 2249 | struct ib_ah *address, u32 dqpn); |
| 2250 | /* multicast */ |
| 2251 | int (*attach_mcast)(struct net_device *dev, struct ib_device *hca, |
| 2252 | union ib_gid *gid, u16 mlid, |
| 2253 | int set_qkey, u32 qkey); |
| 2254 | int (*detach_mcast)(struct net_device *dev, struct ib_device *hca, |
| 2255 | union ib_gid *gid, u16 mlid); |
Mike Marciniszyn | 042a00f | 2021-03-29 09:54:08 -0400 | [diff] [blame] | 2256 | /* timeout */ |
| 2257 | void (*tx_timeout)(struct net_device *dev, unsigned int txqueue); |
Vishwanathapura, Niranjana | 2fc7757 | 2017-04-12 20:29:20 -0700 | [diff] [blame] | 2258 | }; |
| 2259 | |
Denis Drozdov | f6a8a19 | 2018-08-14 14:08:51 +0300 | [diff] [blame] | 2260 | struct rdma_netdev_alloc_params { |
| 2261 | size_t sizeof_priv; |
| 2262 | unsigned int txqs; |
| 2263 | unsigned int rxqs; |
| 2264 | void *param; |
| 2265 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2266 | int (*initialize_rdma_netdev)(struct ib_device *device, u32 port_num, |
Denis Drozdov | f6a8a19 | 2018-08-14 14:08:51 +0300 | [diff] [blame] | 2267 | struct net_device *netdev, void *param); |
| 2268 | }; |
| 2269 | |
Erez Alfasi | a3de94e | 2019-10-16 09:23:05 +0300 | [diff] [blame] | 2270 | struct ib_odp_counters { |
| 2271 | atomic64_t faults; |
| 2272 | atomic64_t invalidations; |
Maor Gottlieb | d473f4d | 2020-06-21 13:41:47 +0300 | [diff] [blame] | 2273 | atomic64_t prefetch; |
Erez Alfasi | a3de94e | 2019-10-16 09:23:05 +0300 | [diff] [blame] | 2274 | }; |
| 2275 | |
Raed Salem | fa9b180 | 2018-05-31 16:43:31 +0300 | [diff] [blame] | 2276 | struct ib_counters { |
| 2277 | struct ib_device *device; |
| 2278 | struct ib_uobject *uobject; |
| 2279 | /* num of objects attached */ |
| 2280 | atomic_t usecnt; |
| 2281 | }; |
| 2282 | |
Raed Salem | 51d7a53 | 2018-05-31 16:43:33 +0300 | [diff] [blame] | 2283 | struct ib_counters_read_attr { |
| 2284 | u64 *counters_buff; |
| 2285 | u32 ncounters; |
| 2286 | u32 flags; /* use enum ib_read_counters_flags */ |
| 2287 | }; |
| 2288 | |
Matan Barak | 2eb9bea | 2018-03-28 09:27:45 +0300 | [diff] [blame] | 2289 | struct uverbs_attr_bundle; |
Kamal Heib | dd05cb8 | 2019-04-29 14:59:06 +0300 | [diff] [blame] | 2290 | struct iw_cm_id; |
| 2291 | struct iw_cm_conn_param; |
Matan Barak | 2eb9bea | 2018-03-28 09:27:45 +0300 | [diff] [blame] | 2292 | |
Leon Romanovsky | 30471d4 | 2019-02-03 14:55:50 +0200 | [diff] [blame] | 2293 | #define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member) \ |
| 2294 | .size_##ib_struct = \ |
| 2295 | (sizeof(struct drv_struct) + \ |
| 2296 | BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) + \ |
| 2297 | BUILD_BUG_ON_ZERO( \ |
| 2298 | !__same_type(((struct drv_struct *)NULL)->member, \ |
| 2299 | struct ib_struct))) |
| 2300 | |
Leon Romanovsky | 514aee6 | 2021-07-23 14:39:50 +0300 | [diff] [blame] | 2301 | #define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp) \ |
| 2302 | ((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \ |
| 2303 | gfp, false)) |
| 2304 | |
| 2305 | #define rdma_zalloc_drv_obj_numa(ib_dev, ib_type) \ |
| 2306 | ((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \ |
| 2307 | GFP_KERNEL, true)) |
Leon Romanovsky | f631603 | 2019-03-28 15:12:58 +0200 | [diff] [blame] | 2308 | |
Leon Romanovsky | 30471d4 | 2019-02-03 14:55:50 +0200 | [diff] [blame] | 2309 | #define rdma_zalloc_drv_obj(ib_dev, ib_type) \ |
Leon Romanovsky | f631603 | 2019-03-28 15:12:58 +0200 | [diff] [blame] | 2310 | rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, GFP_KERNEL) |
Leon Romanovsky | 30471d4 | 2019-02-03 14:55:50 +0200 | [diff] [blame] | 2311 | |
| 2312 | #define DECLARE_RDMA_OBJ_SIZE(ib_struct) size_t size_##ib_struct |
| 2313 | |
Michal Kalderon | 3411f9f | 2019-10-30 11:44:11 +0200 | [diff] [blame] | 2314 | struct rdma_user_mmap_entry { |
| 2315 | struct kref ref; |
| 2316 | struct ib_ucontext *ucontext; |
| 2317 | unsigned long start_pgoff; |
| 2318 | size_t npages; |
| 2319 | bool driver_removed; |
| 2320 | }; |
| 2321 | |
| 2322 | /* Return the offset (in bytes) the user should pass to libc's mmap() */ |
| 2323 | static inline u64 |
| 2324 | rdma_user_mmap_get_offset(const struct rdma_user_mmap_entry *entry) |
| 2325 | { |
| 2326 | return (u64)entry->start_pgoff << PAGE_SHIFT; |
| 2327 | } |
| 2328 | |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2329 | /** |
| 2330 | * struct ib_device_ops - InfiniBand device operations |
| 2331 | * This structure defines all the InfiniBand device operations, providers will |
| 2332 | * need to define the supported operations, otherwise they will be set to null. |
| 2333 | */ |
| 2334 | struct ib_device_ops { |
Jason Gunthorpe | 7a15414 | 2019-06-05 14:39:26 -0300 | [diff] [blame] | 2335 | struct module *owner; |
Jason Gunthorpe | b9560a4 | 2019-06-05 14:39:24 -0300 | [diff] [blame] | 2336 | enum rdma_driver_id driver_id; |
Jason Gunthorpe | 72c6ec1 | 2019-06-05 14:39:25 -0300 | [diff] [blame] | 2337 | u32 uverbs_abi_ver; |
Jason Gunthorpe | 8f71bb0 | 2019-06-13 21:38:19 -0300 | [diff] [blame] | 2338 | unsigned int uverbs_no_driver_id_binding:1; |
Jason Gunthorpe | b9560a4 | 2019-06-05 14:39:24 -0300 | [diff] [blame] | 2339 | |
Jason Gunthorpe | 915e4af | 2021-06-11 19:00:34 +0300 | [diff] [blame] | 2340 | /* |
| 2341 | * NOTE: New drivers should not make use of device_group; instead new |
| 2342 | * device parameter should be exposed via netlink command. This |
| 2343 | * mechanism exists only for existing drivers. |
| 2344 | */ |
| 2345 | const struct attribute_group *device_group; |
Jason Gunthorpe | d7407d1 | 2021-06-11 19:00:32 +0300 | [diff] [blame] | 2346 | const struct attribute_group **port_groups; |
| 2347 | |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2348 | int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr, |
| 2349 | const struct ib_send_wr **bad_send_wr); |
| 2350 | int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, |
| 2351 | const struct ib_recv_wr **bad_recv_wr); |
| 2352 | void (*drain_rq)(struct ib_qp *qp); |
| 2353 | void (*drain_sq)(struct ib_qp *qp); |
| 2354 | int (*poll_cq)(struct ib_cq *cq, int num_entries, struct ib_wc *wc); |
| 2355 | int (*peek_cq)(struct ib_cq *cq, int wc_cnt); |
| 2356 | int (*req_notify_cq)(struct ib_cq *cq, enum ib_cq_notify_flags flags); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2357 | int (*post_srq_recv)(struct ib_srq *srq, |
| 2358 | const struct ib_recv_wr *recv_wr, |
| 2359 | const struct ib_recv_wr **bad_recv_wr); |
| 2360 | int (*process_mad)(struct ib_device *device, int process_mad_flags, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2361 | u32 port_num, const struct ib_wc *in_wc, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2362 | const struct ib_grh *in_grh, |
Leon Romanovsky | e26e7b8 | 2019-10-29 08:27:45 +0200 | [diff] [blame] | 2363 | const struct ib_mad *in_mad, struct ib_mad *out_mad, |
| 2364 | size_t *out_mad_size, u16 *out_mad_pkey_index); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2365 | int (*query_device)(struct ib_device *device, |
| 2366 | struct ib_device_attr *device_attr, |
| 2367 | struct ib_udata *udata); |
| 2368 | int (*modify_device)(struct ib_device *device, int device_modify_mask, |
| 2369 | struct ib_device_modify *device_modify); |
| 2370 | void (*get_dev_fw_str)(struct ib_device *device, char *str); |
| 2371 | const struct cpumask *(*get_vector_affinity)(struct ib_device *ibdev, |
| 2372 | int comp_vector); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2373 | int (*query_port)(struct ib_device *device, u32 port_num, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2374 | struct ib_port_attr *port_attr); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2375 | int (*modify_port)(struct ib_device *device, u32 port_num, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2376 | int port_modify_mask, |
| 2377 | struct ib_port_modify *port_modify); |
| 2378 | /** |
| 2379 | * The following mandatory functions are used only at device |
| 2380 | * registration. Keep functions such as these at the end of this |
| 2381 | * structure to avoid cache line misses when accessing struct ib_device |
| 2382 | * in fast paths. |
| 2383 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2384 | int (*get_port_immutable)(struct ib_device *device, u32 port_num, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2385 | struct ib_port_immutable *immutable); |
| 2386 | enum rdma_link_layer (*get_link_layer)(struct ib_device *device, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2387 | u32 port_num); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2388 | /** |
| 2389 | * When calling get_netdev, the HW vendor's driver should return the |
| 2390 | * net device of device @device at port @port_num or NULL if such |
| 2391 | * a net device doesn't exist. The vendor driver should call dev_hold |
| 2392 | * on this net device. The HW vendor's device driver must guarantee |
| 2393 | * that this function returns NULL before the net device has finished |
| 2394 | * NETDEV_UNREGISTER state. |
| 2395 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2396 | struct net_device *(*get_netdev)(struct ib_device *device, |
| 2397 | u32 port_num); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2398 | /** |
| 2399 | * rdma netdev operation |
| 2400 | * |
| 2401 | * Driver implementing alloc_rdma_netdev or rdma_netdev_get_params |
| 2402 | * must return -EOPNOTSUPP if it doesn't support the specified type. |
| 2403 | */ |
| 2404 | struct net_device *(*alloc_rdma_netdev)( |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2405 | struct ib_device *device, u32 port_num, enum rdma_netdev_t type, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2406 | const char *name, unsigned char name_assign_type, |
| 2407 | void (*setup)(struct net_device *)); |
| 2408 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2409 | int (*rdma_netdev_get_params)(struct ib_device *device, u32 port_num, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2410 | enum rdma_netdev_t type, |
| 2411 | struct rdma_netdev_alloc_params *params); |
| 2412 | /** |
| 2413 | * query_gid should be return GID value for @device, when @port_num |
| 2414 | * link layer is either IB or iWarp. It is no-op if @port_num port |
| 2415 | * is RoCE link layer. |
| 2416 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2417 | int (*query_gid)(struct ib_device *device, u32 port_num, int index, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2418 | union ib_gid *gid); |
| 2419 | /** |
| 2420 | * When calling add_gid, the HW vendor's driver should add the gid |
| 2421 | * of device of port at gid index available at @attr. Meta-info of |
| 2422 | * that gid (for example, the network device related to this gid) is |
| 2423 | * available at @attr. @context allows the HW vendor driver to store |
| 2424 | * extra information together with a GID entry. The HW vendor driver may |
| 2425 | * allocate memory to contain this information and store it in @context |
| 2426 | * when a new GID entry is written to. Params are consistent until the |
| 2427 | * next call of add_gid or delete_gid. The function should return 0 on |
| 2428 | * success or error otherwise. The function could be called |
| 2429 | * concurrently for different ports. This function is only called when |
| 2430 | * roce_gid_table is used. |
| 2431 | */ |
| 2432 | int (*add_gid)(const struct ib_gid_attr *attr, void **context); |
| 2433 | /** |
| 2434 | * When calling del_gid, the HW vendor's driver should delete the |
| 2435 | * gid of device @device at gid index gid_index of port port_num |
| 2436 | * available in @attr. |
| 2437 | * Upon the deletion of a GID entry, the HW vendor must free any |
| 2438 | * allocated memory. The caller will clear @context afterwards. |
| 2439 | * This function is only called when roce_gid_table is used. |
| 2440 | */ |
| 2441 | int (*del_gid)(const struct ib_gid_attr *attr, void **context); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2442 | int (*query_pkey)(struct ib_device *device, u32 port_num, u16 index, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2443 | u16 *pkey); |
Leon Romanovsky | a2a074e | 2019-02-12 20:39:16 +0200 | [diff] [blame] | 2444 | int (*alloc_ucontext)(struct ib_ucontext *context, |
| 2445 | struct ib_udata *udata); |
| 2446 | void (*dealloc_ucontext)(struct ib_ucontext *context); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2447 | int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma); |
Michal Kalderon | 3411f9f | 2019-10-30 11:44:11 +0200 | [diff] [blame] | 2448 | /** |
| 2449 | * This will be called once refcount of an entry in mmap_xa reaches |
| 2450 | * zero. The type of the memory that was mapped may differ between |
| 2451 | * entries and is opaque to the rdma_user_mmap interface. |
| 2452 | * Therefore needs to be implemented by the driver in mmap_free. |
| 2453 | */ |
| 2454 | void (*mmap_free)(struct rdma_user_mmap_entry *entry); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2455 | void (*disassociate_ucontext)(struct ib_ucontext *ibcontext); |
Shamir Rabinovitch | ff23dfa | 2019-03-31 19:10:07 +0300 | [diff] [blame] | 2456 | int (*alloc_pd)(struct ib_pd *pd, struct ib_udata *udata); |
Leon Romanovsky | 91a7c58 | 2020-09-07 15:09:13 +0300 | [diff] [blame] | 2457 | int (*dealloc_pd)(struct ib_pd *pd, struct ib_udata *udata); |
Maor Gottlieb | fa5d010 | 2020-04-30 22:21:42 +0300 | [diff] [blame] | 2458 | int (*create_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr, |
| 2459 | struct ib_udata *udata); |
Jason Gunthorpe | 676a80a | 2020-10-03 20:20:11 -0300 | [diff] [blame] | 2460 | int (*create_user_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr, |
| 2461 | struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2462 | int (*modify_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); |
| 2463 | int (*query_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); |
Leon Romanovsky | 9a9ebf8 | 2020-09-07 15:09:14 +0300 | [diff] [blame] | 2464 | int (*destroy_ah)(struct ib_ah *ah, u32 flags); |
Leon Romanovsky | 68e326d | 2019-04-03 16:42:43 +0300 | [diff] [blame] | 2465 | int (*create_srq)(struct ib_srq *srq, |
| 2466 | struct ib_srq_init_attr *srq_init_attr, |
| 2467 | struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2468 | int (*modify_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr, |
| 2469 | enum ib_srq_attr_mask srq_attr_mask, |
| 2470 | struct ib_udata *udata); |
| 2471 | int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr); |
Leon Romanovsky | 119181d | 2020-09-07 15:09:16 +0300 | [diff] [blame] | 2472 | int (*destroy_srq)(struct ib_srq *srq, struct ib_udata *udata); |
Leon Romanovsky | 514aee6 | 2021-07-23 14:39:50 +0300 | [diff] [blame] | 2473 | int (*create_qp)(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr, |
| 2474 | struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2475 | int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, |
| 2476 | int qp_attr_mask, struct ib_udata *udata); |
| 2477 | int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr, |
| 2478 | int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 2479 | int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata); |
Leon Romanovsky | e39afe3 | 2019-05-28 14:37:29 +0300 | [diff] [blame] | 2480 | int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr, |
| 2481 | struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2482 | int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); |
Leon Romanovsky | 43d781b | 2020-09-07 15:09:18 +0300 | [diff] [blame] | 2483 | int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2484 | int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata); |
| 2485 | struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags); |
| 2486 | struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length, |
| 2487 | u64 virt_addr, int mr_access_flags, |
| 2488 | struct ib_udata *udata); |
Jianxin Xiong | 3bc489e | 2020-12-15 13:27:14 -0800 | [diff] [blame] | 2489 | struct ib_mr *(*reg_user_mr_dmabuf)(struct ib_pd *pd, u64 offset, |
| 2490 | u64 length, u64 virt_addr, int fd, |
| 2491 | int mr_access_flags, |
| 2492 | struct ib_udata *udata); |
Jason Gunthorpe | 6e0954b | 2020-11-30 09:58:37 +0200 | [diff] [blame] | 2493 | struct ib_mr *(*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start, |
| 2494 | u64 length, u64 virt_addr, |
| 2495 | int mr_access_flags, struct ib_pd *pd, |
| 2496 | struct ib_udata *udata); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 2497 | int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2498 | struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type, |
Gal Pressman | 42a3b15 | 2020-07-06 15:03:43 +0300 | [diff] [blame] | 2499 | u32 max_num_sg); |
Israel Rukshin | 26bc7ea | 2019-06-11 18:52:39 +0300 | [diff] [blame] | 2500 | struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd, |
| 2501 | u32 max_num_data_sg, |
| 2502 | u32 max_num_meta_sg); |
Moni Shoua | ad8a449 | 2018-12-11 13:37:52 +0200 | [diff] [blame] | 2503 | int (*advise_mr)(struct ib_pd *pd, |
| 2504 | enum ib_uverbs_advise_mr_advice advice, u32 flags, |
| 2505 | struct ib_sge *sg_list, u32 num_sge, |
| 2506 | struct uverbs_attr_bundle *attrs); |
Avihai Horon | 1477d44 | 2021-06-09 14:05:03 +0300 | [diff] [blame] | 2507 | |
| 2508 | /* |
| 2509 | * Kernel users should universally support relaxed ordering (RO), as |
| 2510 | * they are designed to read data only after observing the CQE and use |
| 2511 | * the DMA API correctly. |
| 2512 | * |
| 2513 | * Some drivers implicitly enable RO if platform supports it. |
| 2514 | */ |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2515 | int (*map_mr_sg)(struct ib_mr *mr, struct scatterlist *sg, int sg_nents, |
| 2516 | unsigned int *sg_offset); |
| 2517 | int (*check_mr_status)(struct ib_mr *mr, u32 check_mask, |
| 2518 | struct ib_mr_status *mr_status); |
Leon Romanovsky | d18bb3e | 2020-09-02 11:16:22 +0300 | [diff] [blame] | 2519 | int (*alloc_mw)(struct ib_mw *mw, struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2520 | int (*dealloc_mw)(struct ib_mw *mw); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2521 | int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid); |
| 2522 | int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid); |
Leon Romanovsky | 28ad5f65 | 2020-06-30 13:18:54 +0300 | [diff] [blame] | 2523 | int (*alloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata); |
Leon Romanovsky | d0c45c8 | 2020-09-07 15:09:19 +0300 | [diff] [blame] | 2524 | int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2525 | struct ib_flow *(*create_flow)(struct ib_qp *qp, |
| 2526 | struct ib_flow_attr *flow_attr, |
Leon Romanovsky | d667374 | 2020-07-30 11:12:35 +0300 | [diff] [blame] | 2527 | struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2528 | int (*destroy_flow)(struct ib_flow *flow_id); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2529 | int (*destroy_flow_action)(struct ib_flow_action *action); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2530 | int (*set_vf_link_state)(struct ib_device *device, int vf, u32 port, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2531 | int state); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2532 | int (*get_vf_config)(struct ib_device *device, int vf, u32 port, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2533 | struct ifla_vf_info *ivf); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2534 | int (*get_vf_stats)(struct ib_device *device, int vf, u32 port, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2535 | struct ifla_vf_stats *stats); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2536 | int (*get_vf_guid)(struct ib_device *device, int vf, u32 port, |
Danit Goldberg | bfcb3c5d | 2019-11-06 15:08:32 +0200 | [diff] [blame] | 2537 | struct ifla_vf_guid *node_guid, |
| 2538 | struct ifla_vf_guid *port_guid); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2539 | int (*set_vf_guid)(struct ib_device *device, int vf, u32 port, u64 guid, |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2540 | int type); |
| 2541 | struct ib_wq *(*create_wq)(struct ib_pd *pd, |
| 2542 | struct ib_wq_init_attr *init_attr, |
| 2543 | struct ib_udata *udata); |
Leon Romanovsky | add5353 | 2020-09-07 15:09:20 +0300 | [diff] [blame] | 2544 | int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2545 | int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr, |
| 2546 | u32 wq_attr_mask, struct ib_udata *udata); |
Leon Romanovsky | c0a6b5e | 2020-09-02 11:16:23 +0300 | [diff] [blame] | 2547 | int (*create_rwq_ind_table)(struct ib_rwq_ind_table *ib_rwq_ind_table, |
| 2548 | struct ib_rwq_ind_table_init_attr *init_attr, |
| 2549 | struct ib_udata *udata); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2550 | int (*destroy_rwq_ind_table)(struct ib_rwq_ind_table *wq_ind_table); |
| 2551 | struct ib_dm *(*alloc_dm)(struct ib_device *device, |
| 2552 | struct ib_ucontext *context, |
| 2553 | struct ib_dm_alloc_attr *attr, |
| 2554 | struct uverbs_attr_bundle *attrs); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 2555 | int (*dealloc_dm)(struct ib_dm *dm, struct uverbs_attr_bundle *attrs); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2556 | struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm, |
| 2557 | struct ib_dm_mr_attr *attr, |
| 2558 | struct uverbs_attr_bundle *attrs); |
Leon Romanovsky | 3b023e1 | 2020-06-30 13:18:52 +0300 | [diff] [blame] | 2559 | int (*create_counters)(struct ib_counters *counters, |
| 2560 | struct uverbs_attr_bundle *attrs); |
Leon Romanovsky | 71ff3f6 | 2020-09-07 15:09:21 +0300 | [diff] [blame] | 2561 | int (*destroy_counters)(struct ib_counters *counters); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2562 | int (*read_counters)(struct ib_counters *counters, |
| 2563 | struct ib_counters_read_attr *counters_read_attr, |
| 2564 | struct uverbs_attr_bundle *attrs); |
Max Gurtovoy | 2cdfcdd | 2019-06-11 18:52:40 +0300 | [diff] [blame] | 2565 | int (*map_mr_sg_pi)(struct ib_mr *mr, struct scatterlist *data_sg, |
| 2566 | int data_sg_nents, unsigned int *data_sg_offset, |
| 2567 | struct scatterlist *meta_sg, int meta_sg_nents, |
| 2568 | unsigned int *meta_sg_offset); |
| 2569 | |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2570 | /** |
Jason Gunthorpe | 4b5f4d3 | 2021-06-11 19:00:20 +0300 | [diff] [blame] | 2571 | * alloc_hw_[device,port]_stats - Allocate a struct rdma_hw_stats and |
| 2572 | * fill in the driver initialized data. The struct is kfree()'ed by |
| 2573 | * the sysfs core when the device is removed. A lifespan of -1 in the |
| 2574 | * return struct tells the core to set a default lifespan. |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2575 | */ |
Jason Gunthorpe | 4b5f4d3 | 2021-06-11 19:00:20 +0300 | [diff] [blame] | 2576 | struct rdma_hw_stats *(*alloc_hw_device_stats)(struct ib_device *device); |
| 2577 | struct rdma_hw_stats *(*alloc_hw_port_stats)(struct ib_device *device, |
| 2578 | u32 port_num); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2579 | /** |
| 2580 | * get_hw_stats - Fill in the counter value(s) in the stats struct. |
| 2581 | * @index - The index in the value array we wish to have updated, or |
| 2582 | * num_counters if we want all stats updated |
| 2583 | * Return codes - |
| 2584 | * < 0 - Error, no counters updated |
| 2585 | * index - Updated the single counter pointed to by index |
| 2586 | * num_counters - Updated all counters (will reset the timestamp |
| 2587 | * and prevent further calls for lifespan milliseconds) |
| 2588 | * Drivers are allowed to update all counters in leiu of just the |
| 2589 | * one given in index at their option |
| 2590 | */ |
| 2591 | int (*get_hw_stats)(struct ib_device *device, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2592 | struct rdma_hw_stats *stats, u32 port, int index); |
Jason Gunthorpe | d7407d1 | 2021-06-11 19:00:32 +0300 | [diff] [blame] | 2593 | |
Leon Romanovsky | 02da375 | 2019-01-30 12:49:02 +0200 | [diff] [blame] | 2594 | /** |
Aharon Landau | 5e2ddd1 | 2021-10-08 15:24:32 +0300 | [diff] [blame] | 2595 | * modify_hw_stat - Modify the counter configuration |
| 2596 | * @enable: true/false when enable/disable a counter |
| 2597 | * Return codes - 0 on success or error code otherwise. |
| 2598 | */ |
| 2599 | int (*modify_hw_stat)(struct ib_device *device, u32 port, |
| 2600 | unsigned int counter_index, bool enable); |
| 2601 | /** |
Leon Romanovsky | 02da375 | 2019-01-30 12:49:02 +0200 | [diff] [blame] | 2602 | * Allows rdma drivers to add their own restrack attributes. |
| 2603 | */ |
Maor Gottlieb | f443452 | 2020-06-23 14:30:36 +0300 | [diff] [blame] | 2604 | int (*fill_res_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr); |
Maor Gottlieb | 65959522 | 2020-06-23 14:30:40 +0300 | [diff] [blame] | 2605 | int (*fill_res_mr_entry_raw)(struct sk_buff *msg, struct ib_mr *ibmr); |
Maor Gottlieb | 9e2a187 | 2020-06-23 14:30:37 +0300 | [diff] [blame] | 2606 | int (*fill_res_cq_entry)(struct sk_buff *msg, struct ib_cq *ibcq); |
Maor Gottlieb | 65959522 | 2020-06-23 14:30:40 +0300 | [diff] [blame] | 2607 | int (*fill_res_cq_entry_raw)(struct sk_buff *msg, struct ib_cq *ibcq); |
Maor Gottlieb | 5cc3411 | 2020-06-23 14:30:38 +0300 | [diff] [blame] | 2608 | int (*fill_res_qp_entry)(struct sk_buff *msg, struct ib_qp *ibqp); |
Maor Gottlieb | 65959522 | 2020-06-23 14:30:40 +0300 | [diff] [blame] | 2609 | int (*fill_res_qp_entry_raw)(struct sk_buff *msg, struct ib_qp *ibqp); |
Maor Gottlieb | 211cd94 | 2020-06-23 14:30:39 +0300 | [diff] [blame] | 2610 | int (*fill_res_cm_id_entry)(struct sk_buff *msg, struct rdma_cm_id *id); |
Leon Romanovsky | 21a428a | 2019-02-03 14:55:51 +0200 | [diff] [blame] | 2611 | |
Jason Gunthorpe | d089989 | 2019-02-12 21:12:53 -0700 | [diff] [blame] | 2612 | /* Device lifecycle callbacks */ |
| 2613 | /* |
Jason Gunthorpe | ca22354 | 2019-02-12 21:12:56 -0700 | [diff] [blame] | 2614 | * Called after the device becomes registered, before clients are |
| 2615 | * attached |
| 2616 | */ |
| 2617 | int (*enable_driver)(struct ib_device *dev); |
| 2618 | /* |
Jason Gunthorpe | d089989 | 2019-02-12 21:12:53 -0700 | [diff] [blame] | 2619 | * This is called as part of ib_dealloc_device(). |
| 2620 | */ |
| 2621 | void (*dealloc_driver)(struct ib_device *dev); |
| 2622 | |
Kamal Heib | dd05cb8 | 2019-04-29 14:59:06 +0300 | [diff] [blame] | 2623 | /* iWarp CM callbacks */ |
| 2624 | void (*iw_add_ref)(struct ib_qp *qp); |
| 2625 | void (*iw_rem_ref)(struct ib_qp *qp); |
| 2626 | struct ib_qp *(*iw_get_qp)(struct ib_device *device, int qpn); |
| 2627 | int (*iw_connect)(struct iw_cm_id *cm_id, |
| 2628 | struct iw_cm_conn_param *conn_param); |
| 2629 | int (*iw_accept)(struct iw_cm_id *cm_id, |
| 2630 | struct iw_cm_conn_param *conn_param); |
| 2631 | int (*iw_reject)(struct iw_cm_id *cm_id, const void *pdata, |
| 2632 | u8 pdata_len); |
| 2633 | int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog); |
| 2634 | int (*iw_destroy_listen)(struct iw_cm_id *cm_id); |
Mark Zhang | 99fa331 | 2019-07-02 13:02:35 +0300 | [diff] [blame] | 2635 | /** |
| 2636 | * counter_bind_qp - Bind a QP to a counter. |
| 2637 | * @counter - The counter to be bound. If counter->id is zero then |
| 2638 | * the driver needs to allocate a new counter and set counter->id |
| 2639 | */ |
| 2640 | int (*counter_bind_qp)(struct rdma_counter *counter, struct ib_qp *qp); |
| 2641 | /** |
| 2642 | * counter_unbind_qp - Unbind the qp from the dynamically-allocated |
| 2643 | * counter and bind it onto the default one |
| 2644 | */ |
| 2645 | int (*counter_unbind_qp)(struct ib_qp *qp); |
| 2646 | /** |
| 2647 | * counter_dealloc -De-allocate the hw counter |
| 2648 | */ |
| 2649 | int (*counter_dealloc)(struct rdma_counter *counter); |
Mark Zhang | c4ffee7 | 2019-07-02 13:02:40 +0300 | [diff] [blame] | 2650 | /** |
| 2651 | * counter_alloc_stats - Allocate a struct rdma_hw_stats and fill in |
| 2652 | * the driver initialized data. |
| 2653 | */ |
| 2654 | struct rdma_hw_stats *(*counter_alloc_stats)( |
| 2655 | struct rdma_counter *counter); |
| 2656 | /** |
| 2657 | * counter_update_stats - Query the stats value of this counter |
| 2658 | */ |
| 2659 | int (*counter_update_stats)(struct rdma_counter *counter); |
Kamal Heib | dd05cb8 | 2019-04-29 14:59:06 +0300 | [diff] [blame] | 2660 | |
Erez Alfasi | 4061ff7 | 2019-10-16 09:23:08 +0300 | [diff] [blame] | 2661 | /** |
| 2662 | * Allows rdma drivers to add their own restrack attributes |
| 2663 | * dumped via 'rdma stat' iproute2 command. |
| 2664 | */ |
Maor Gottlieb | f443452 | 2020-06-23 14:30:36 +0300 | [diff] [blame] | 2665 | int (*fill_stat_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr); |
Erez Alfasi | 4061ff7 | 2019-10-16 09:23:08 +0300 | [diff] [blame] | 2666 | |
Yishai Hadas | 1c8fb1ea | 2020-06-30 12:39:12 +0300 | [diff] [blame] | 2667 | /* query driver for its ucontext properties */ |
| 2668 | int (*query_ucontext)(struct ib_ucontext *context, |
| 2669 | struct uverbs_attr_bundle *attrs); |
| 2670 | |
Leon Romanovsky | 514aee6 | 2021-07-23 14:39:50 +0300 | [diff] [blame] | 2671 | /* |
| 2672 | * Provide NUMA node. This API exists for rdmavt/hfi1 only. |
| 2673 | * Everyone else relies on Linux memory management model. |
| 2674 | */ |
| 2675 | int (*get_numa_node)(struct ib_device *dev); |
| 2676 | |
Leon Romanovsky | d345691 | 2019-04-03 16:42:42 +0300 | [diff] [blame] | 2677 | DECLARE_RDMA_OBJ_SIZE(ib_ah); |
Leon Romanovsky | 3b023e1 | 2020-06-30 13:18:52 +0300 | [diff] [blame] | 2678 | DECLARE_RDMA_OBJ_SIZE(ib_counters); |
Leon Romanovsky | e39afe3 | 2019-05-28 14:37:29 +0300 | [diff] [blame] | 2679 | DECLARE_RDMA_OBJ_SIZE(ib_cq); |
Leon Romanovsky | d18bb3e | 2020-09-02 11:16:22 +0300 | [diff] [blame] | 2680 | DECLARE_RDMA_OBJ_SIZE(ib_mw); |
Leon Romanovsky | 21a428a | 2019-02-03 14:55:51 +0200 | [diff] [blame] | 2681 | DECLARE_RDMA_OBJ_SIZE(ib_pd); |
Leon Romanovsky | 514aee6 | 2021-07-23 14:39:50 +0300 | [diff] [blame] | 2682 | DECLARE_RDMA_OBJ_SIZE(ib_qp); |
Leon Romanovsky | c0a6b5e | 2020-09-02 11:16:23 +0300 | [diff] [blame] | 2683 | DECLARE_RDMA_OBJ_SIZE(ib_rwq_ind_table); |
Leon Romanovsky | 68e326d | 2019-04-03 16:42:43 +0300 | [diff] [blame] | 2684 | DECLARE_RDMA_OBJ_SIZE(ib_srq); |
Leon Romanovsky | a2a074e | 2019-02-12 20:39:16 +0200 | [diff] [blame] | 2685 | DECLARE_RDMA_OBJ_SIZE(ib_ucontext); |
Leon Romanovsky | 28ad5f65 | 2020-06-30 13:18:54 +0300 | [diff] [blame] | 2686 | DECLARE_RDMA_OBJ_SIZE(ib_xrcd); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2687 | }; |
| 2688 | |
Parav Pandit | cebe556 | 2019-02-26 13:56:11 +0200 | [diff] [blame] | 2689 | struct ib_core_device { |
| 2690 | /* device must be the first element in structure until, |
| 2691 | * union of ib_core_device and device exists in ib_device. |
| 2692 | */ |
| 2693 | struct device dev; |
Parav Pandit | 4e0f7b9 | 2019-02-26 13:56:13 +0200 | [diff] [blame] | 2694 | possible_net_t rdma_net; |
Parav Pandit | cebe556 | 2019-02-26 13:56:11 +0200 | [diff] [blame] | 2695 | struct kobject *ports_kobj; |
| 2696 | struct list_head port_list; |
| 2697 | struct ib_device *owner; /* reach back to owner ib_device */ |
| 2698 | }; |
Leon Romanovsky | 41eda65 | 2019-02-18 22:25:47 +0200 | [diff] [blame] | 2699 | |
Parav Pandit | cebe556 | 2019-02-26 13:56:11 +0200 | [diff] [blame] | 2700 | struct rdma_restrack_root; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2701 | struct ib_device { |
Bart Van Assche | 0957c29 | 2017-03-07 22:56:53 +0000 | [diff] [blame] | 2702 | /* Do not access @dma_device directly from ULP nor from HW drivers. */ |
| 2703 | struct device *dma_device; |
Kamal Heib | 3023a1e | 2018-12-10 21:09:48 +0200 | [diff] [blame] | 2704 | struct ib_device_ops ops; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2705 | char name[IB_DEVICE_NAME_MAX]; |
Jason Gunthorpe | 324e227 | 2019-02-12 21:12:51 -0700 | [diff] [blame] | 2706 | struct rcu_head rcu_head; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2707 | |
| 2708 | struct list_head event_handler_list; |
Parav Pandit | 6b57cea | 2019-12-12 13:30:22 +0200 | [diff] [blame] | 2709 | /* Protects event_handler_list */ |
| 2710 | struct rw_semaphore event_handler_rwsem; |
| 2711 | |
| 2712 | /* Protects QP's event_handler calls and open_qp list */ |
Parav Pandit | 40adf68 | 2019-12-12 13:30:24 +0200 | [diff] [blame] | 2713 | spinlock_t qp_open_list_lock; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2714 | |
Jason Gunthorpe | 921eab1 | 2019-02-06 22:41:54 -0700 | [diff] [blame] | 2715 | struct rw_semaphore client_data_rwsem; |
Jason Gunthorpe | 0df91bb | 2019-02-06 22:41:53 -0700 | [diff] [blame] | 2716 | struct xarray client_data; |
Jason Gunthorpe | d089989 | 2019-02-12 21:12:53 -0700 | [diff] [blame] | 2717 | struct mutex unregistration_lock; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2718 | |
Parav Pandit | 17e1064 | 2019-12-12 13:30:23 +0200 | [diff] [blame] | 2719 | /* Synchronize GID, Pkey cache entries, subnet prefix, LMC */ |
| 2720 | rwlock_t cache_lock; |
Ira Weiny | 7738613 | 2015-05-13 20:02:58 -0400 | [diff] [blame] | 2721 | /** |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 2722 | * port_data is indexed by port number |
Ira Weiny | 7738613 | 2015-05-13 20:02:58 -0400 | [diff] [blame] | 2723 | */ |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 2724 | struct ib_port_data *port_data; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2725 | |
Michael S. Tsirkin | f4fd0b2 | 2007-05-03 13:48:47 +0300 | [diff] [blame] | 2726 | int num_comp_vectors; |
| 2727 | |
Parav Pandit | cebe556 | 2019-02-26 13:56:11 +0200 | [diff] [blame] | 2728 | union { |
| 2729 | struct device dev; |
| 2730 | struct ib_core_device coredev; |
| 2731 | }; |
| 2732 | |
Jason Gunthorpe | b7066b3 | 2021-06-11 19:00:26 +0300 | [diff] [blame] | 2733 | /* First group is for device attributes, |
| 2734 | * Second group is for driver provided attributes (optional). |
| 2735 | * Third group is for the hw_stats |
| 2736 | * It is a NULL terminated array. |
Parav Pandit | d4122f5 | 2018-10-11 22:31:53 +0300 | [diff] [blame] | 2737 | */ |
Jason Gunthorpe | b7066b3 | 2021-06-11 19:00:26 +0300 | [diff] [blame] | 2738 | const struct attribute_group *groups[4]; |
Parav Pandit | adee9f3 | 2018-09-05 09:47:58 +0300 | [diff] [blame] | 2739 | |
Alexander Chiang | 17a55f7 | 2010-02-02 19:09:16 +0000 | [diff] [blame] | 2740 | u64 uverbs_cmd_mask; |
Roland Dreier | 274c089 | 2005-09-29 14:17:48 -0700 | [diff] [blame] | 2741 | |
Yuval Shaia | bd99fde | 2016-08-25 10:57:07 -0700 | [diff] [blame] | 2742 | char node_desc[IB_DEVICE_NODE_DESC_MAX]; |
Sean Hefty | cf311cd | 2006-01-10 07:39:34 -0800 | [diff] [blame] | 2743 | __be64 node_guid; |
Steve Wise | 96f15c0 | 2008-07-14 23:48:53 -0700 | [diff] [blame] | 2744 | u32 local_dma_lkey; |
Hal Rosenstock | 4139032 | 2015-06-29 09:57:00 -0400 | [diff] [blame] | 2745 | u16 is_switch:1; |
Gal Pressman | 6780c4f | 2019-01-22 10:08:22 +0200 | [diff] [blame] | 2746 | /* Indicates kernel verbs support, should not be used in drivers */ |
| 2747 | u16 kverbs_provider:1; |
Yamin Friedman | da66297 | 2019-07-08 13:59:03 +0300 | [diff] [blame] | 2748 | /* CQ adaptive moderation (RDMA DIM) */ |
| 2749 | u16 use_cq_dim:1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2750 | u8 node_type; |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2751 | u32 phys_port_cnt; |
Ira Weiny | 3e153a9 | 2015-12-18 10:59:44 +0200 | [diff] [blame] | 2752 | struct ib_device_attr attrs; |
Jason Gunthorpe | 467f432 | 2021-06-11 19:00:22 +0300 | [diff] [blame] | 2753 | struct hw_stats_device_data *hw_stats_data; |
Ira Weiny | 7738613 | 2015-05-13 20:02:58 -0400 | [diff] [blame] | 2754 | |
Parav Pandit | 43579b5 | 2017-01-10 00:02:14 +0000 | [diff] [blame] | 2755 | #ifdef CONFIG_CGROUP_RDMA |
| 2756 | struct rdmacg_device cg_device; |
| 2757 | #endif |
| 2758 | |
Leon Romanovsky | ecc82c5 | 2017-06-18 14:39:59 +0300 | [diff] [blame] | 2759 | u32 index; |
Yamin Friedman | c7ff819 | 2020-05-27 11:34:53 +0300 | [diff] [blame] | 2760 | |
| 2761 | spinlock_t cq_pools_lock; |
| 2762 | struct list_head cq_pools[IB_POLL_LAST_POOL_TYPE + 1]; |
| 2763 | |
Leon Romanovsky | 41eda65 | 2019-02-18 22:25:47 +0200 | [diff] [blame] | 2764 | struct rdma_restrack_root *res; |
Leon Romanovsky | ecc82c5 | 2017-06-18 14:39:59 +0300 | [diff] [blame] | 2765 | |
Jason Gunthorpe | 0cbf432 | 2018-11-12 22:59:50 +0200 | [diff] [blame] | 2766 | const struct uapi_definition *driver_def; |
Jason Gunthorpe | d79af72 | 2019-01-10 14:02:24 -0700 | [diff] [blame] | 2767 | |
Parav Pandit | 01b6711 | 2018-11-16 03:50:57 +0200 | [diff] [blame] | 2768 | /* |
Jason Gunthorpe | d79af72 | 2019-01-10 14:02:24 -0700 | [diff] [blame] | 2769 | * Positive refcount indicates that the device is currently |
| 2770 | * registered and cannot be unregistered. |
Parav Pandit | 01b6711 | 2018-11-16 03:50:57 +0200 | [diff] [blame] | 2771 | */ |
| 2772 | refcount_t refcount; |
| 2773 | struct completion unreg_completion; |
Jason Gunthorpe | d089989 | 2019-02-12 21:12:53 -0700 | [diff] [blame] | 2774 | struct work_struct unregistration_work; |
Steve Wise | 3856ec4 | 2019-02-15 11:03:53 -0800 | [diff] [blame] | 2775 | |
| 2776 | const struct rdma_link_ops *link_ops; |
Parav Pandit | 4e0f7b9 | 2019-02-26 13:56:13 +0200 | [diff] [blame] | 2777 | |
| 2778 | /* Protects compat_devs xarray modifications */ |
| 2779 | struct mutex compat_devs_mutex; |
| 2780 | /* Maintains compat devices for each net namespace */ |
| 2781 | struct xarray compat_devs; |
Kamal Heib | dd05cb8 | 2019-04-29 14:59:06 +0300 | [diff] [blame] | 2782 | |
| 2783 | /* Used by iWarp CM */ |
| 2784 | char iw_ifname[IFNAMSIZ]; |
| 2785 | u32 iw_driver_flags; |
Maor Gottlieb | bd3920e | 2020-04-30 22:21:43 +0300 | [diff] [blame] | 2786 | u32 lag_flags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2787 | }; |
| 2788 | |
Leon Romanovsky | 514aee6 | 2021-07-23 14:39:50 +0300 | [diff] [blame] | 2789 | static inline void *rdma_zalloc_obj(struct ib_device *dev, size_t size, |
| 2790 | gfp_t gfp, bool is_numa_aware) |
| 2791 | { |
| 2792 | if (is_numa_aware && dev->ops.get_numa_node) |
| 2793 | return kzalloc_node(size, gfp, dev->ops.get_numa_node(dev)); |
| 2794 | |
| 2795 | return kzalloc(size, gfp); |
| 2796 | } |
| 2797 | |
Jason Gunthorpe | 0e2d00e | 2019-06-13 21:38:18 -0300 | [diff] [blame] | 2798 | struct ib_client_nl_info; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2799 | struct ib_client { |
Jason Gunthorpe | e59178d | 2019-02-06 22:41:52 -0700 | [diff] [blame] | 2800 | const char *name; |
Jason Gunthorpe | 11a0ae4 | 2020-04-21 20:24:40 +0300 | [diff] [blame] | 2801 | int (*add)(struct ib_device *ibdev); |
Haggai Eran | 7c1eb45 | 2015-07-30 17:50:14 +0300 | [diff] [blame] | 2802 | void (*remove)(struct ib_device *, void *client_data); |
Leon Romanovsky | dc1435c | 2019-05-17 15:43:10 +0300 | [diff] [blame] | 2803 | void (*rename)(struct ib_device *dev, void *client_data); |
Jason Gunthorpe | 0e2d00e | 2019-06-13 21:38:18 -0300 | [diff] [blame] | 2804 | int (*get_nl_info)(struct ib_device *ibdev, void *client_data, |
| 2805 | struct ib_client_nl_info *res); |
| 2806 | int (*get_global_nl_info)(struct ib_client_nl_info *res); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2807 | |
Yotam Kenneth | 9268f72 | 2015-07-30 17:50:15 +0300 | [diff] [blame] | 2808 | /* Returns the net_dev belonging to this ib_client and matching the |
| 2809 | * given parameters. |
| 2810 | * @dev: An RDMA device that the net_dev use for communication. |
| 2811 | * @port: A physical port number on the RDMA device. |
| 2812 | * @pkey: P_Key that the net_dev uses if applicable. |
| 2813 | * @gid: A GID that the net_dev uses to communicate. |
| 2814 | * @addr: An IP address the net_dev is configured with. |
| 2815 | * @client_data: The device's client data set by ib_set_client_data(). |
| 2816 | * |
| 2817 | * An ib_client that implements a net_dev on top of RDMA devices |
| 2818 | * (such as IP over IB) should implement this callback, allowing the |
| 2819 | * rdma_cm module to find the right net_dev for a given request. |
| 2820 | * |
| 2821 | * The caller is responsible for calling dev_put on the returned |
| 2822 | * netdev. */ |
| 2823 | struct net_device *(*get_net_dev_by_params)( |
| 2824 | struct ib_device *dev, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 2825 | u32 port, |
Yotam Kenneth | 9268f72 | 2015-07-30 17:50:15 +0300 | [diff] [blame] | 2826 | u16 pkey, |
| 2827 | const union ib_gid *gid, |
| 2828 | const struct sockaddr *addr, |
| 2829 | void *client_data); |
Jason Gunthorpe | 621e55f | 2019-07-31 11:18:40 +0300 | [diff] [blame] | 2830 | |
| 2831 | refcount_t uses; |
| 2832 | struct completion uses_zero; |
Jason Gunthorpe | e59178d | 2019-02-06 22:41:52 -0700 | [diff] [blame] | 2833 | u32 client_id; |
Gal Pressman | 6780c4f | 2019-01-22 10:08:22 +0200 | [diff] [blame] | 2834 | |
| 2835 | /* kverbs are not required by the client */ |
| 2836 | u8 no_kverbs_req:1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2837 | }; |
| 2838 | |
Shiraz Saleem | a808273 | 2019-05-06 08:53:33 -0500 | [diff] [blame] | 2839 | /* |
| 2840 | * IB block DMA iterator |
| 2841 | * |
| 2842 | * Iterates the DMA-mapped SGL in contiguous memory blocks aligned |
| 2843 | * to a HW supported page size. |
| 2844 | */ |
| 2845 | struct ib_block_iter { |
| 2846 | /* internal states */ |
| 2847 | struct scatterlist *__sg; /* sg holding the current aligned block */ |
| 2848 | dma_addr_t __dma_addr; /* unaligned DMA address of this block */ |
| 2849 | unsigned int __sg_nents; /* number of SG entries */ |
| 2850 | unsigned int __sg_advance; /* number of bytes to advance in sg in next step */ |
| 2851 | unsigned int __pg_bit; /* alignment of current block */ |
| 2852 | }; |
| 2853 | |
Leon Romanovsky | 459cc69 | 2019-01-30 12:49:11 +0200 | [diff] [blame] | 2854 | struct ib_device *_ib_alloc_device(size_t size); |
| 2855 | #define ib_alloc_device(drv_struct, member) \ |
| 2856 | container_of(_ib_alloc_device(sizeof(struct drv_struct) + \ |
| 2857 | BUILD_BUG_ON_ZERO(offsetof( \ |
| 2858 | struct drv_struct, member))), \ |
| 2859 | struct drv_struct, member) |
| 2860 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2861 | void ib_dealloc_device(struct ib_device *device); |
| 2862 | |
Leon Romanovsky | 9abb0d1 | 2017-06-27 16:49:53 +0300 | [diff] [blame] | 2863 | void ib_get_device_fw_str(struct ib_device *device, char *str); |
Ira Weiny | 5fa76c20 | 2016-06-15 02:21:56 -0400 | [diff] [blame] | 2864 | |
Jason Gunthorpe | e0477b3 | 2020-10-08 11:27:52 +0300 | [diff] [blame] | 2865 | int ib_register_device(struct ib_device *device, const char *name, |
| 2866 | struct device *dma_device); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2867 | void ib_unregister_device(struct ib_device *device); |
Jason Gunthorpe | d089989 | 2019-02-12 21:12:53 -0700 | [diff] [blame] | 2868 | void ib_unregister_driver(enum rdma_driver_id driver_id); |
| 2869 | void ib_unregister_device_and_put(struct ib_device *device); |
| 2870 | void ib_unregister_device_queued(struct ib_device *ib_dev); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2871 | |
| 2872 | int ib_register_client (struct ib_client *client); |
| 2873 | void ib_unregister_client(struct ib_client *client); |
| 2874 | |
Shiraz Saleem | a808273 | 2019-05-06 08:53:33 -0500 | [diff] [blame] | 2875 | void __rdma_block_iter_start(struct ib_block_iter *biter, |
| 2876 | struct scatterlist *sglist, |
| 2877 | unsigned int nents, |
| 2878 | unsigned long pgsz); |
| 2879 | bool __rdma_block_iter_next(struct ib_block_iter *biter); |
| 2880 | |
| 2881 | /** |
| 2882 | * rdma_block_iter_dma_address - get the aligned dma address of the current |
| 2883 | * block held by the block iterator. |
| 2884 | * @biter: block iterator holding the memory block |
| 2885 | */ |
| 2886 | static inline dma_addr_t |
| 2887 | rdma_block_iter_dma_address(struct ib_block_iter *biter) |
| 2888 | { |
| 2889 | return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1); |
| 2890 | } |
| 2891 | |
| 2892 | /** |
| 2893 | * rdma_for_each_block - iterate over contiguous memory blocks of the sg list |
| 2894 | * @sglist: sglist to iterate over |
| 2895 | * @biter: block iterator holding the memory block |
| 2896 | * @nents: maximum number of sg entries to iterate over |
| 2897 | * @pgsz: best HW supported page size to use |
| 2898 | * |
| 2899 | * Callers may use rdma_block_iter_dma_address() to get each |
| 2900 | * blocks aligned DMA address. |
| 2901 | */ |
| 2902 | #define rdma_for_each_block(sglist, biter, nents, pgsz) \ |
| 2903 | for (__rdma_block_iter_start(biter, sglist, nents, \ |
| 2904 | pgsz); \ |
| 2905 | __rdma_block_iter_next(biter);) |
| 2906 | |
Jason Gunthorpe | 0df91bb | 2019-02-06 22:41:53 -0700 | [diff] [blame] | 2907 | /** |
| 2908 | * ib_get_client_data - Get IB client context |
| 2909 | * @device:Device to get context for |
| 2910 | * @client:Client to get context for |
| 2911 | * |
| 2912 | * ib_get_client_data() returns the client context data set with |
| 2913 | * ib_set_client_data(). This can only be called while the client is |
| 2914 | * registered to the device, once the ib_client remove() callback returns this |
| 2915 | * cannot be called. |
| 2916 | */ |
| 2917 | static inline void *ib_get_client_data(struct ib_device *device, |
| 2918 | struct ib_client *client) |
| 2919 | { |
| 2920 | return xa_load(&device->client_data, client->client_id); |
| 2921 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2922 | void ib_set_client_data(struct ib_device *device, struct ib_client *client, |
| 2923 | void *data); |
Kamal Heib | 521ed0d | 2018-12-10 21:09:30 +0200 | [diff] [blame] | 2924 | void ib_set_device_ops(struct ib_device *device, |
| 2925 | const struct ib_device_ops *ops); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2926 | |
Jason Gunthorpe | 5f9794d | 2018-09-16 20:43:08 +0300 | [diff] [blame] | 2927 | int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma, |
Michal Kalderon | c043ff2 | 2019-10-30 11:44:12 +0200 | [diff] [blame] | 2928 | unsigned long pfn, unsigned long size, pgprot_t prot, |
| 2929 | struct rdma_user_mmap_entry *entry); |
Michal Kalderon | 3411f9f | 2019-10-30 11:44:11 +0200 | [diff] [blame] | 2930 | int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext, |
| 2931 | struct rdma_user_mmap_entry *entry, |
| 2932 | size_t length); |
Yishai Hadas | 7a763d1 | 2019-12-12 12:02:36 +0200 | [diff] [blame] | 2933 | int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext, |
| 2934 | struct rdma_user_mmap_entry *entry, |
| 2935 | size_t length, u32 min_pgoff, |
| 2936 | u32 max_pgoff); |
| 2937 | |
Chengchang Tang | 6d202d9 | 2021-10-28 18:56:40 +0800 | [diff] [blame] | 2938 | static inline int |
| 2939 | rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext, |
| 2940 | struct rdma_user_mmap_entry *entry, |
| 2941 | size_t length, u32 pgoff) |
| 2942 | { |
| 2943 | return rdma_user_mmap_entry_insert_range(ucontext, entry, length, pgoff, |
| 2944 | pgoff); |
| 2945 | } |
| 2946 | |
Michal Kalderon | 3411f9f | 2019-10-30 11:44:11 +0200 | [diff] [blame] | 2947 | struct rdma_user_mmap_entry * |
| 2948 | rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext, |
| 2949 | unsigned long pgoff); |
| 2950 | struct rdma_user_mmap_entry * |
| 2951 | rdma_user_mmap_entry_get(struct ib_ucontext *ucontext, |
| 2952 | struct vm_area_struct *vma); |
| 2953 | void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry); |
| 2954 | |
| 2955 | void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry); |
Jason Gunthorpe | 5f9794d | 2018-09-16 20:43:08 +0300 | [diff] [blame] | 2956 | |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 2957 | static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len) |
| 2958 | { |
| 2959 | return copy_from_user(dest, udata->inbuf, len) ? -EFAULT : 0; |
| 2960 | } |
| 2961 | |
| 2962 | static inline int ib_copy_to_udata(struct ib_udata *udata, void *src, size_t len) |
| 2963 | { |
Yann Droneaud | 43c61165 | 2015-02-05 22:10:18 +0100 | [diff] [blame] | 2964 | return copy_to_user(udata->outbuf, src, len) ? -EFAULT : 0; |
Roland Dreier | e2773c0 | 2005-07-07 17:57:10 -0700 | [diff] [blame] | 2965 | } |
| 2966 | |
Matan Barak | c66db31 | 2018-03-19 15:02:36 +0200 | [diff] [blame] | 2967 | static inline bool ib_is_buffer_cleared(const void __user *p, |
| 2968 | size_t len) |
Matan Barak | 301a721 | 2015-12-15 20:30:10 +0200 | [diff] [blame] | 2969 | { |
Markus Elfring | 92d27ae | 2016-08-22 18:23:24 +0200 | [diff] [blame] | 2970 | bool ret; |
Matan Barak | 301a721 | 2015-12-15 20:30:10 +0200 | [diff] [blame] | 2971 | u8 *buf; |
| 2972 | |
| 2973 | if (len > USHRT_MAX) |
| 2974 | return false; |
| 2975 | |
Markus Elfring | 92d27ae | 2016-08-22 18:23:24 +0200 | [diff] [blame] | 2976 | buf = memdup_user(p, len); |
| 2977 | if (IS_ERR(buf)) |
Matan Barak | 301a721 | 2015-12-15 20:30:10 +0200 | [diff] [blame] | 2978 | return false; |
| 2979 | |
Matan Barak | 301a721 | 2015-12-15 20:30:10 +0200 | [diff] [blame] | 2980 | ret = !memchr_inv(buf, 0, len); |
Matan Barak | 301a721 | 2015-12-15 20:30:10 +0200 | [diff] [blame] | 2981 | kfree(buf); |
| 2982 | return ret; |
| 2983 | } |
| 2984 | |
Matan Barak | c66db31 | 2018-03-19 15:02:36 +0200 | [diff] [blame] | 2985 | static inline bool ib_is_udata_cleared(struct ib_udata *udata, |
| 2986 | size_t offset, |
| 2987 | size_t len) |
| 2988 | { |
| 2989 | return ib_is_buffer_cleared(udata->inbuf + offset, len); |
| 2990 | } |
| 2991 | |
Roland Dreier | 8a51866 | 2006-02-13 12:48:12 -0800 | [diff] [blame] | 2992 | /** |
| 2993 | * ib_modify_qp_is_ok - Check that the supplied attribute mask |
| 2994 | * contains all required attributes and no attributes not allowed for |
| 2995 | * the given QP state transition. |
| 2996 | * @cur_state: Current QP state |
| 2997 | * @next_state: Next QP state |
| 2998 | * @type: QP type |
| 2999 | * @mask: Mask of supplied QP attributes |
| 3000 | * |
| 3001 | * This function is a helper function that a low-level driver's |
| 3002 | * modify_qp method can use to validate the consumer's input. It |
| 3003 | * checks that cur_state and next_state are valid QP states, that a |
| 3004 | * transition from cur_state to next_state is allowed by the IB spec, |
| 3005 | * and that the attribute mask supplied is allowed for the transition. |
| 3006 | */ |
Leon Romanovsky | 19b1f54 | 2018-03-11 13:51:35 +0200 | [diff] [blame] | 3007 | bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, |
Kamal Heib | d31131b | 2018-10-02 16:11:21 +0300 | [diff] [blame] | 3008 | enum ib_qp_type type, enum ib_qp_attr_mask mask); |
Roland Dreier | 8a51866 | 2006-02-13 12:48:12 -0800 | [diff] [blame] | 3009 | |
Leon Romanovsky | dcc9881 | 2017-08-17 15:50:36 +0300 | [diff] [blame] | 3010 | void ib_register_event_handler(struct ib_event_handler *event_handler); |
| 3011 | void ib_unregister_event_handler(struct ib_event_handler *event_handler); |
Parav Pandit | 6b57cea | 2019-12-12 13:30:22 +0200 | [diff] [blame] | 3012 | void ib_dispatch_event(const struct ib_event *event); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3013 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3014 | int ib_query_port(struct ib_device *device, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3015 | u32 port_num, struct ib_port_attr *port_attr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3016 | |
Eli Cohen | a3f5ada | 2010-09-27 17:51:10 -0700 | [diff] [blame] | 3017 | enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3018 | u32 port_num); |
Eli Cohen | a3f5ada | 2010-09-27 17:51:10 -0700 | [diff] [blame] | 3019 | |
Ira Weiny | 0cf18d7 | 2015-05-13 20:02:55 -0400 | [diff] [blame] | 3020 | /** |
Hal Rosenstock | 4139032 | 2015-06-29 09:57:00 -0400 | [diff] [blame] | 3021 | * rdma_cap_ib_switch - Check if the device is IB switch |
| 3022 | * @device: Device to check |
| 3023 | * |
| 3024 | * Device driver is responsible for setting is_switch bit on |
| 3025 | * in ib_device structure at init time. |
| 3026 | * |
| 3027 | * Return: true if the device is IB switch. |
| 3028 | */ |
| 3029 | static inline bool rdma_cap_ib_switch(const struct ib_device *device) |
| 3030 | { |
| 3031 | return device->is_switch; |
| 3032 | } |
| 3033 | |
| 3034 | /** |
Ira Weiny | 0cf18d7 | 2015-05-13 20:02:55 -0400 | [diff] [blame] | 3035 | * rdma_start_port - Return the first valid port number for the device |
| 3036 | * specified |
| 3037 | * |
| 3038 | * @device: Device to be checked |
| 3039 | * |
| 3040 | * Return start port number |
| 3041 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3042 | static inline u32 rdma_start_port(const struct ib_device *device) |
Ira Weiny | 0cf18d7 | 2015-05-13 20:02:55 -0400 | [diff] [blame] | 3043 | { |
Hal Rosenstock | 4139032 | 2015-06-29 09:57:00 -0400 | [diff] [blame] | 3044 | return rdma_cap_ib_switch(device) ? 0 : 1; |
Ira Weiny | 0cf18d7 | 2015-05-13 20:02:55 -0400 | [diff] [blame] | 3045 | } |
| 3046 | |
| 3047 | /** |
Jason Gunthorpe | ea1075e | 2019-02-12 21:12:47 -0700 | [diff] [blame] | 3048 | * rdma_for_each_port - Iterate over all valid port numbers of the IB device |
| 3049 | * @device - The struct ib_device * to iterate over |
| 3050 | * @iter - The unsigned int to store the port number |
| 3051 | */ |
| 3052 | #define rdma_for_each_port(device, iter) \ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3053 | for (iter = rdma_start_port(device + \ |
| 3054 | BUILD_BUG_ON_ZERO(!__same_type(u32, \ |
| 3055 | iter))); \ |
| 3056 | iter <= rdma_end_port(device); iter++) |
Jason Gunthorpe | ea1075e | 2019-02-12 21:12:47 -0700 | [diff] [blame] | 3057 | |
| 3058 | /** |
Ira Weiny | 0cf18d7 | 2015-05-13 20:02:55 -0400 | [diff] [blame] | 3059 | * rdma_end_port - Return the last valid port number for the device |
| 3060 | * specified |
| 3061 | * |
| 3062 | * @device: Device to be checked |
| 3063 | * |
| 3064 | * Return last port number |
| 3065 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3066 | static inline u32 rdma_end_port(const struct ib_device *device) |
Ira Weiny | 0cf18d7 | 2015-05-13 20:02:55 -0400 | [diff] [blame] | 3067 | { |
Hal Rosenstock | 4139032 | 2015-06-29 09:57:00 -0400 | [diff] [blame] | 3068 | return rdma_cap_ib_switch(device) ? 0 : device->phys_port_cnt; |
Ira Weiny | 0cf18d7 | 2015-05-13 20:02:55 -0400 | [diff] [blame] | 3069 | } |
| 3070 | |
Yuval Shaia | 24dc831 | 2017-01-25 18:41:37 +0200 | [diff] [blame] | 3071 | static inline int rdma_is_port_valid(const struct ib_device *device, |
| 3072 | unsigned int port) |
| 3073 | { |
| 3074 | return (port >= rdma_start_port(device) && |
| 3075 | port <= rdma_end_port(device)); |
| 3076 | } |
| 3077 | |
Artemy Kovalyov | b02289b | 2018-07-04 15:57:50 +0300 | [diff] [blame] | 3078 | static inline bool rdma_is_grh_required(const struct ib_device *device, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3079 | u32 port_num) |
Artemy Kovalyov | b02289b | 2018-07-04 15:57:50 +0300 | [diff] [blame] | 3080 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3081 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3082 | RDMA_CORE_PORT_IB_GRH_REQUIRED; |
Artemy Kovalyov | b02289b | 2018-07-04 15:57:50 +0300 | [diff] [blame] | 3083 | } |
| 3084 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3085 | static inline bool rdma_protocol_ib(const struct ib_device *device, |
| 3086 | u32 port_num) |
Michael Wang | de66be9 | 2015-05-05 14:50:19 +0200 | [diff] [blame] | 3087 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3088 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3089 | RDMA_CORE_CAP_PROT_IB; |
Michael Wang | de66be9 | 2015-05-05 14:50:19 +0200 | [diff] [blame] | 3090 | } |
| 3091 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3092 | static inline bool rdma_protocol_roce(const struct ib_device *device, |
| 3093 | u32 port_num) |
Michael Wang | de66be9 | 2015-05-05 14:50:19 +0200 | [diff] [blame] | 3094 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3095 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3096 | (RDMA_CORE_CAP_PROT_ROCE | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP); |
Matan Barak | 7766a99 | 2015-12-23 14:56:50 +0200 | [diff] [blame] | 3097 | } |
| 3098 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3099 | static inline bool rdma_protocol_roce_udp_encap(const struct ib_device *device, |
| 3100 | u32 port_num) |
Matan Barak | 7766a99 | 2015-12-23 14:56:50 +0200 | [diff] [blame] | 3101 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3102 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3103 | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP; |
Matan Barak | 7766a99 | 2015-12-23 14:56:50 +0200 | [diff] [blame] | 3104 | } |
| 3105 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3106 | static inline bool rdma_protocol_roce_eth_encap(const struct ib_device *device, |
| 3107 | u32 port_num) |
Matan Barak | 7766a99 | 2015-12-23 14:56:50 +0200 | [diff] [blame] | 3108 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3109 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3110 | RDMA_CORE_CAP_PROT_ROCE; |
Michael Wang | de66be9 | 2015-05-05 14:50:19 +0200 | [diff] [blame] | 3111 | } |
| 3112 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3113 | static inline bool rdma_protocol_iwarp(const struct ib_device *device, |
| 3114 | u32 port_num) |
Michael Wang | de66be9 | 2015-05-05 14:50:19 +0200 | [diff] [blame] | 3115 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3116 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3117 | RDMA_CORE_CAP_PROT_IWARP; |
Michael Wang | de66be9 | 2015-05-05 14:50:19 +0200 | [diff] [blame] | 3118 | } |
| 3119 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3120 | static inline bool rdma_ib_or_roce(const struct ib_device *device, |
| 3121 | u32 port_num) |
Michael Wang | de66be9 | 2015-05-05 14:50:19 +0200 | [diff] [blame] | 3122 | { |
Matan Barak | 7766a99 | 2015-12-23 14:56:50 +0200 | [diff] [blame] | 3123 | return rdma_protocol_ib(device, port_num) || |
| 3124 | rdma_protocol_roce(device, port_num); |
Michael Wang | de66be9 | 2015-05-05 14:50:19 +0200 | [diff] [blame] | 3125 | } |
| 3126 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3127 | static inline bool rdma_protocol_raw_packet(const struct ib_device *device, |
| 3128 | u32 port_num) |
Or Gerlitz | aa773bd | 2017-01-24 13:02:35 +0200 | [diff] [blame] | 3129 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3130 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3131 | RDMA_CORE_CAP_PROT_RAW_PACKET; |
Or Gerlitz | aa773bd | 2017-01-24 13:02:35 +0200 | [diff] [blame] | 3132 | } |
| 3133 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3134 | static inline bool rdma_protocol_usnic(const struct ib_device *device, |
| 3135 | u32 port_num) |
Or Gerlitz | ce1e055 | 2017-01-24 13:02:38 +0200 | [diff] [blame] | 3136 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3137 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3138 | RDMA_CORE_CAP_PROT_USNIC; |
Or Gerlitz | ce1e055 | 2017-01-24 13:02:38 +0200 | [diff] [blame] | 3139 | } |
| 3140 | |
Michael Wang | c757dea | 2015-05-05 14:50:32 +0200 | [diff] [blame] | 3141 | /** |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3142 | * rdma_cap_ib_mad - Check if the port of a device supports Infiniband |
Michael Wang | c757dea | 2015-05-05 14:50:32 +0200 | [diff] [blame] | 3143 | * Management Datagrams. |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3144 | * @device: Device to check |
| 3145 | * @port_num: Port number to check |
Michael Wang | c757dea | 2015-05-05 14:50:32 +0200 | [diff] [blame] | 3146 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3147 | * Management Datagrams (MAD) are a required part of the InfiniBand |
| 3148 | * specification and are supported on all InfiniBand devices. A slightly |
| 3149 | * extended version are also supported on OPA interfaces. |
Michael Wang | c757dea | 2015-05-05 14:50:32 +0200 | [diff] [blame] | 3150 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3151 | * Return: true if the port supports sending/receiving of MAD packets. |
Michael Wang | c757dea | 2015-05-05 14:50:32 +0200 | [diff] [blame] | 3152 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3153 | static inline bool rdma_cap_ib_mad(const struct ib_device *device, u32 port_num) |
Michael Wang | c757dea | 2015-05-05 14:50:32 +0200 | [diff] [blame] | 3154 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3155 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3156 | RDMA_CORE_CAP_IB_MAD; |
Michael Wang | c757dea | 2015-05-05 14:50:32 +0200 | [diff] [blame] | 3157 | } |
| 3158 | |
Michael Wang | 29541e3 | 2015-05-05 14:50:33 +0200 | [diff] [blame] | 3159 | /** |
Ira Weiny | 65995fe | 2015-06-06 14:38:32 -0400 | [diff] [blame] | 3160 | * rdma_cap_opa_mad - Check if the port of device provides support for OPA |
| 3161 | * Management Datagrams. |
| 3162 | * @device: Device to check |
| 3163 | * @port_num: Port number to check |
| 3164 | * |
| 3165 | * Intel OmniPath devices extend and/or replace the InfiniBand Management |
| 3166 | * datagrams with their own versions. These OPA MADs share many but not all of |
| 3167 | * the characteristics of InfiniBand MADs. |
| 3168 | * |
| 3169 | * OPA MADs differ in the following ways: |
| 3170 | * |
| 3171 | * 1) MADs are variable size up to 2K |
| 3172 | * IBTA defined MADs remain fixed at 256 bytes |
| 3173 | * 2) OPA SMPs must carry valid PKeys |
| 3174 | * 3) OPA SMP packets are a different format |
| 3175 | * |
| 3176 | * Return: true if the port supports OPA MAD packet formats. |
| 3177 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3178 | static inline bool rdma_cap_opa_mad(struct ib_device *device, u32 port_num) |
Ira Weiny | 65995fe | 2015-06-06 14:38:32 -0400 | [diff] [blame] | 3179 | { |
Leon Romanovsky | d3243da | 2019-03-10 17:27:46 +0200 | [diff] [blame] | 3180 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3181 | RDMA_CORE_CAP_OPA_MAD; |
Ira Weiny | 65995fe | 2015-06-06 14:38:32 -0400 | [diff] [blame] | 3182 | } |
| 3183 | |
| 3184 | /** |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3185 | * rdma_cap_ib_smi - Check if the port of a device provides an Infiniband |
| 3186 | * Subnet Management Agent (SMA) on the Subnet Management Interface (SMI). |
| 3187 | * @device: Device to check |
| 3188 | * @port_num: Port number to check |
Michael Wang | 29541e3 | 2015-05-05 14:50:33 +0200 | [diff] [blame] | 3189 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3190 | * Each InfiniBand node is required to provide a Subnet Management Agent |
| 3191 | * that the subnet manager can access. Prior to the fabric being fully |
| 3192 | * configured by the subnet manager, the SMA is accessed via a well known |
| 3193 | * interface called the Subnet Management Interface (SMI). This interface |
| 3194 | * uses directed route packets to communicate with the SM to get around the |
| 3195 | * chicken and egg problem of the SM needing to know what's on the fabric |
| 3196 | * in order to configure the fabric, and needing to configure the fabric in |
| 3197 | * order to send packets to the devices on the fabric. These directed |
| 3198 | * route packets do not need the fabric fully configured in order to reach |
| 3199 | * their destination. The SMI is the only method allowed to send |
| 3200 | * directed route packets on an InfiniBand fabric. |
Michael Wang | 29541e3 | 2015-05-05 14:50:33 +0200 | [diff] [blame] | 3201 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3202 | * Return: true if the port provides an SMI. |
Michael Wang | 29541e3 | 2015-05-05 14:50:33 +0200 | [diff] [blame] | 3203 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3204 | static inline bool rdma_cap_ib_smi(const struct ib_device *device, u32 port_num) |
Michael Wang | 29541e3 | 2015-05-05 14:50:33 +0200 | [diff] [blame] | 3205 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3206 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3207 | RDMA_CORE_CAP_IB_SMI; |
Michael Wang | 29541e3 | 2015-05-05 14:50:33 +0200 | [diff] [blame] | 3208 | } |
| 3209 | |
Michael Wang | 72219cea | 2015-05-05 14:50:34 +0200 | [diff] [blame] | 3210 | /** |
| 3211 | * rdma_cap_ib_cm - Check if the port of device has the capability Infiniband |
| 3212 | * Communication Manager. |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3213 | * @device: Device to check |
| 3214 | * @port_num: Port number to check |
Michael Wang | 72219cea | 2015-05-05 14:50:34 +0200 | [diff] [blame] | 3215 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3216 | * The InfiniBand Communication Manager is one of many pre-defined General |
| 3217 | * Service Agents (GSA) that are accessed via the General Service |
| 3218 | * Interface (GSI). It's role is to facilitate establishment of connections |
| 3219 | * between nodes as well as other management related tasks for established |
| 3220 | * connections. |
Michael Wang | 72219cea | 2015-05-05 14:50:34 +0200 | [diff] [blame] | 3221 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3222 | * Return: true if the port supports an IB CM (this does not guarantee that |
| 3223 | * a CM is actually running however). |
Michael Wang | 72219cea | 2015-05-05 14:50:34 +0200 | [diff] [blame] | 3224 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3225 | static inline bool rdma_cap_ib_cm(const struct ib_device *device, u32 port_num) |
Michael Wang | 72219cea | 2015-05-05 14:50:34 +0200 | [diff] [blame] | 3226 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3227 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3228 | RDMA_CORE_CAP_IB_CM; |
Michael Wang | 72219cea | 2015-05-05 14:50:34 +0200 | [diff] [blame] | 3229 | } |
| 3230 | |
Michael Wang | 0421533 | 2015-05-05 14:50:35 +0200 | [diff] [blame] | 3231 | /** |
| 3232 | * rdma_cap_iw_cm - Check if the port of device has the capability IWARP |
| 3233 | * Communication Manager. |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3234 | * @device: Device to check |
| 3235 | * @port_num: Port number to check |
Michael Wang | 0421533 | 2015-05-05 14:50:35 +0200 | [diff] [blame] | 3236 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3237 | * Similar to above, but specific to iWARP connections which have a different |
| 3238 | * managment protocol than InfiniBand. |
Michael Wang | 0421533 | 2015-05-05 14:50:35 +0200 | [diff] [blame] | 3239 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3240 | * Return: true if the port supports an iWARP CM (this does not guarantee that |
| 3241 | * a CM is actually running however). |
Michael Wang | 0421533 | 2015-05-05 14:50:35 +0200 | [diff] [blame] | 3242 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3243 | static inline bool rdma_cap_iw_cm(const struct ib_device *device, u32 port_num) |
Michael Wang | 0421533 | 2015-05-05 14:50:35 +0200 | [diff] [blame] | 3244 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3245 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3246 | RDMA_CORE_CAP_IW_CM; |
Michael Wang | 0421533 | 2015-05-05 14:50:35 +0200 | [diff] [blame] | 3247 | } |
| 3248 | |
Michael Wang | fe53ba2 | 2015-05-05 14:50:36 +0200 | [diff] [blame] | 3249 | /** |
| 3250 | * rdma_cap_ib_sa - Check if the port of device has the capability Infiniband |
| 3251 | * Subnet Administration. |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3252 | * @device: Device to check |
| 3253 | * @port_num: Port number to check |
Michael Wang | fe53ba2 | 2015-05-05 14:50:36 +0200 | [diff] [blame] | 3254 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3255 | * An InfiniBand Subnet Administration (SA) service is a pre-defined General |
| 3256 | * Service Agent (GSA) provided by the Subnet Manager (SM). On InfiniBand |
| 3257 | * fabrics, devices should resolve routes to other hosts by contacting the |
| 3258 | * SA to query the proper route. |
Michael Wang | fe53ba2 | 2015-05-05 14:50:36 +0200 | [diff] [blame] | 3259 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3260 | * Return: true if the port should act as a client to the fabric Subnet |
| 3261 | * Administration interface. This does not imply that the SA service is |
| 3262 | * running locally. |
Michael Wang | fe53ba2 | 2015-05-05 14:50:36 +0200 | [diff] [blame] | 3263 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3264 | static inline bool rdma_cap_ib_sa(const struct ib_device *device, u32 port_num) |
Michael Wang | fe53ba2 | 2015-05-05 14:50:36 +0200 | [diff] [blame] | 3265 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3266 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3267 | RDMA_CORE_CAP_IB_SA; |
Michael Wang | fe53ba2 | 2015-05-05 14:50:36 +0200 | [diff] [blame] | 3268 | } |
| 3269 | |
Michael Wang | a31ad3b | 2015-05-05 14:50:37 +0200 | [diff] [blame] | 3270 | /** |
| 3271 | * rdma_cap_ib_mcast - Check if the port of device has the capability Infiniband |
| 3272 | * Multicast. |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3273 | * @device: Device to check |
| 3274 | * @port_num: Port number to check |
Michael Wang | a31ad3b | 2015-05-05 14:50:37 +0200 | [diff] [blame] | 3275 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3276 | * InfiniBand multicast registration is more complex than normal IPv4 or |
| 3277 | * IPv6 multicast registration. Each Host Channel Adapter must register |
| 3278 | * with the Subnet Manager when it wishes to join a multicast group. It |
| 3279 | * should do so only once regardless of how many queue pairs it subscribes |
| 3280 | * to this group. And it should leave the group only after all queue pairs |
| 3281 | * attached to the group have been detached. |
Michael Wang | a31ad3b | 2015-05-05 14:50:37 +0200 | [diff] [blame] | 3282 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3283 | * Return: true if the port must undertake the additional adminstrative |
| 3284 | * overhead of registering/unregistering with the SM and tracking of the |
| 3285 | * total number of queue pairs attached to the multicast group. |
Michael Wang | a31ad3b | 2015-05-05 14:50:37 +0200 | [diff] [blame] | 3286 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3287 | static inline bool rdma_cap_ib_mcast(const struct ib_device *device, |
| 3288 | u32 port_num) |
Michael Wang | a31ad3b | 2015-05-05 14:50:37 +0200 | [diff] [blame] | 3289 | { |
| 3290 | return rdma_cap_ib_sa(device, port_num); |
| 3291 | } |
| 3292 | |
Michael Wang | bc0f1d7 | 2015-05-05 14:50:38 +0200 | [diff] [blame] | 3293 | /** |
Michael Wang | 30a74ef | 2015-05-05 14:50:39 +0200 | [diff] [blame] | 3294 | * rdma_cap_af_ib - Check if the port of device has the capability |
| 3295 | * Native Infiniband Address. |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3296 | * @device: Device to check |
| 3297 | * @port_num: Port number to check |
Michael Wang | 30a74ef | 2015-05-05 14:50:39 +0200 | [diff] [blame] | 3298 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3299 | * InfiniBand addressing uses a port's GUID + Subnet Prefix to make a default |
| 3300 | * GID. RoCE uses a different mechanism, but still generates a GID via |
| 3301 | * a prescribed mechanism and port specific data. |
Michael Wang | 30a74ef | 2015-05-05 14:50:39 +0200 | [diff] [blame] | 3302 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3303 | * Return: true if the port uses a GID address to identify devices on the |
| 3304 | * network. |
Michael Wang | 30a74ef | 2015-05-05 14:50:39 +0200 | [diff] [blame] | 3305 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3306 | static inline bool rdma_cap_af_ib(const struct ib_device *device, u32 port_num) |
Michael Wang | 30a74ef | 2015-05-05 14:50:39 +0200 | [diff] [blame] | 3307 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3308 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3309 | RDMA_CORE_CAP_AF_IB; |
Michael Wang | 30a74ef | 2015-05-05 14:50:39 +0200 | [diff] [blame] | 3310 | } |
| 3311 | |
| 3312 | /** |
Michael Wang | 227128f | 2015-05-05 14:50:40 +0200 | [diff] [blame] | 3313 | * rdma_cap_eth_ah - Check if the port of device has the capability |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3314 | * Ethernet Address Handle. |
| 3315 | * @device: Device to check |
| 3316 | * @port_num: Port number to check |
Michael Wang | 227128f | 2015-05-05 14:50:40 +0200 | [diff] [blame] | 3317 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3318 | * RoCE is InfiniBand over Ethernet, and it uses a well defined technique |
| 3319 | * to fabricate GIDs over Ethernet/IP specific addresses native to the |
| 3320 | * port. Normally, packet headers are generated by the sending host |
| 3321 | * adapter, but when sending connectionless datagrams, we must manually |
| 3322 | * inject the proper headers for the fabric we are communicating over. |
Michael Wang | 227128f | 2015-05-05 14:50:40 +0200 | [diff] [blame] | 3323 | * |
Michael Wang | 296ec00 | 2015-05-18 10:41:45 +0200 | [diff] [blame] | 3324 | * Return: true if we are running as a RoCE port and must force the |
| 3325 | * addition of a Global Route Header built from our Ethernet Address |
| 3326 | * Handle into our header list for connectionless packets. |
Michael Wang | 227128f | 2015-05-05 14:50:40 +0200 | [diff] [blame] | 3327 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3328 | static inline bool rdma_cap_eth_ah(const struct ib_device *device, u32 port_num) |
Michael Wang | 227128f | 2015-05-05 14:50:40 +0200 | [diff] [blame] | 3329 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3330 | return device->port_data[port_num].immutable.core_cap_flags & |
| 3331 | RDMA_CORE_CAP_ETH_AH; |
Michael Wang | 227128f | 2015-05-05 14:50:40 +0200 | [diff] [blame] | 3332 | } |
| 3333 | |
| 3334 | /** |
Dasaratharaman Chandramouli | 94d595c | 2017-03-20 19:38:09 -0400 | [diff] [blame] | 3335 | * rdma_cap_opa_ah - Check if the port of device supports |
| 3336 | * OPA Address handles |
| 3337 | * @device: Device to check |
| 3338 | * @port_num: Port number to check |
| 3339 | * |
| 3340 | * Return: true if we are running on an OPA device which supports |
| 3341 | * the extended OPA addressing. |
| 3342 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3343 | static inline bool rdma_cap_opa_ah(struct ib_device *device, u32 port_num) |
Dasaratharaman Chandramouli | 94d595c | 2017-03-20 19:38:09 -0400 | [diff] [blame] | 3344 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3345 | return (device->port_data[port_num].immutable.core_cap_flags & |
Dasaratharaman Chandramouli | 94d595c | 2017-03-20 19:38:09 -0400 | [diff] [blame] | 3346 | RDMA_CORE_CAP_OPA_AH) == RDMA_CORE_CAP_OPA_AH; |
| 3347 | } |
| 3348 | |
| 3349 | /** |
Ira Weiny | 337877a | 2015-06-06 14:38:29 -0400 | [diff] [blame] | 3350 | * rdma_max_mad_size - Return the max MAD size required by this RDMA Port. |
| 3351 | * |
| 3352 | * @device: Device |
| 3353 | * @port_num: Port number |
| 3354 | * |
| 3355 | * This MAD size includes the MAD headers and MAD payload. No other headers |
| 3356 | * are included. |
| 3357 | * |
| 3358 | * Return the max MAD size required by the Port. Will return 0 if the port |
| 3359 | * does not support MADs |
| 3360 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3361 | static inline size_t rdma_max_mad_size(const struct ib_device *device, |
| 3362 | u32 port_num) |
Ira Weiny | 337877a | 2015-06-06 14:38:29 -0400 | [diff] [blame] | 3363 | { |
Jason Gunthorpe | 8ceb135 | 2019-02-12 21:12:48 -0700 | [diff] [blame] | 3364 | return device->port_data[port_num].immutable.max_mad_size; |
Ira Weiny | 337877a | 2015-06-06 14:38:29 -0400 | [diff] [blame] | 3365 | } |
| 3366 | |
Matan Barak | 03db3a2 | 2015-07-30 18:33:26 +0300 | [diff] [blame] | 3367 | /** |
| 3368 | * rdma_cap_roce_gid_table - Check if the port of device uses roce_gid_table |
| 3369 | * @device: Device to check |
| 3370 | * @port_num: Port number to check |
| 3371 | * |
| 3372 | * RoCE GID table mechanism manages the various GIDs for a device. |
| 3373 | * |
| 3374 | * NOTE: if allocating the port's GID table has failed, this call will still |
| 3375 | * return true, but any RoCE GID table API will fail. |
| 3376 | * |
| 3377 | * Return: true if the port uses RoCE GID table mechanism in order to manage |
| 3378 | * its GIDs. |
| 3379 | */ |
| 3380 | static inline bool rdma_cap_roce_gid_table(const struct ib_device *device, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3381 | u32 port_num) |
Matan Barak | 03db3a2 | 2015-07-30 18:33:26 +0300 | [diff] [blame] | 3382 | { |
| 3383 | return rdma_protocol_roce(device, port_num) && |
Kamal Heib | 3023a1e | 2018-12-10 21:09:48 +0200 | [diff] [blame] | 3384 | device->ops.add_gid && device->ops.del_gid; |
Matan Barak | 03db3a2 | 2015-07-30 18:33:26 +0300 | [diff] [blame] | 3385 | } |
| 3386 | |
Christoph Hellwig | 002516e | 2016-05-03 18:01:05 +0200 | [diff] [blame] | 3387 | /* |
| 3388 | * Check if the device supports READ W/ INVALIDATE. |
| 3389 | */ |
| 3390 | static inline bool rdma_cap_read_inv(struct ib_device *dev, u32 port_num) |
| 3391 | { |
| 3392 | /* |
| 3393 | * iWarp drivers must support READ W/ INVALIDATE. No other protocol |
| 3394 | * has support for it yet. |
| 3395 | */ |
| 3396 | return rdma_protocol_iwarp(dev, port_num); |
| 3397 | } |
| 3398 | |
Shiraz Saleem | 4a35339 | 2019-05-06 08:53:32 -0500 | [diff] [blame] | 3399 | /** |
Kaike Wan | 6d72344 | 2020-05-11 12:06:18 -0400 | [diff] [blame] | 3400 | * rdma_core_cap_opa_port - Return whether the RDMA Port is OPA or not. |
| 3401 | * @device: Device |
| 3402 | * @port_num: 1 based Port number |
| 3403 | * |
| 3404 | * Return true if port is an Intel OPA port , false if not |
| 3405 | */ |
| 3406 | static inline bool rdma_core_cap_opa_port(struct ib_device *device, |
| 3407 | u32 port_num) |
| 3408 | { |
| 3409 | return (device->port_data[port_num].immutable.core_cap_flags & |
| 3410 | RDMA_CORE_PORT_INTEL_OPA) == RDMA_CORE_PORT_INTEL_OPA; |
| 3411 | } |
| 3412 | |
| 3413 | /** |
| 3414 | * rdma_mtu_enum_to_int - Return the mtu of the port as an integer value. |
| 3415 | * @device: Device |
| 3416 | * @port_num: Port number |
| 3417 | * @mtu: enum value of MTU |
| 3418 | * |
| 3419 | * Return the MTU size supported by the port as an integer value. Will return |
| 3420 | * -1 if enum value of mtu is not supported. |
| 3421 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3422 | static inline int rdma_mtu_enum_to_int(struct ib_device *device, u32 port, |
Kaike Wan | 6d72344 | 2020-05-11 12:06:18 -0400 | [diff] [blame] | 3423 | int mtu) |
| 3424 | { |
| 3425 | if (rdma_core_cap_opa_port(device, port)) |
| 3426 | return opa_mtu_enum_to_int((enum opa_mtu)mtu); |
| 3427 | else |
| 3428 | return ib_mtu_enum_to_int((enum ib_mtu)mtu); |
| 3429 | } |
| 3430 | |
| 3431 | /** |
| 3432 | * rdma_mtu_from_attr - Return the mtu of the port from the port attribute. |
| 3433 | * @device: Device |
| 3434 | * @port_num: Port number |
| 3435 | * @attr: port attribute |
| 3436 | * |
| 3437 | * Return the MTU size supported by the port as an integer value. |
| 3438 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3439 | static inline int rdma_mtu_from_attr(struct ib_device *device, u32 port, |
Kaike Wan | 6d72344 | 2020-05-11 12:06:18 -0400 | [diff] [blame] | 3440 | struct ib_port_attr *attr) |
| 3441 | { |
| 3442 | if (rdma_core_cap_opa_port(device, port)) |
| 3443 | return attr->phys_mtu; |
| 3444 | else |
| 3445 | return ib_mtu_enum_to_int(attr->max_mtu); |
| 3446 | } |
| 3447 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3448 | int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port, |
Eli Cohen | 50174a7 | 2016-03-11 22:58:38 +0200 | [diff] [blame] | 3449 | int state); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3450 | int ib_get_vf_config(struct ib_device *device, int vf, u32 port, |
Eli Cohen | 50174a7 | 2016-03-11 22:58:38 +0200 | [diff] [blame] | 3451 | struct ifla_vf_info *info); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3452 | int ib_get_vf_stats(struct ib_device *device, int vf, u32 port, |
Eli Cohen | 50174a7 | 2016-03-11 22:58:38 +0200 | [diff] [blame] | 3453 | struct ifla_vf_stats *stats); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3454 | int ib_get_vf_guid(struct ib_device *device, int vf, u32 port, |
Danit Goldberg | bfcb3c5d | 2019-11-06 15:08:32 +0200 | [diff] [blame] | 3455 | struct ifla_vf_guid *node_guid, |
| 3456 | struct ifla_vf_guid *port_guid); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3457 | int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid, |
Eli Cohen | 50174a7 | 2016-03-11 22:58:38 +0200 | [diff] [blame] | 3458 | int type); |
| 3459 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3460 | int ib_query_pkey(struct ib_device *device, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3461 | u32 port_num, u16 index, u16 *pkey); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3462 | |
| 3463 | int ib_modify_device(struct ib_device *device, |
| 3464 | int device_modify_mask, |
| 3465 | struct ib_device_modify *device_modify); |
| 3466 | |
| 3467 | int ib_modify_port(struct ib_device *device, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3468 | u32 port_num, int port_modify_mask, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3469 | struct ib_port_modify *port_modify); |
| 3470 | |
Yosef Etigin | 5eb620c | 2007-05-14 07:26:51 +0300 | [diff] [blame] | 3471 | int ib_find_gid(struct ib_device *device, union ib_gid *gid, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3472 | u32 *port_num, u16 *index); |
Yosef Etigin | 5eb620c | 2007-05-14 07:26:51 +0300 | [diff] [blame] | 3473 | |
| 3474 | int ib_find_pkey(struct ib_device *device, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3475 | u32 port_num, u16 pkey, u16 *index); |
Yosef Etigin | 5eb620c | 2007-05-14 07:26:51 +0300 | [diff] [blame] | 3476 | |
Christoph Hellwig | ed082d3 | 2016-09-05 12:56:17 +0200 | [diff] [blame] | 3477 | enum ib_pd_flags { |
| 3478 | /* |
| 3479 | * Create a memory registration for all memory in the system and place |
| 3480 | * the rkey for it into pd->unsafe_global_rkey. This can be used by |
| 3481 | * ULPs to avoid the overhead of dynamic MRs. |
| 3482 | * |
| 3483 | * This flag is generally considered unsafe and must only be used in |
| 3484 | * extremly trusted environments. Every use of it will log a warning |
| 3485 | * in the kernel log. |
| 3486 | */ |
| 3487 | IB_PD_UNSAFE_GLOBAL_RKEY = 0x01, |
| 3488 | }; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3489 | |
Christoph Hellwig | ed082d3 | 2016-09-05 12:56:17 +0200 | [diff] [blame] | 3490 | struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags, |
| 3491 | const char *caller); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3492 | |
Mauro Carvalho Chehab | 2988ca0 | 2020-12-01 13:08:55 +0100 | [diff] [blame] | 3493 | /** |
| 3494 | * ib_alloc_pd - Allocates an unused protection domain. |
| 3495 | * @device: The device on which to allocate the protection domain. |
| 3496 | * @flags: protection domain flags |
| 3497 | * |
| 3498 | * A protection domain object provides an association between QPs, shared |
| 3499 | * receive queues, address handles, memory regions, and memory windows. |
| 3500 | * |
| 3501 | * Every PD has a local_dma_lkey which can be used as the lkey value for local |
| 3502 | * memory operations. |
| 3503 | */ |
Christoph Hellwig | ed082d3 | 2016-09-05 12:56:17 +0200 | [diff] [blame] | 3504 | #define ib_alloc_pd(device, flags) \ |
Leon Romanovsky | e449644 | 2018-01-28 11:17:18 +0200 | [diff] [blame] | 3505 | __ib_alloc_pd((device), (flags), KBUILD_MODNAME) |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3506 | |
Leon Romanovsky | 91a7c58 | 2020-09-07 15:09:13 +0300 | [diff] [blame] | 3507 | int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3508 | |
| 3509 | /** |
| 3510 | * ib_dealloc_pd - Deallocate kernel PD |
| 3511 | * @pd: The protection domain |
| 3512 | * |
| 3513 | * NOTE: for user PD use ib_dealloc_pd_user with valid udata! |
| 3514 | */ |
| 3515 | static inline void ib_dealloc_pd(struct ib_pd *pd) |
| 3516 | { |
Leon Romanovsky | 91a7c58 | 2020-09-07 15:09:13 +0300 | [diff] [blame] | 3517 | int ret = ib_dealloc_pd_user(pd, NULL); |
| 3518 | |
| 3519 | WARN_ONCE(ret, "Destroy of kernel PD shouldn't fail"); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3520 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3521 | |
Gal Pressman | b090c4e | 2018-12-12 11:09:05 +0200 | [diff] [blame] | 3522 | enum rdma_create_ah_flags { |
| 3523 | /* In a sleepable context */ |
| 3524 | RDMA_CREATE_AH_SLEEPABLE = BIT(0), |
| 3525 | }; |
| 3526 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3527 | /** |
Dasaratharaman Chandramouli | 0a18cfe | 2017-04-29 14:41:19 -0400 | [diff] [blame] | 3528 | * rdma_create_ah - Creates an address handle for the given address vector. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3529 | * @pd: The protection domain associated with the address handle. |
| 3530 | * @ah_attr: The attributes of the address vector. |
Gal Pressman | b090c4e | 2018-12-12 11:09:05 +0200 | [diff] [blame] | 3531 | * @flags: Create address handle flags (see enum rdma_create_ah_flags). |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3532 | * |
| 3533 | * The address handle is used to reference a local or global destination |
| 3534 | * in all UD QP post sends. |
| 3535 | */ |
Gal Pressman | b090c4e | 2018-12-12 11:09:05 +0200 | [diff] [blame] | 3536 | struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr, |
| 3537 | u32 flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3538 | |
| 3539 | /** |
Parav Pandit | 5cda658 | 2017-10-16 08:45:12 +0300 | [diff] [blame] | 3540 | * rdma_create_user_ah - Creates an address handle for the given address vector. |
| 3541 | * It resolves destination mac address for ah attribute of RoCE type. |
| 3542 | * @pd: The protection domain associated with the address handle. |
| 3543 | * @ah_attr: The attributes of the address vector. |
| 3544 | * @udata: pointer to user's input output buffer information need by |
| 3545 | * provider driver. |
| 3546 | * |
| 3547 | * It returns 0 on success and returns appropriate error code on error. |
| 3548 | * The address handle is used to reference a local or global destination |
| 3549 | * in all UD QP post sends. |
| 3550 | */ |
| 3551 | struct ib_ah *rdma_create_user_ah(struct ib_pd *pd, |
| 3552 | struct rdma_ah_attr *ah_attr, |
| 3553 | struct ib_udata *udata); |
| 3554 | /** |
Moni Shoua | 850d8fd | 2016-11-10 11:30:56 +0200 | [diff] [blame] | 3555 | * ib_get_gids_from_rdma_hdr - Get sgid and dgid from GRH or IPv4 header |
| 3556 | * work completion. |
| 3557 | * @hdr: the L3 header to parse |
| 3558 | * @net_type: type of header to parse |
| 3559 | * @sgid: place to store source gid |
| 3560 | * @dgid: place to store destination gid |
| 3561 | */ |
| 3562 | int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr, |
| 3563 | enum rdma_network_type net_type, |
| 3564 | union ib_gid *sgid, union ib_gid *dgid); |
| 3565 | |
| 3566 | /** |
| 3567 | * ib_get_rdma_header_version - Get the header version |
| 3568 | * @hdr: the L3 header to parse |
| 3569 | */ |
| 3570 | int ib_get_rdma_header_version(const union rdma_network_hdr *hdr); |
| 3571 | |
| 3572 | /** |
Parav Pandit | f6bdb14 | 2017-11-14 14:52:17 +0200 | [diff] [blame] | 3573 | * ib_init_ah_attr_from_wc - Initializes address handle attributes from a |
Sean Hefty | 4e00d69 | 2006-06-17 20:37:39 -0700 | [diff] [blame] | 3574 | * work completion. |
| 3575 | * @device: Device on which the received message arrived. |
| 3576 | * @port_num: Port on which the received message arrived. |
| 3577 | * @wc: Work completion associated with the received message. |
| 3578 | * @grh: References the received global route header. This parameter is |
| 3579 | * ignored unless the work completion indicates that the GRH is valid. |
| 3580 | * @ah_attr: Returned attributes that can be used when creating an address |
| 3581 | * handle for replying to the message. |
Parav Pandit | b740321 | 2018-06-19 10:59:14 +0300 | [diff] [blame] | 3582 | * When ib_init_ah_attr_from_wc() returns success, |
| 3583 | * (a) for IB link layer it optionally contains a reference to SGID attribute |
| 3584 | * when GRH is present for IB link layer. |
| 3585 | * (b) for RoCE link layer it contains a reference to SGID attribute. |
| 3586 | * User must invoke rdma_cleanup_ah_attr_gid_attr() to release reference to SGID |
| 3587 | * attributes which are initialized using ib_init_ah_attr_from_wc(). |
| 3588 | * |
Sean Hefty | 4e00d69 | 2006-06-17 20:37:39 -0700 | [diff] [blame] | 3589 | */ |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3590 | int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num, |
Parav Pandit | f6bdb14 | 2017-11-14 14:52:17 +0200 | [diff] [blame] | 3591 | const struct ib_wc *wc, const struct ib_grh *grh, |
| 3592 | struct rdma_ah_attr *ah_attr); |
Sean Hefty | 4e00d69 | 2006-06-17 20:37:39 -0700 | [diff] [blame] | 3593 | |
| 3594 | /** |
Hal Rosenstock | 513789e | 2005-07-27 11:45:34 -0700 | [diff] [blame] | 3595 | * ib_create_ah_from_wc - Creates an address handle associated with the |
| 3596 | * sender of the specified work completion. |
| 3597 | * @pd: The protection domain associated with the address handle. |
| 3598 | * @wc: Work completion information associated with a received message. |
| 3599 | * @grh: References the received global route header. This parameter is |
| 3600 | * ignored unless the work completion indicates that the GRH is valid. |
| 3601 | * @port_num: The outbound port number to associate with the address. |
| 3602 | * |
| 3603 | * The address handle is used to reference a local or global destination |
| 3604 | * in all UD QP post sends. |
| 3605 | */ |
Ira Weiny | 73cdaae | 2015-05-31 17:15:31 -0400 | [diff] [blame] | 3606 | struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 3607 | const struct ib_grh *grh, u32 port_num); |
Hal Rosenstock | 513789e | 2005-07-27 11:45:34 -0700 | [diff] [blame] | 3608 | |
| 3609 | /** |
Dasaratharaman Chandramouli | 67b985b | 2017-04-29 14:41:20 -0400 | [diff] [blame] | 3610 | * rdma_modify_ah - Modifies the address vector associated with an address |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3611 | * handle. |
| 3612 | * @ah: The address handle to modify. |
| 3613 | * @ah_attr: The new address vector attributes to associate with the |
| 3614 | * address handle. |
| 3615 | */ |
Dasaratharaman Chandramouli | 67b985b | 2017-04-29 14:41:20 -0400 | [diff] [blame] | 3616 | int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3617 | |
| 3618 | /** |
Dasaratharaman Chandramouli | bfbfd66 | 2017-04-29 14:41:21 -0400 | [diff] [blame] | 3619 | * rdma_query_ah - Queries the address vector associated with an address |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3620 | * handle. |
| 3621 | * @ah: The address handle to query. |
| 3622 | * @ah_attr: The address vector attributes associated with the address |
| 3623 | * handle. |
| 3624 | */ |
Dasaratharaman Chandramouli | bfbfd66 | 2017-04-29 14:41:21 -0400 | [diff] [blame] | 3625 | int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3626 | |
Gal Pressman | 2553ba21 | 2018-12-12 11:09:06 +0200 | [diff] [blame] | 3627 | enum rdma_destroy_ah_flags { |
| 3628 | /* In a sleepable context */ |
| 3629 | RDMA_DESTROY_AH_SLEEPABLE = BIT(0), |
| 3630 | }; |
| 3631 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3632 | /** |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3633 | * rdma_destroy_ah_user - Destroys an address handle. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3634 | * @ah: The address handle to destroy. |
Gal Pressman | 2553ba21 | 2018-12-12 11:09:06 +0200 | [diff] [blame] | 3635 | * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags). |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3636 | * @udata: Valid user data or NULL for kernel objects |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3637 | */ |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3638 | int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata); |
| 3639 | |
| 3640 | /** |
| 3641 | * rdma_destroy_ah - Destroys an kernel address handle. |
| 3642 | * @ah: The address handle to destroy. |
| 3643 | * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags). |
| 3644 | * |
| 3645 | * NOTE: for user ah use rdma_destroy_ah_user with valid udata! |
| 3646 | */ |
Leon Romanovsky | 9a9ebf8 | 2020-09-07 15:09:14 +0300 | [diff] [blame] | 3647 | static inline void rdma_destroy_ah(struct ib_ah *ah, u32 flags) |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3648 | { |
Leon Romanovsky | 9a9ebf8 | 2020-09-07 15:09:14 +0300 | [diff] [blame] | 3649 | int ret = rdma_destroy_ah_user(ah, flags, NULL); |
| 3650 | |
| 3651 | WARN_ONCE(ret, "Destroy of kernel AH shouldn't fail"); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3652 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3653 | |
Jason Gunthorpe | b0810b0 | 2020-05-06 11:24:39 +0300 | [diff] [blame] | 3654 | struct ib_srq *ib_create_srq_user(struct ib_pd *pd, |
| 3655 | struct ib_srq_init_attr *srq_init_attr, |
| 3656 | struct ib_usrq_object *uobject, |
| 3657 | struct ib_udata *udata); |
| 3658 | static inline struct ib_srq * |
| 3659 | ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr) |
| 3660 | { |
| 3661 | if (!pd->device->ops.create_srq) |
| 3662 | return ERR_PTR(-EOPNOTSUPP); |
| 3663 | |
| 3664 | return ib_create_srq_user(pd, srq_init_attr, NULL, NULL); |
| 3665 | } |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 3666 | |
| 3667 | /** |
| 3668 | * ib_modify_srq - Modifies the attributes for the specified SRQ. |
| 3669 | * @srq: The SRQ to modify. |
| 3670 | * @srq_attr: On input, specifies the SRQ attributes to modify. On output, |
| 3671 | * the current values of selected SRQ attributes are returned. |
| 3672 | * @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ |
| 3673 | * are being modified. |
| 3674 | * |
| 3675 | * The mask may contain IB_SRQ_MAX_WR to resize the SRQ and/or |
| 3676 | * IB_SRQ_LIMIT to set the SRQ's limit and request notification when |
| 3677 | * the number of receives queued drops below the limit. |
| 3678 | */ |
| 3679 | int ib_modify_srq(struct ib_srq *srq, |
| 3680 | struct ib_srq_attr *srq_attr, |
| 3681 | enum ib_srq_attr_mask srq_attr_mask); |
| 3682 | |
| 3683 | /** |
| 3684 | * ib_query_srq - Returns the attribute list and current values for the |
| 3685 | * specified SRQ. |
| 3686 | * @srq: The SRQ to query. |
| 3687 | * @srq_attr: The attributes of the specified SRQ. |
| 3688 | */ |
| 3689 | int ib_query_srq(struct ib_srq *srq, |
| 3690 | struct ib_srq_attr *srq_attr); |
| 3691 | |
| 3692 | /** |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3693 | * ib_destroy_srq_user - Destroys the specified SRQ. |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 3694 | * @srq: The SRQ to destroy. |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3695 | * @udata: Valid user data or NULL for kernel objects |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 3696 | */ |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3697 | int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata); |
| 3698 | |
| 3699 | /** |
| 3700 | * ib_destroy_srq - Destroys the specified kernel SRQ. |
| 3701 | * @srq: The SRQ to destroy. |
| 3702 | * |
| 3703 | * NOTE: for user srq use ib_destroy_srq_user with valid udata! |
| 3704 | */ |
Leon Romanovsky | 119181d | 2020-09-07 15:09:16 +0300 | [diff] [blame] | 3705 | static inline void ib_destroy_srq(struct ib_srq *srq) |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3706 | { |
Leon Romanovsky | 119181d | 2020-09-07 15:09:16 +0300 | [diff] [blame] | 3707 | int ret = ib_destroy_srq_user(srq, NULL); |
| 3708 | |
| 3709 | WARN_ONCE(ret, "Destroy of kernel SRQ shouldn't fail"); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3710 | } |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 3711 | |
| 3712 | /** |
| 3713 | * ib_post_srq_recv - Posts a list of work requests to the specified SRQ. |
| 3714 | * @srq: The SRQ to post the work request on. |
| 3715 | * @recv_wr: A list of work requests to post on the receive queue. |
| 3716 | * @bad_recv_wr: On an immediate failure, this parameter will reference |
| 3717 | * the work request that failed to be posted on the QP. |
| 3718 | */ |
| 3719 | static inline int ib_post_srq_recv(struct ib_srq *srq, |
Bart Van Assche | d34ac5c | 2018-07-18 09:25:32 -0700 | [diff] [blame] | 3720 | const struct ib_recv_wr *recv_wr, |
| 3721 | const struct ib_recv_wr **bad_recv_wr) |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 3722 | { |
Bart Van Assche | d34ac5c | 2018-07-18 09:25:32 -0700 | [diff] [blame] | 3723 | const struct ib_recv_wr *dummy; |
Bart Van Assche | bb039a8 | 2018-07-18 09:25:16 -0700 | [diff] [blame] | 3724 | |
Kamal Heib | 3023a1e | 2018-12-10 21:09:48 +0200 | [diff] [blame] | 3725 | return srq->device->ops.post_srq_recv(srq, recv_wr, |
| 3726 | bad_recv_wr ? : &dummy); |
Roland Dreier | d41fcc6 | 2005-08-18 12:23:08 -0700 | [diff] [blame] | 3727 | } |
| 3728 | |
Leon Romanovsky | 8da9fe4 | 2021-08-03 21:20:35 +0300 | [diff] [blame] | 3729 | struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd, |
| 3730 | struct ib_qp_init_attr *qp_init_attr, |
| 3731 | const char *caller); |
| 3732 | /** |
| 3733 | * ib_create_qp - Creates a kernel QP associated with the specific protection |
| 3734 | * domain. |
| 3735 | * @pd: The protection domain associated with the QP. |
| 3736 | * @init_attr: A list of initial attributes required to create the |
| 3737 | * QP. If QP creation succeeds, then the attributes are updated to |
| 3738 | * the actual capabilities of the created QP. |
| 3739 | */ |
Leon Romanovsky | 66f57b8 | 2020-11-17 09:01:48 +0200 | [diff] [blame] | 3740 | static inline struct ib_qp *ib_create_qp(struct ib_pd *pd, |
| 3741 | struct ib_qp_init_attr *init_attr) |
| 3742 | { |
Leon Romanovsky | 8da9fe4 | 2021-08-03 21:20:35 +0300 | [diff] [blame] | 3743 | return ib_create_qp_kernel(pd, init_attr, KBUILD_MODNAME); |
Leon Romanovsky | 66f57b8 | 2020-11-17 09:01:48 +0200 | [diff] [blame] | 3744 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3745 | |
| 3746 | /** |
Parav Pandit | a512c2f | 2017-05-23 11:26:08 +0300 | [diff] [blame] | 3747 | * ib_modify_qp_with_udata - Modifies the attributes for the specified QP. |
| 3748 | * @qp: The QP to modify. |
| 3749 | * @attr: On input, specifies the QP attributes to modify. On output, |
| 3750 | * the current values of selected QP attributes are returned. |
| 3751 | * @attr_mask: A bit-mask used to specify which attributes of the QP |
| 3752 | * are being modified. |
| 3753 | * @udata: pointer to user's input output buffer information |
| 3754 | * are being modified. |
| 3755 | * It returns 0 on success and returns appropriate error code on error. |
| 3756 | */ |
| 3757 | int ib_modify_qp_with_udata(struct ib_qp *qp, |
| 3758 | struct ib_qp_attr *attr, |
| 3759 | int attr_mask, |
| 3760 | struct ib_udata *udata); |
| 3761 | |
| 3762 | /** |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3763 | * ib_modify_qp - Modifies the attributes for the specified QP and then |
| 3764 | * transitions the QP to the given state. |
| 3765 | * @qp: The QP to modify. |
| 3766 | * @qp_attr: On input, specifies the QP attributes to modify. On output, |
| 3767 | * the current values of selected QP attributes are returned. |
| 3768 | * @qp_attr_mask: A bit-mask used to specify which attributes of the QP |
| 3769 | * are being modified. |
| 3770 | */ |
| 3771 | int ib_modify_qp(struct ib_qp *qp, |
| 3772 | struct ib_qp_attr *qp_attr, |
| 3773 | int qp_attr_mask); |
| 3774 | |
| 3775 | /** |
| 3776 | * ib_query_qp - Returns the attribute list and current values for the |
| 3777 | * specified QP. |
| 3778 | * @qp: The QP to query. |
| 3779 | * @qp_attr: The attributes of the specified QP. |
| 3780 | * @qp_attr_mask: A bit-mask used to select specific attributes to query. |
| 3781 | * @qp_init_attr: Additional attributes of the selected QP. |
| 3782 | * |
| 3783 | * The qp_attr_mask may be used to limit the query to gathering only the |
| 3784 | * selected attributes. |
| 3785 | */ |
| 3786 | int ib_query_qp(struct ib_qp *qp, |
| 3787 | struct ib_qp_attr *qp_attr, |
| 3788 | int qp_attr_mask, |
| 3789 | struct ib_qp_init_attr *qp_init_attr); |
| 3790 | |
| 3791 | /** |
| 3792 | * ib_destroy_qp - Destroys the specified QP. |
| 3793 | * @qp: The QP to destroy. |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3794 | * @udata: Valid udata or NULL for kernel objects |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3795 | */ |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3796 | int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata); |
| 3797 | |
| 3798 | /** |
| 3799 | * ib_destroy_qp - Destroys the specified kernel QP. |
| 3800 | * @qp: The QP to destroy. |
| 3801 | * |
| 3802 | * NOTE: for user qp use ib_destroy_qp_user with valid udata! |
| 3803 | */ |
| 3804 | static inline int ib_destroy_qp(struct ib_qp *qp) |
| 3805 | { |
| 3806 | return ib_destroy_qp_user(qp, NULL); |
| 3807 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3808 | |
| 3809 | /** |
Sean Hefty | 0e0ec7e | 2011-08-08 15:31:51 -0700 | [diff] [blame] | 3810 | * ib_open_qp - Obtain a reference to an existing sharable QP. |
| 3811 | * @xrcd - XRC domain |
| 3812 | * @qp_open_attr: Attributes identifying the QP to open. |
| 3813 | * |
| 3814 | * Returns a reference to a sharable QP. |
| 3815 | */ |
| 3816 | struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd, |
| 3817 | struct ib_qp_open_attr *qp_open_attr); |
| 3818 | |
| 3819 | /** |
| 3820 | * ib_close_qp - Release an external reference to a QP. |
Sean Hefty | d3d72d9 | 2011-05-26 23:06:44 -0700 | [diff] [blame] | 3821 | * @qp: The QP handle to release |
| 3822 | * |
Sean Hefty | 0e0ec7e | 2011-08-08 15:31:51 -0700 | [diff] [blame] | 3823 | * The opened QP handle is released by the caller. The underlying |
| 3824 | * shared QP is not destroyed until all internal references are released. |
Sean Hefty | d3d72d9 | 2011-05-26 23:06:44 -0700 | [diff] [blame] | 3825 | */ |
Sean Hefty | 0e0ec7e | 2011-08-08 15:31:51 -0700 | [diff] [blame] | 3826 | int ib_close_qp(struct ib_qp *qp); |
Sean Hefty | d3d72d9 | 2011-05-26 23:06:44 -0700 | [diff] [blame] | 3827 | |
| 3828 | /** |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3829 | * ib_post_send - Posts a list of work requests to the send queue of |
| 3830 | * the specified QP. |
| 3831 | * @qp: The QP to post the work request on. |
| 3832 | * @send_wr: A list of work requests to post on the send queue. |
| 3833 | * @bad_send_wr: On an immediate failure, this parameter will reference |
| 3834 | * the work request that failed to be posted on the QP. |
Bart Van Assche | 55464d4 | 2009-12-09 14:20:04 -0800 | [diff] [blame] | 3835 | * |
| 3836 | * While IBA Vol. 1 section 11.4.1.1 specifies that if an immediate |
| 3837 | * error is returned, the QP state shall not be affected, |
| 3838 | * ib_post_send() will return an immediate error after queueing any |
| 3839 | * earlier work requests in the list. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3840 | */ |
| 3841 | static inline int ib_post_send(struct ib_qp *qp, |
Bart Van Assche | d34ac5c | 2018-07-18 09:25:32 -0700 | [diff] [blame] | 3842 | const struct ib_send_wr *send_wr, |
| 3843 | const struct ib_send_wr **bad_send_wr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3844 | { |
Bart Van Assche | d34ac5c | 2018-07-18 09:25:32 -0700 | [diff] [blame] | 3845 | const struct ib_send_wr *dummy; |
Bart Van Assche | bb039a8 | 2018-07-18 09:25:16 -0700 | [diff] [blame] | 3846 | |
Kamal Heib | 3023a1e | 2018-12-10 21:09:48 +0200 | [diff] [blame] | 3847 | return qp->device->ops.post_send(qp, send_wr, bad_send_wr ? : &dummy); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3848 | } |
| 3849 | |
| 3850 | /** |
| 3851 | * ib_post_recv - Posts a list of work requests to the receive queue of |
| 3852 | * the specified QP. |
| 3853 | * @qp: The QP to post the work request on. |
| 3854 | * @recv_wr: A list of work requests to post on the receive queue. |
| 3855 | * @bad_recv_wr: On an immediate failure, this parameter will reference |
| 3856 | * the work request that failed to be posted on the QP. |
| 3857 | */ |
| 3858 | static inline int ib_post_recv(struct ib_qp *qp, |
Bart Van Assche | d34ac5c | 2018-07-18 09:25:32 -0700 | [diff] [blame] | 3859 | const struct ib_recv_wr *recv_wr, |
| 3860 | const struct ib_recv_wr **bad_recv_wr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3861 | { |
Bart Van Assche | d34ac5c | 2018-07-18 09:25:32 -0700 | [diff] [blame] | 3862 | const struct ib_recv_wr *dummy; |
Bart Van Assche | bb039a8 | 2018-07-18 09:25:16 -0700 | [diff] [blame] | 3863 | |
Kamal Heib | 3023a1e | 2018-12-10 21:09:48 +0200 | [diff] [blame] | 3864 | return qp->device->ops.post_recv(qp, recv_wr, bad_recv_wr ? : &dummy); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3865 | } |
| 3866 | |
Leon Romanovsky | 7e3c66c | 2020-09-07 15:09:17 +0300 | [diff] [blame] | 3867 | struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe, |
| 3868 | int comp_vector, enum ib_poll_context poll_ctx, |
| 3869 | const char *caller); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3870 | static inline struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private, |
| 3871 | int nr_cqe, int comp_vector, |
| 3872 | enum ib_poll_context poll_ctx) |
| 3873 | { |
Leon Romanovsky | 7e3c66c | 2020-09-07 15:09:17 +0300 | [diff] [blame] | 3874 | return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx, |
| 3875 | KBUILD_MODNAME); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3876 | } |
| 3877 | |
Chuck Lever | 20cf4e0 | 2019-07-29 13:22:09 -0400 | [diff] [blame] | 3878 | struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private, |
| 3879 | int nr_cqe, enum ib_poll_context poll_ctx, |
| 3880 | const char *caller); |
| 3881 | |
| 3882 | /** |
| 3883 | * ib_alloc_cq_any: Allocate kernel CQ |
| 3884 | * @dev: The IB device |
| 3885 | * @private: Private data attached to the CQE |
| 3886 | * @nr_cqe: Number of CQEs in the CQ |
| 3887 | * @poll_ctx: Context used for polling the CQ |
| 3888 | */ |
| 3889 | static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev, |
| 3890 | void *private, int nr_cqe, |
| 3891 | enum ib_poll_context poll_ctx) |
| 3892 | { |
| 3893 | return __ib_alloc_cq_any(dev, private, nr_cqe, poll_ctx, |
| 3894 | KBUILD_MODNAME); |
| 3895 | } |
| 3896 | |
Leon Romanovsky | 7e3c66c | 2020-09-07 15:09:17 +0300 | [diff] [blame] | 3897 | void ib_free_cq(struct ib_cq *cq); |
Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 3898 | int ib_process_cq_direct(struct ib_cq *cq, int budget); |
| 3899 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3900 | /** |
| 3901 | * ib_create_cq - Creates a CQ on the specified device. |
| 3902 | * @device: The device on which to create the CQ. |
| 3903 | * @comp_handler: A user-specified callback that is invoked when a |
| 3904 | * completion event occurs on the CQ. |
| 3905 | * @event_handler: A user-specified callback that is invoked when an |
| 3906 | * asynchronous event not associated with a completion occurs on the CQ. |
| 3907 | * @cq_context: Context associated with the CQ returned to the user via |
| 3908 | * the associated completion and event handlers. |
Matan Barak | 8e37210 | 2015-06-11 16:35:21 +0300 | [diff] [blame] | 3909 | * @cq_attr: The attributes the CQ should be created upon. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3910 | * |
| 3911 | * Users can examine the cq structure to determine the actual CQ size. |
| 3912 | */ |
Bharat Potnuri | 7350cdd | 2018-06-15 20:52:33 +0530 | [diff] [blame] | 3913 | struct ib_cq *__ib_create_cq(struct ib_device *device, |
| 3914 | ib_comp_handler comp_handler, |
| 3915 | void (*event_handler)(struct ib_event *, void *), |
| 3916 | void *cq_context, |
| 3917 | const struct ib_cq_init_attr *cq_attr, |
| 3918 | const char *caller); |
| 3919 | #define ib_create_cq(device, cmp_hndlr, evt_hndlr, cq_ctxt, cq_attr) \ |
| 3920 | __ib_create_cq((device), (cmp_hndlr), (evt_hndlr), (cq_ctxt), (cq_attr), KBUILD_MODNAME) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3921 | |
| 3922 | /** |
| 3923 | * ib_resize_cq - Modifies the capacity of the CQ. |
| 3924 | * @cq: The CQ to resize. |
| 3925 | * @cqe: The minimum size of the CQ. |
| 3926 | * |
| 3927 | * Users can examine the cq structure to determine the actual CQ size. |
| 3928 | */ |
| 3929 | int ib_resize_cq(struct ib_cq *cq, int cqe); |
| 3930 | |
| 3931 | /** |
Leon Romanovsky | 4190b4e | 2017-11-13 10:51:19 +0200 | [diff] [blame] | 3932 | * rdma_set_cq_moderation - Modifies moderation params of the CQ |
Eli Cohen | 2dd5716 | 2008-04-16 21:09:33 -0700 | [diff] [blame] | 3933 | * @cq: The CQ to modify. |
| 3934 | * @cq_count: number of CQEs that will trigger an event |
| 3935 | * @cq_period: max period of time in usec before triggering an event |
| 3936 | * |
| 3937 | */ |
Leon Romanovsky | 4190b4e | 2017-11-13 10:51:19 +0200 | [diff] [blame] | 3938 | int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period); |
Eli Cohen | 2dd5716 | 2008-04-16 21:09:33 -0700 | [diff] [blame] | 3939 | |
| 3940 | /** |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3941 | * ib_destroy_cq_user - Destroys the specified CQ. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3942 | * @cq: The CQ to destroy. |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3943 | * @udata: Valid user data or NULL for kernel objects |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3944 | */ |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3945 | int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata); |
| 3946 | |
| 3947 | /** |
| 3948 | * ib_destroy_cq - Destroys the specified kernel CQ. |
| 3949 | * @cq: The CQ to destroy. |
| 3950 | * |
| 3951 | * NOTE: for user cq use ib_destroy_cq_user with valid udata! |
| 3952 | */ |
Leon Romanovsky | 890ac8d | 2019-05-20 09:54:21 +0300 | [diff] [blame] | 3953 | static inline void ib_destroy_cq(struct ib_cq *cq) |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3954 | { |
Leon Romanovsky | 43d781b | 2020-09-07 15:09:18 +0300 | [diff] [blame] | 3955 | int ret = ib_destroy_cq_user(cq, NULL); |
| 3956 | |
| 3957 | WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail"); |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 3958 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3959 | |
| 3960 | /** |
| 3961 | * ib_poll_cq - poll a CQ for completion(s) |
| 3962 | * @cq:the CQ being polled |
| 3963 | * @num_entries:maximum number of completions to return |
| 3964 | * @wc:array of at least @num_entries &struct ib_wc where completions |
| 3965 | * will be returned |
| 3966 | * |
| 3967 | * Poll a CQ for (possibly multiple) completions. If the return value |
| 3968 | * is < 0, an error occurred. If the return value is >= 0, it is the |
| 3969 | * number of completions returned. If the return value is |
| 3970 | * non-negative and < num_entries, then the CQ was emptied. |
| 3971 | */ |
| 3972 | static inline int ib_poll_cq(struct ib_cq *cq, int num_entries, |
| 3973 | struct ib_wc *wc) |
| 3974 | { |
Kamal Heib | 3023a1e | 2018-12-10 21:09:48 +0200 | [diff] [blame] | 3975 | return cq->device->ops.poll_cq(cq, num_entries, wc); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3976 | } |
| 3977 | |
| 3978 | /** |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 3979 | * ib_req_notify_cq - Request completion notification on a CQ. |
| 3980 | * @cq: The CQ to generate an event for. |
Roland Dreier | ed23a72 | 2007-05-06 21:02:48 -0700 | [diff] [blame] | 3981 | * @flags: |
| 3982 | * Must contain exactly one of %IB_CQ_SOLICITED or %IB_CQ_NEXT_COMP |
| 3983 | * to request an event on the next solicited event or next work |
| 3984 | * completion at any type, respectively. %IB_CQ_REPORT_MISSED_EVENTS |
| 3985 | * may also be |ed in to request a hint about missed events, as |
| 3986 | * described below. |
| 3987 | * |
| 3988 | * Return Value: |
| 3989 | * < 0 means an error occurred while requesting notification |
| 3990 | * == 0 means notification was requested successfully, and if |
| 3991 | * IB_CQ_REPORT_MISSED_EVENTS was passed in, then no events |
| 3992 | * were missed and it is safe to wait for another event. In |
| 3993 | * this case is it guaranteed that any work completions added |
| 3994 | * to the CQ since the last CQ poll will trigger a completion |
| 3995 | * notification event. |
| 3996 | * > 0 is only returned if IB_CQ_REPORT_MISSED_EVENTS was passed |
| 3997 | * in. It means that the consumer must poll the CQ again to |
| 3998 | * make sure it is empty to avoid missing an event because of a |
| 3999 | * race between requesting notification and an entry being |
| 4000 | * added to the CQ. This return value means it is possible |
| 4001 | * (but not guaranteed) that a work completion has been added |
| 4002 | * to the CQ since the last poll without triggering a |
| 4003 | * completion notification event. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4004 | */ |
| 4005 | static inline int ib_req_notify_cq(struct ib_cq *cq, |
Roland Dreier | ed23a72 | 2007-05-06 21:02:48 -0700 | [diff] [blame] | 4006 | enum ib_cq_notify_flags flags) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4007 | { |
Kamal Heib | 3023a1e | 2018-12-10 21:09:48 +0200 | [diff] [blame] | 4008 | return cq->device->ops.req_notify_cq(cq, flags); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4009 | } |
| 4010 | |
Yamin Friedman | c7ff819 | 2020-05-27 11:34:53 +0300 | [diff] [blame] | 4011 | struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe, |
| 4012 | int comp_vector_hint, |
| 4013 | enum ib_poll_context poll_ctx); |
| 4014 | |
| 4015 | void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe); |
| 4016 | |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4017 | /* |
| 4018 | * Drivers that don't need a DMA mapping at the RDMA layer, set dma_device to |
| 4019 | * NULL. This causes the ib_dma* helpers to just stash the kernel virtual |
| 4020 | * address into the dma address. |
| 4021 | */ |
| 4022 | static inline bool ib_uses_virt_dma(struct ib_device *dev) |
| 4023 | { |
| 4024 | return IS_ENABLED(CONFIG_INFINIBAND_VIRT_DMA) && !dev->dma_device; |
| 4025 | } |
| 4026 | |
Logan Gunthorpe | 495758b | 2022-07-08 10:51:02 -0600 | [diff] [blame] | 4027 | /* |
| 4028 | * Check if a IB device's underlying DMA mapping supports P2PDMA transfers. |
| 4029 | */ |
| 4030 | static inline bool ib_dma_pci_p2p_dma_supported(struct ib_device *dev) |
| 4031 | { |
| 4032 | if (ib_uses_virt_dma(dev)) |
| 4033 | return false; |
| 4034 | |
| 4035 | return dma_pci_p2pdma_supported(dev->dma_device); |
| 4036 | } |
| 4037 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4038 | /** |
Jason Gunthorpe | 8d7c7c0 | 2023-04-14 10:58:29 -0300 | [diff] [blame] | 4039 | * ib_virt_dma_to_ptr - Convert a dma_addr to a kernel pointer |
| 4040 | * @dma_addr: The DMA address |
| 4041 | * |
| 4042 | * Used by ib_uses_virt_dma() devices to get back to the kernel pointer after |
| 4043 | * going through the dma_addr marshalling. |
| 4044 | */ |
| 4045 | static inline void *ib_virt_dma_to_ptr(u64 dma_addr) |
| 4046 | { |
| 4047 | /* virt_dma mode maps the kvs's directly into the dma addr */ |
| 4048 | return (void *)(uintptr_t)dma_addr; |
| 4049 | } |
| 4050 | |
| 4051 | /** |
| 4052 | * ib_virt_dma_to_page - Convert a dma_addr to a struct page |
| 4053 | * @dma_addr: The DMA address |
| 4054 | * |
| 4055 | * Used by ib_uses_virt_dma() device to get back to the struct page after going |
| 4056 | * through the dma_addr marshalling. |
| 4057 | */ |
| 4058 | static inline struct page *ib_virt_dma_to_page(u64 dma_addr) |
| 4059 | { |
| 4060 | return virt_to_page(ib_virt_dma_to_ptr(dma_addr)); |
| 4061 | } |
| 4062 | |
| 4063 | /** |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4064 | * ib_dma_mapping_error - check a DMA addr for error |
| 4065 | * @dev: The device for which the dma_addr was created |
| 4066 | * @dma_addr: The DMA address to check |
| 4067 | */ |
| 4068 | static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr) |
| 4069 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4070 | if (ib_uses_virt_dma(dev)) |
| 4071 | return 0; |
Bart Van Assche | 0957c29 | 2017-03-07 22:56:53 +0000 | [diff] [blame] | 4072 | return dma_mapping_error(dev->dma_device, dma_addr); |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4073 | } |
| 4074 | |
| 4075 | /** |
| 4076 | * ib_dma_map_single - Map a kernel virtual address to DMA address |
| 4077 | * @dev: The device for which the dma_addr is to be created |
| 4078 | * @cpu_addr: The kernel virtual address |
| 4079 | * @size: The size of the region in bytes |
| 4080 | * @direction: The direction of the DMA |
| 4081 | */ |
| 4082 | static inline u64 ib_dma_map_single(struct ib_device *dev, |
| 4083 | void *cpu_addr, size_t size, |
| 4084 | enum dma_data_direction direction) |
| 4085 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4086 | if (ib_uses_virt_dma(dev)) |
| 4087 | return (uintptr_t)cpu_addr; |
Bart Van Assche | 0957c29 | 2017-03-07 22:56:53 +0000 | [diff] [blame] | 4088 | return dma_map_single(dev->dma_device, cpu_addr, size, direction); |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4089 | } |
| 4090 | |
| 4091 | /** |
| 4092 | * ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single() |
| 4093 | * @dev: The device for which the DMA address was created |
| 4094 | * @addr: The DMA address |
| 4095 | * @size: The size of the region in bytes |
| 4096 | * @direction: The direction of the DMA |
| 4097 | */ |
| 4098 | static inline void ib_dma_unmap_single(struct ib_device *dev, |
| 4099 | u64 addr, size_t size, |
| 4100 | enum dma_data_direction direction) |
| 4101 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4102 | if (!ib_uses_virt_dma(dev)) |
| 4103 | dma_unmap_single(dev->dma_device, addr, size, direction); |
Arthur Kepner | cb9fbc5 | 2008-04-29 01:00:34 -0700 | [diff] [blame] | 4104 | } |
| 4105 | |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4106 | /** |
| 4107 | * ib_dma_map_page - Map a physical page to DMA address |
| 4108 | * @dev: The device for which the dma_addr is to be created |
| 4109 | * @page: The page to be mapped |
| 4110 | * @offset: The offset within the page |
| 4111 | * @size: The size of the region in bytes |
| 4112 | * @direction: The direction of the DMA |
| 4113 | */ |
| 4114 | static inline u64 ib_dma_map_page(struct ib_device *dev, |
| 4115 | struct page *page, |
| 4116 | unsigned long offset, |
| 4117 | size_t size, |
| 4118 | enum dma_data_direction direction) |
| 4119 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4120 | if (ib_uses_virt_dma(dev)) |
| 4121 | return (uintptr_t)(page_address(page) + offset); |
Bart Van Assche | 0957c29 | 2017-03-07 22:56:53 +0000 | [diff] [blame] | 4122 | return dma_map_page(dev->dma_device, page, offset, size, direction); |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4123 | } |
| 4124 | |
| 4125 | /** |
| 4126 | * ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page() |
| 4127 | * @dev: The device for which the DMA address was created |
| 4128 | * @addr: The DMA address |
| 4129 | * @size: The size of the region in bytes |
| 4130 | * @direction: The direction of the DMA |
| 4131 | */ |
| 4132 | static inline void ib_dma_unmap_page(struct ib_device *dev, |
| 4133 | u64 addr, size_t size, |
| 4134 | enum dma_data_direction direction) |
| 4135 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4136 | if (!ib_uses_virt_dma(dev)) |
| 4137 | dma_unmap_page(dev->dma_device, addr, size, direction); |
| 4138 | } |
| 4139 | |
| 4140 | int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents); |
| 4141 | static inline int ib_dma_map_sg_attrs(struct ib_device *dev, |
| 4142 | struct scatterlist *sg, int nents, |
| 4143 | enum dma_data_direction direction, |
| 4144 | unsigned long dma_attrs) |
| 4145 | { |
| 4146 | if (ib_uses_virt_dma(dev)) |
| 4147 | return ib_dma_virt_map_sg(dev, sg, nents); |
| 4148 | return dma_map_sg_attrs(dev->dma_device, sg, nents, direction, |
| 4149 | dma_attrs); |
| 4150 | } |
| 4151 | |
| 4152 | static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev, |
| 4153 | struct scatterlist *sg, int nents, |
| 4154 | enum dma_data_direction direction, |
| 4155 | unsigned long dma_attrs) |
| 4156 | { |
| 4157 | if (!ib_uses_virt_dma(dev)) |
| 4158 | dma_unmap_sg_attrs(dev->dma_device, sg, nents, direction, |
| 4159 | dma_attrs); |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4160 | } |
| 4161 | |
| 4162 | /** |
Maor Gottlieb | 79fbd3e | 2021-08-24 17:25:31 +0300 | [diff] [blame] | 4163 | * ib_dma_map_sgtable_attrs - Map a scatter/gather table to DMA addresses |
| 4164 | * @dev: The device for which the DMA addresses are to be created |
| 4165 | * @sg: The sg_table object describing the buffer |
| 4166 | * @direction: The direction of the DMA |
| 4167 | * @attrs: Optional DMA attributes for the map operation |
| 4168 | */ |
| 4169 | static inline int ib_dma_map_sgtable_attrs(struct ib_device *dev, |
| 4170 | struct sg_table *sgt, |
| 4171 | enum dma_data_direction direction, |
| 4172 | unsigned long dma_attrs) |
| 4173 | { |
Logan Gunthorpe | ac0fffa | 2021-10-13 10:59:42 -0600 | [diff] [blame] | 4174 | int nents; |
| 4175 | |
Maor Gottlieb | 79fbd3e | 2021-08-24 17:25:31 +0300 | [diff] [blame] | 4176 | if (ib_uses_virt_dma(dev)) { |
Logan Gunthorpe | ac0fffa | 2021-10-13 10:59:42 -0600 | [diff] [blame] | 4177 | nents = ib_dma_virt_map_sg(dev, sgt->sgl, sgt->orig_nents); |
| 4178 | if (!nents) |
| 4179 | return -EIO; |
| 4180 | sgt->nents = nents; |
Maor Gottlieb | 79fbd3e | 2021-08-24 17:25:31 +0300 | [diff] [blame] | 4181 | return 0; |
| 4182 | } |
| 4183 | return dma_map_sgtable(dev->dma_device, sgt, direction, dma_attrs); |
| 4184 | } |
| 4185 | |
| 4186 | static inline void ib_dma_unmap_sgtable_attrs(struct ib_device *dev, |
| 4187 | struct sg_table *sgt, |
| 4188 | enum dma_data_direction direction, |
| 4189 | unsigned long dma_attrs) |
| 4190 | { |
| 4191 | if (!ib_uses_virt_dma(dev)) |
| 4192 | dma_unmap_sgtable(dev->dma_device, sgt, direction, dma_attrs); |
| 4193 | } |
| 4194 | |
| 4195 | /** |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4196 | * ib_dma_map_sg - Map a scatter/gather list to DMA addresses |
| 4197 | * @dev: The device for which the DMA addresses are to be created |
| 4198 | * @sg: The array of scatter/gather entries |
| 4199 | * @nents: The number of scatter/gather entries |
| 4200 | * @direction: The direction of the DMA |
| 4201 | */ |
| 4202 | static inline int ib_dma_map_sg(struct ib_device *dev, |
| 4203 | struct scatterlist *sg, int nents, |
| 4204 | enum dma_data_direction direction) |
| 4205 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4206 | return ib_dma_map_sg_attrs(dev, sg, nents, direction, 0); |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4207 | } |
| 4208 | |
| 4209 | /** |
| 4210 | * ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses |
| 4211 | * @dev: The device for which the DMA addresses were created |
| 4212 | * @sg: The array of scatter/gather entries |
| 4213 | * @nents: The number of scatter/gather entries |
| 4214 | * @direction: The direction of the DMA |
| 4215 | */ |
| 4216 | static inline void ib_dma_unmap_sg(struct ib_device *dev, |
| 4217 | struct scatterlist *sg, int nents, |
| 4218 | enum dma_data_direction direction) |
| 4219 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4220 | ib_dma_unmap_sg_attrs(dev, sg, nents, direction, 0); |
Arthur Kepner | cb9fbc5 | 2008-04-29 01:00:34 -0700 | [diff] [blame] | 4221 | } |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4222 | |
| 4223 | /** |
Bart Van Assche | 0b5cb33 | 2019-01-22 10:25:20 -0800 | [diff] [blame] | 4224 | * ib_dma_max_seg_size - Return the size limit of a single DMA transfer |
| 4225 | * @dev: The device to query |
| 4226 | * |
| 4227 | * The returned value represents a size in bytes. |
| 4228 | */ |
| 4229 | static inline unsigned int ib_dma_max_seg_size(struct ib_device *dev) |
| 4230 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4231 | if (ib_uses_virt_dma(dev)) |
| 4232 | return UINT_MAX; |
Bart Van Assche | ecdfdfd | 2019-10-25 15:58:27 -0700 | [diff] [blame] | 4233 | return dma_get_max_seg_size(dev->dma_device); |
Bart Van Assche | 0b5cb33 | 2019-01-22 10:25:20 -0800 | [diff] [blame] | 4234 | } |
| 4235 | |
| 4236 | /** |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4237 | * ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU |
| 4238 | * @dev: The device for which the DMA address was created |
| 4239 | * @addr: The DMA address |
| 4240 | * @size: The size of the region in bytes |
| 4241 | * @dir: The direction of the DMA |
| 4242 | */ |
| 4243 | static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev, |
| 4244 | u64 addr, |
| 4245 | size_t size, |
| 4246 | enum dma_data_direction dir) |
| 4247 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4248 | if (!ib_uses_virt_dma(dev)) |
| 4249 | dma_sync_single_for_cpu(dev->dma_device, addr, size, dir); |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4250 | } |
| 4251 | |
| 4252 | /** |
| 4253 | * ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device |
| 4254 | * @dev: The device for which the DMA address was created |
| 4255 | * @addr: The DMA address |
| 4256 | * @size: The size of the region in bytes |
| 4257 | * @dir: The direction of the DMA |
| 4258 | */ |
| 4259 | static inline void ib_dma_sync_single_for_device(struct ib_device *dev, |
| 4260 | u64 addr, |
| 4261 | size_t size, |
| 4262 | enum dma_data_direction dir) |
| 4263 | { |
Christoph Hellwig | 5a7a9e0 | 2020-11-06 19:19:38 +0100 | [diff] [blame] | 4264 | if (!ib_uses_virt_dma(dev)) |
| 4265 | dma_sync_single_for_device(dev->dma_device, addr, size, dir); |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4266 | } |
| 4267 | |
Moni Shoua | 33006bd | 2020-01-15 14:43:32 +0200 | [diff] [blame] | 4268 | /* ib_reg_user_mr - register a memory region for virtual addresses from kernel |
| 4269 | * space. This function should be called when 'current' is the owning MM. |
| 4270 | */ |
| 4271 | struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, |
| 4272 | u64 virt_addr, int mr_access_flags); |
| 4273 | |
Moni Shoua | 87d8069f | 2020-01-15 14:43:33 +0200 | [diff] [blame] | 4274 | /* ib_advise_mr - give an advice about an address range in a memory region */ |
| 4275 | int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, |
| 4276 | u32 flags, struct ib_sge *sg_list, u32 num_sge); |
Ralph Campbell | 9b51309 | 2006-12-12 14:27:41 -0800 | [diff] [blame] | 4277 | /** |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 4278 | * ib_dereg_mr_user - Deregisters a memory region and removes it from the |
| 4279 | * HCA translation table. |
| 4280 | * @mr: The memory region to deregister. |
| 4281 | * @udata: Valid user data or NULL for kernel object |
| 4282 | * |
| 4283 | * This function can fail, if the memory region has memory windows bound to it. |
| 4284 | */ |
| 4285 | int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata); |
| 4286 | |
| 4287 | /** |
| 4288 | * ib_dereg_mr - Deregisters a kernel memory region and removes it from the |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4289 | * HCA translation table. |
| 4290 | * @mr: The memory region to deregister. |
Shani Michaeli | 7083e42 | 2013-02-06 16:19:12 +0000 | [diff] [blame] | 4291 | * |
| 4292 | * This function can fail, if the memory region has memory windows bound to it. |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 4293 | * |
| 4294 | * NOTE: for user mr use ib_dereg_mr_user with valid udata! |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4295 | */ |
Shamir Rabinovitch | c4367a2 | 2019-03-31 19:10:05 +0300 | [diff] [blame] | 4296 | static inline int ib_dereg_mr(struct ib_mr *mr) |
| 4297 | { |
| 4298 | return ib_dereg_mr_user(mr, NULL); |
| 4299 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4300 | |
Gal Pressman | b64b74b1 | 2020-07-06 15:03:42 +0300 | [diff] [blame] | 4301 | struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, |
| 4302 | u32 max_num_sg); |
Steve Wise | 00f7ec3 | 2008-07-14 23:48:45 -0700 | [diff] [blame] | 4303 | |
Israel Rukshin | 26bc7ea | 2019-06-11 18:52:39 +0300 | [diff] [blame] | 4304 | struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd, |
| 4305 | u32 max_num_data_sg, |
| 4306 | u32 max_num_meta_sg); |
| 4307 | |
Steve Wise | 00f7ec3 | 2008-07-14 23:48:45 -0700 | [diff] [blame] | 4308 | /** |
Steve Wise | 00f7ec3 | 2008-07-14 23:48:45 -0700 | [diff] [blame] | 4309 | * ib_update_fast_reg_key - updates the key portion of the fast_reg MR |
| 4310 | * R_Key and L_Key. |
| 4311 | * @mr - struct ib_mr pointer to be updated. |
| 4312 | * @newkey - new key to be used. |
| 4313 | */ |
| 4314 | static inline void ib_update_fast_reg_key(struct ib_mr *mr, u8 newkey) |
| 4315 | { |
| 4316 | mr->lkey = (mr->lkey & 0xffffff00) | newkey; |
| 4317 | mr->rkey = (mr->rkey & 0xffffff00) | newkey; |
| 4318 | } |
| 4319 | |
| 4320 | /** |
Shani Michaeli | 7083e42 | 2013-02-06 16:19:12 +0000 | [diff] [blame] | 4321 | * ib_inc_rkey - increments the key portion of the given rkey. Can be used |
| 4322 | * for calculating a new rkey for type 2 memory windows. |
| 4323 | * @rkey - the rkey to increment. |
| 4324 | */ |
| 4325 | static inline u32 ib_inc_rkey(u32 rkey) |
| 4326 | { |
| 4327 | const u32 mask = 0x000000ff; |
| 4328 | return ((rkey + 1) & mask) | (rkey & ~mask); |
| 4329 | } |
| 4330 | |
| 4331 | /** |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4332 | * ib_attach_mcast - Attaches the specified QP to a multicast group. |
| 4333 | * @qp: QP to attach to the multicast group. The QP must be type |
| 4334 | * IB_QPT_UD. |
| 4335 | * @gid: Multicast group GID. |
| 4336 | * @lid: Multicast group LID in host byte order. |
| 4337 | * |
| 4338 | * In order to send and receive multicast packets, subnet |
| 4339 | * administration must have created the multicast group and configured |
| 4340 | * the fabric appropriately. The port associated with the specified |
| 4341 | * QP must also be a member of the multicast group. |
| 4342 | */ |
| 4343 | int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); |
| 4344 | |
| 4345 | /** |
| 4346 | * ib_detach_mcast - Detaches the specified QP from a multicast group. |
| 4347 | * @qp: QP to detach from the multicast group. |
| 4348 | * @gid: Multicast group GID. |
| 4349 | * @lid: Multicast group LID in host byte order. |
| 4350 | */ |
| 4351 | int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid); |
| 4352 | |
Maor Gottlieb | b73efcb | 2020-07-06 15:27:15 +0300 | [diff] [blame] | 4353 | struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device, |
| 4354 | struct inode *inode, struct ib_udata *udata); |
| 4355 | int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata); |
Sean Hefty | 59991f9 | 2011-05-23 17:52:46 -0700 | [diff] [blame] | 4356 | |
Jason Gunthorpe | adac4cb | 2020-11-30 09:58:36 +0200 | [diff] [blame] | 4357 | static inline int ib_check_mr_access(struct ib_device *ib_dev, |
| 4358 | unsigned int flags) |
Eli Cohen | 1c636f8 | 2013-10-31 15:26:32 +0200 | [diff] [blame] | 4359 | { |
Li Zhijian | 208e3a1 | 2022-12-06 21:01:53 +0800 | [diff] [blame] | 4360 | u64 device_cap = ib_dev->attrs.device_cap_flags; |
| 4361 | |
Eli Cohen | 1c636f8 | 2013-10-31 15:26:32 +0200 | [diff] [blame] | 4362 | /* |
| 4363 | * Local write permission is required if remote write or |
| 4364 | * remote atomic permission is also requested. |
| 4365 | */ |
| 4366 | if (flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) && |
| 4367 | !(flags & IB_ACCESS_LOCAL_WRITE)) |
| 4368 | return -EINVAL; |
| 4369 | |
Michael Guralnik | ca95c14 | 2020-01-08 20:05:35 +0200 | [diff] [blame] | 4370 | if (flags & ~IB_ACCESS_SUPPORTED) |
| 4371 | return -EINVAL; |
| 4372 | |
Jason Gunthorpe | adac4cb | 2020-11-30 09:58:36 +0200 | [diff] [blame] | 4373 | if (flags & IB_ACCESS_ON_DEMAND && |
Jason Gunthorpe | e945c65 | 2022-04-04 12:26:42 -0300 | [diff] [blame] | 4374 | !(ib_dev->attrs.kernel_cap_flags & IBK_ON_DEMAND_PAGING)) |
Li Zhijian | 53c2d5b | 2022-10-01 10:00:45 +0800 | [diff] [blame] | 4375 | return -EOPNOTSUPP; |
Li Zhijian | 208e3a1 | 2022-12-06 21:01:53 +0800 | [diff] [blame] | 4376 | |
| 4377 | if ((flags & IB_ACCESS_FLUSH_GLOBAL && |
| 4378 | !(device_cap & IB_DEVICE_FLUSH_GLOBAL)) || |
| 4379 | (flags & IB_ACCESS_FLUSH_PERSISTENT && |
| 4380 | !(device_cap & IB_DEVICE_FLUSH_PERSISTENT))) |
| 4381 | return -EOPNOTSUPP; |
| 4382 | |
Eli Cohen | 1c636f8 | 2013-10-31 15:26:32 +0200 | [diff] [blame] | 4383 | return 0; |
| 4384 | } |
| 4385 | |
Jack Morgenstein | 08bb558 | 2018-05-23 15:30:30 +0300 | [diff] [blame] | 4386 | static inline bool ib_access_writable(int access_flags) |
| 4387 | { |
| 4388 | /* |
| 4389 | * We have writable memory backing the MR if any of the following |
| 4390 | * access flags are set. "Local write" and "remote write" obviously |
| 4391 | * require write access. "Remote atomic" can do things like fetch and |
| 4392 | * add, which will modify memory, and "MW bind" can change permissions |
| 4393 | * by binding a window. |
| 4394 | */ |
| 4395 | return access_flags & |
| 4396 | (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | |
| 4397 | IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND); |
| 4398 | } |
| 4399 | |
Sagi Grimberg | 1b01d33 | 2014-02-23 14:19:05 +0200 | [diff] [blame] | 4400 | /** |
| 4401 | * ib_check_mr_status: lightweight check of MR status. |
| 4402 | * This routine may provide status checks on a selected |
| 4403 | * ib_mr. first use is for signature status check. |
| 4404 | * |
| 4405 | * @mr: A memory region. |
| 4406 | * @check_mask: Bitmask of which checks to perform from |
| 4407 | * ib_mr_status_check enumeration. |
| 4408 | * @mr_status: The container of relevant status checks. |
| 4409 | * failed checks will be indicated in the status bitmask |
| 4410 | * and the relevant info shall be in the error item. |
| 4411 | */ |
| 4412 | int ib_check_mr_status(struct ib_mr *mr, u32 check_mask, |
| 4413 | struct ib_mr_status *mr_status); |
| 4414 | |
Jason Gunthorpe | d79af72 | 2019-01-10 14:02:24 -0700 | [diff] [blame] | 4415 | /** |
| 4416 | * ib_device_try_get: Hold a registration lock |
| 4417 | * device: The device to lock |
| 4418 | * |
| 4419 | * A device under an active registration lock cannot become unregistered. It |
| 4420 | * is only possible to obtain a registration lock on a device that is fully |
| 4421 | * registered, otherwise this function returns false. |
| 4422 | * |
| 4423 | * The registration lock is only necessary for actions which require the |
| 4424 | * device to still be registered. Uses that only require the device pointer to |
| 4425 | * be valid should use get_device(&ibdev->dev) to hold the memory. |
| 4426 | * |
| 4427 | */ |
| 4428 | static inline bool ib_device_try_get(struct ib_device *dev) |
| 4429 | { |
| 4430 | return refcount_inc_not_zero(&dev->refcount); |
| 4431 | } |
| 4432 | |
| 4433 | void ib_device_put(struct ib_device *device); |
Jason Gunthorpe | 324e227 | 2019-02-12 21:12:51 -0700 | [diff] [blame] | 4434 | struct ib_device *ib_device_get_by_netdev(struct net_device *ndev, |
| 4435 | enum rdma_driver_id driver_id); |
| 4436 | struct ib_device *ib_device_get_by_name(const char *name, |
| 4437 | enum rdma_driver_id driver_id); |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 4438 | struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, u32 port, |
Yotam Kenneth | 9268f72 | 2015-07-30 17:50:15 +0300 | [diff] [blame] | 4439 | u16 pkey, const union ib_gid *gid, |
| 4440 | const struct sockaddr *addr); |
Jason Gunthorpe | c2261dd | 2019-02-12 21:12:50 -0700 | [diff] [blame] | 4441 | int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev, |
| 4442 | unsigned int port); |
Yishai Hadas | 5fd251c | 2016-05-23 15:20:48 +0300 | [diff] [blame] | 4443 | struct ib_wq *ib_create_wq(struct ib_pd *pd, |
| 4444 | struct ib_wq_init_attr *init_attr); |
Leon Romanovsky | add5353 | 2020-09-07 15:09:20 +0300 | [diff] [blame] | 4445 | int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata); |
Yotam Kenneth | 9268f72 | 2015-07-30 17:50:15 +0300 | [diff] [blame] | 4446 | |
Christoph Hellwig | ff2ba99 | 2016-05-03 18:01:04 +0200 | [diff] [blame] | 4447 | int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents, |
Bart Van Assche | 9aa8b32 | 2016-05-12 10:49:15 -0700 | [diff] [blame] | 4448 | unsigned int *sg_offset, unsigned int page_size); |
Max Gurtovoy | 2cdfcdd | 2019-06-11 18:52:40 +0300 | [diff] [blame] | 4449 | int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg, |
| 4450 | int data_sg_nents, unsigned int *data_sg_offset, |
| 4451 | struct scatterlist *meta_sg, int meta_sg_nents, |
| 4452 | unsigned int *meta_sg_offset, unsigned int page_size); |
Sagi Grimberg | 4c67e2b | 2015-10-13 19:11:24 +0300 | [diff] [blame] | 4453 | |
| 4454 | static inline int |
Christoph Hellwig | ff2ba99 | 2016-05-03 18:01:04 +0200 | [diff] [blame] | 4455 | ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents, |
Bart Van Assche | 9aa8b32 | 2016-05-12 10:49:15 -0700 | [diff] [blame] | 4456 | unsigned int *sg_offset, unsigned int page_size) |
Sagi Grimberg | 4c67e2b | 2015-10-13 19:11:24 +0300 | [diff] [blame] | 4457 | { |
| 4458 | int n; |
| 4459 | |
Christoph Hellwig | ff2ba99 | 2016-05-03 18:01:04 +0200 | [diff] [blame] | 4460 | n = ib_map_mr_sg(mr, sg, sg_nents, sg_offset, page_size); |
Sagi Grimberg | 4c67e2b | 2015-10-13 19:11:24 +0300 | [diff] [blame] | 4461 | mr->iova = 0; |
| 4462 | |
| 4463 | return n; |
| 4464 | } |
| 4465 | |
Christoph Hellwig | ff2ba99 | 2016-05-03 18:01:04 +0200 | [diff] [blame] | 4466 | int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents, |
Bart Van Assche | 9aa8b32 | 2016-05-12 10:49:15 -0700 | [diff] [blame] | 4467 | unsigned int *sg_offset, int (*set_page)(struct ib_mr *, u64)); |
Sagi Grimberg | 4c67e2b | 2015-10-13 19:11:24 +0300 | [diff] [blame] | 4468 | |
Steve Wise | 765d677 | 2016-02-17 08:15:41 -0800 | [diff] [blame] | 4469 | void ib_drain_rq(struct ib_qp *qp); |
| 4470 | void ib_drain_sq(struct ib_qp *qp); |
| 4471 | void ib_drain_qp(struct ib_qp *qp); |
Moni Shoua | 850d8fd | 2016-11-10 11:30:56 +0200 | [diff] [blame] | 4472 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 4473 | int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, |
| 4474 | u8 *width); |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4475 | |
| 4476 | static inline u8 *rdma_ah_retrieve_dmac(struct rdma_ah_attr *attr) |
| 4477 | { |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4478 | if (attr->type == RDMA_AH_ATTR_TYPE_ROCE) |
| 4479 | return attr->roce.dmac; |
| 4480 | return NULL; |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4481 | } |
| 4482 | |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 4483 | static inline void rdma_ah_set_dlid(struct rdma_ah_attr *attr, u32 dlid) |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4484 | { |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4485 | if (attr->type == RDMA_AH_ATTR_TYPE_IB) |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 4486 | attr->ib.dlid = (u16)dlid; |
| 4487 | else if (attr->type == RDMA_AH_ATTR_TYPE_OPA) |
| 4488 | attr->opa.dlid = dlid; |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4489 | } |
| 4490 | |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 4491 | static inline u32 rdma_ah_get_dlid(const struct rdma_ah_attr *attr) |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4492 | { |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4493 | if (attr->type == RDMA_AH_ATTR_TYPE_IB) |
| 4494 | return attr->ib.dlid; |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 4495 | else if (attr->type == RDMA_AH_ATTR_TYPE_OPA) |
| 4496 | return attr->opa.dlid; |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4497 | return 0; |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4498 | } |
| 4499 | |
| 4500 | static inline void rdma_ah_set_sl(struct rdma_ah_attr *attr, u8 sl) |
| 4501 | { |
| 4502 | attr->sl = sl; |
| 4503 | } |
| 4504 | |
| 4505 | static inline u8 rdma_ah_get_sl(const struct rdma_ah_attr *attr) |
| 4506 | { |
| 4507 | return attr->sl; |
| 4508 | } |
| 4509 | |
| 4510 | static inline void rdma_ah_set_path_bits(struct rdma_ah_attr *attr, |
| 4511 | u8 src_path_bits) |
| 4512 | { |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4513 | if (attr->type == RDMA_AH_ATTR_TYPE_IB) |
| 4514 | attr->ib.src_path_bits = src_path_bits; |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 4515 | else if (attr->type == RDMA_AH_ATTR_TYPE_OPA) |
| 4516 | attr->opa.src_path_bits = src_path_bits; |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4517 | } |
| 4518 | |
| 4519 | static inline u8 rdma_ah_get_path_bits(const struct rdma_ah_attr *attr) |
| 4520 | { |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4521 | if (attr->type == RDMA_AH_ATTR_TYPE_IB) |
| 4522 | return attr->ib.src_path_bits; |
Dasaratharaman Chandramouli | 64b4646 | 2017-04-29 14:41:30 -0400 | [diff] [blame] | 4523 | else if (attr->type == RDMA_AH_ATTR_TYPE_OPA) |
| 4524 | return attr->opa.src_path_bits; |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4525 | return 0; |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4526 | } |
| 4527 | |
Don Hiatt | d98bb7f | 2017-08-04 13:54:16 -0700 | [diff] [blame] | 4528 | static inline void rdma_ah_set_make_grd(struct rdma_ah_attr *attr, |
| 4529 | bool make_grd) |
| 4530 | { |
| 4531 | if (attr->type == RDMA_AH_ATTR_TYPE_OPA) |
| 4532 | attr->opa.make_grd = make_grd; |
| 4533 | } |
| 4534 | |
| 4535 | static inline bool rdma_ah_get_make_grd(const struct rdma_ah_attr *attr) |
| 4536 | { |
| 4537 | if (attr->type == RDMA_AH_ATTR_TYPE_OPA) |
| 4538 | return attr->opa.make_grd; |
| 4539 | return false; |
| 4540 | } |
| 4541 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 4542 | static inline void rdma_ah_set_port_num(struct rdma_ah_attr *attr, u32 port_num) |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4543 | { |
| 4544 | attr->port_num = port_num; |
| 4545 | } |
| 4546 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 4547 | static inline u32 rdma_ah_get_port_num(const struct rdma_ah_attr *attr) |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4548 | { |
| 4549 | return attr->port_num; |
| 4550 | } |
| 4551 | |
| 4552 | static inline void rdma_ah_set_static_rate(struct rdma_ah_attr *attr, |
| 4553 | u8 static_rate) |
| 4554 | { |
| 4555 | attr->static_rate = static_rate; |
| 4556 | } |
| 4557 | |
| 4558 | static inline u8 rdma_ah_get_static_rate(const struct rdma_ah_attr *attr) |
| 4559 | { |
| 4560 | return attr->static_rate; |
| 4561 | } |
| 4562 | |
| 4563 | static inline void rdma_ah_set_ah_flags(struct rdma_ah_attr *attr, |
| 4564 | enum ib_ah_flags flag) |
| 4565 | { |
| 4566 | attr->ah_flags = flag; |
| 4567 | } |
| 4568 | |
| 4569 | static inline enum ib_ah_flags |
| 4570 | rdma_ah_get_ah_flags(const struct rdma_ah_attr *attr) |
| 4571 | { |
| 4572 | return attr->ah_flags; |
| 4573 | } |
| 4574 | |
| 4575 | static inline const struct ib_global_route |
| 4576 | *rdma_ah_read_grh(const struct rdma_ah_attr *attr) |
| 4577 | { |
| 4578 | return &attr->grh; |
| 4579 | } |
| 4580 | |
| 4581 | /*To retrieve and modify the grh */ |
| 4582 | static inline struct ib_global_route |
| 4583 | *rdma_ah_retrieve_grh(struct rdma_ah_attr *attr) |
| 4584 | { |
| 4585 | return &attr->grh; |
| 4586 | } |
| 4587 | |
| 4588 | static inline void rdma_ah_set_dgid_raw(struct rdma_ah_attr *attr, void *dgid) |
| 4589 | { |
| 4590 | struct ib_global_route *grh = rdma_ah_retrieve_grh(attr); |
| 4591 | |
| 4592 | memcpy(grh->dgid.raw, dgid, sizeof(grh->dgid)); |
| 4593 | } |
| 4594 | |
| 4595 | static inline void rdma_ah_set_subnet_prefix(struct rdma_ah_attr *attr, |
| 4596 | __be64 prefix) |
| 4597 | { |
| 4598 | struct ib_global_route *grh = rdma_ah_retrieve_grh(attr); |
| 4599 | |
| 4600 | grh->dgid.global.subnet_prefix = prefix; |
| 4601 | } |
| 4602 | |
| 4603 | static inline void rdma_ah_set_interface_id(struct rdma_ah_attr *attr, |
| 4604 | __be64 if_id) |
| 4605 | { |
| 4606 | struct ib_global_route *grh = rdma_ah_retrieve_grh(attr); |
| 4607 | |
| 4608 | grh->dgid.global.interface_id = if_id; |
| 4609 | } |
| 4610 | |
| 4611 | static inline void rdma_ah_set_grh(struct rdma_ah_attr *attr, |
| 4612 | union ib_gid *dgid, u32 flow_label, |
| 4613 | u8 sgid_index, u8 hop_limit, |
| 4614 | u8 traffic_class) |
| 4615 | { |
| 4616 | struct ib_global_route *grh = rdma_ah_retrieve_grh(attr); |
| 4617 | |
| 4618 | attr->ah_flags = IB_AH_GRH; |
| 4619 | if (dgid) |
| 4620 | grh->dgid = *dgid; |
| 4621 | grh->flow_label = flow_label; |
| 4622 | grh->sgid_index = sgid_index; |
| 4623 | grh->hop_limit = hop_limit; |
| 4624 | grh->traffic_class = traffic_class; |
Jason Gunthorpe | 8d9ec9a | 2018-06-13 10:22:03 +0300 | [diff] [blame] | 4625 | grh->sgid_attr = NULL; |
Dasaratharaman Chandramouli | 2224c47 | 2017-04-29 14:41:27 -0400 | [diff] [blame] | 4626 | } |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4627 | |
Jason Gunthorpe | 8d9ec9a | 2018-06-13 10:22:03 +0300 | [diff] [blame] | 4628 | void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr); |
| 4629 | void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid, |
| 4630 | u32 flow_label, u8 hop_limit, u8 traffic_class, |
| 4631 | const struct ib_gid_attr *sgid_attr); |
Jason Gunthorpe | d97099f | 2018-06-13 10:22:05 +0300 | [diff] [blame] | 4632 | void rdma_copy_ah_attr(struct rdma_ah_attr *dest, |
| 4633 | const struct rdma_ah_attr *src); |
| 4634 | void rdma_replace_ah_attr(struct rdma_ah_attr *old, |
| 4635 | const struct rdma_ah_attr *new); |
| 4636 | void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src); |
Jason Gunthorpe | 8d9ec9a | 2018-06-13 10:22:03 +0300 | [diff] [blame] | 4637 | |
Don Hiatt | 87daac6 | 2018-02-01 10:57:03 -0800 | [diff] [blame] | 4638 | /** |
| 4639 | * rdma_ah_find_type - Return address handle type. |
| 4640 | * |
| 4641 | * @dev: Device to be checked |
| 4642 | * @port_num: Port number |
| 4643 | */ |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4644 | static inline enum rdma_ah_attr_type rdma_ah_find_type(struct ib_device *dev, |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 4645 | u32 port_num) |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4646 | { |
Parav Pandit | a6532e7 | 2018-01-12 07:58:42 +0200 | [diff] [blame] | 4647 | if (rdma_protocol_roce(dev, port_num)) |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4648 | return RDMA_AH_ATTR_TYPE_ROCE; |
Don Hiatt | 87daac6 | 2018-02-01 10:57:03 -0800 | [diff] [blame] | 4649 | if (rdma_protocol_ib(dev, port_num)) { |
| 4650 | if (rdma_cap_opa_ah(dev, port_num)) |
| 4651 | return RDMA_AH_ATTR_TYPE_OPA; |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4652 | return RDMA_AH_ATTR_TYPE_IB; |
Don Hiatt | 87daac6 | 2018-02-01 10:57:03 -0800 | [diff] [blame] | 4653 | } |
| 4654 | |
| 4655 | return RDMA_AH_ATTR_TYPE_UNDEFINED; |
Dasaratharaman Chandramouli | 44c5848 | 2017-04-29 14:41:29 -0400 | [diff] [blame] | 4656 | } |
Hiatt, Don | 7db20ec | 2017-06-08 13:37:49 -0400 | [diff] [blame] | 4657 | |
Hiatt, Don | 62ede77 | 2017-08-14 14:17:43 -0400 | [diff] [blame] | 4658 | /** |
| 4659 | * ib_lid_cpu16 - Return lid in 16bit CPU encoding. |
Xin Gao | 8937e28 | 2022-07-22 10:18:33 +0800 | [diff] [blame] | 4660 | * In the current implementation the only way to |
Hiatt, Don | 62ede77 | 2017-08-14 14:17:43 -0400 | [diff] [blame] | 4661 | * get the 32bit lid is from other sources for OPA. |
| 4662 | * For IB, lids will always be 16bits so cast the |
| 4663 | * value accordingly. |
| 4664 | * |
| 4665 | * @lid: A 32bit LID |
| 4666 | */ |
| 4667 | static inline u16 ib_lid_cpu16(u32 lid) |
Hiatt, Don | 7db20ec | 2017-06-08 13:37:49 -0400 | [diff] [blame] | 4668 | { |
Hiatt, Don | 62ede77 | 2017-08-14 14:17:43 -0400 | [diff] [blame] | 4669 | WARN_ON_ONCE(lid & 0xFFFF0000); |
| 4670 | return (u16)lid; |
Hiatt, Don | 7db20ec | 2017-06-08 13:37:49 -0400 | [diff] [blame] | 4671 | } |
| 4672 | |
Hiatt, Don | 62ede77 | 2017-08-14 14:17:43 -0400 | [diff] [blame] | 4673 | /** |
| 4674 | * ib_lid_be16 - Return lid in 16bit BE encoding. |
| 4675 | * |
| 4676 | * @lid: A 32bit LID |
| 4677 | */ |
| 4678 | static inline __be16 ib_lid_be16(u32 lid) |
Hiatt, Don | 7db20ec | 2017-06-08 13:37:49 -0400 | [diff] [blame] | 4679 | { |
Hiatt, Don | 62ede77 | 2017-08-14 14:17:43 -0400 | [diff] [blame] | 4680 | WARN_ON_ONCE(lid & 0xFFFF0000); |
| 4681 | return cpu_to_be16((u16)lid); |
Hiatt, Don | 7db20ec | 2017-06-08 13:37:49 -0400 | [diff] [blame] | 4682 | } |
Doug Ledford | 3204383 | 2017-08-10 14:31:29 -0400 | [diff] [blame] | 4683 | |
Sagi Grimberg | c66cd35 | 2017-07-13 11:09:41 +0300 | [diff] [blame] | 4684 | /** |
| 4685 | * ib_get_vector_affinity - Get the affinity mappings of a given completion |
| 4686 | * vector |
| 4687 | * @device: the rdma device |
| 4688 | * @comp_vector: index of completion vector |
| 4689 | * |
| 4690 | * Returns NULL on failure, otherwise a corresponding cpu map of the |
| 4691 | * completion vector (returns all-cpus map if the device driver doesn't |
| 4692 | * implement get_vector_affinity). |
| 4693 | */ |
| 4694 | static inline const struct cpumask * |
| 4695 | ib_get_vector_affinity(struct ib_device *device, int comp_vector) |
| 4696 | { |
| 4697 | if (comp_vector < 0 || comp_vector >= device->num_comp_vectors || |
Kamal Heib | 3023a1e | 2018-12-10 21:09:48 +0200 | [diff] [blame] | 4698 | !device->ops.get_vector_affinity) |
Sagi Grimberg | c66cd35 | 2017-07-13 11:09:41 +0300 | [diff] [blame] | 4699 | return NULL; |
| 4700 | |
Kamal Heib | 3023a1e | 2018-12-10 21:09:48 +0200 | [diff] [blame] | 4701 | return device->ops.get_vector_affinity(device, comp_vector); |
Sagi Grimberg | c66cd35 | 2017-07-13 11:09:41 +0300 | [diff] [blame] | 4702 | |
| 4703 | } |
| 4704 | |
Daniel Jurgens | 32f69e4 | 2018-01-04 17:25:36 +0200 | [diff] [blame] | 4705 | /** |
| 4706 | * rdma_roce_rescan_device - Rescan all of the network devices in the system |
| 4707 | * and add their gids, as needed, to the relevant RoCE devices. |
| 4708 | * |
| 4709 | * @device: the rdma device |
| 4710 | */ |
| 4711 | void rdma_roce_rescan_device(struct ib_device *ibdev); |
| 4712 | |
Jason Gunthorpe | 8313c10 | 2018-11-25 20:51:13 +0200 | [diff] [blame] | 4713 | struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile); |
Yishai Hadas | 7dc08dc | 2018-06-17 12:59:59 +0300 | [diff] [blame] | 4714 | |
Jason Gunthorpe | 15a1b4b | 2018-11-25 20:51:15 +0200 | [diff] [blame] | 4715 | int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs); |
Denis Drozdov | f6a8a19 | 2018-08-14 14:08:51 +0300 | [diff] [blame] | 4716 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 4717 | struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num, |
Denis Drozdov | f6a8a19 | 2018-08-14 14:08:51 +0300 | [diff] [blame] | 4718 | enum rdma_netdev_t type, const char *name, |
| 4719 | unsigned char name_assign_type, |
| 4720 | void (*setup)(struct net_device *)); |
Denis Drozdov | 5d6b0cb | 2018-08-14 14:22:35 +0300 | [diff] [blame] | 4721 | |
Mark Bloch | 1fb7f89 | 2021-03-01 09:04:20 +0200 | [diff] [blame] | 4722 | int rdma_init_netdev(struct ib_device *device, u32 port_num, |
Denis Drozdov | 5d6b0cb | 2018-08-14 14:22:35 +0300 | [diff] [blame] | 4723 | enum rdma_netdev_t type, const char *name, |
| 4724 | unsigned char name_assign_type, |
| 4725 | void (*setup)(struct net_device *), |
| 4726 | struct net_device *netdev); |
| 4727 | |
Parav Pandit | d4122f5 | 2018-10-11 22:31:53 +0300 | [diff] [blame] | 4728 | /** |
Parav Pandit | 5474723 | 2018-12-18 14:15:56 +0200 | [diff] [blame] | 4729 | * rdma_device_to_ibdev - Get ib_device pointer from device pointer |
| 4730 | * |
| 4731 | * @device: device pointer for which ib_device pointer to retrieve |
| 4732 | * |
| 4733 | * rdma_device_to_ibdev() retrieves ib_device pointer from device. |
| 4734 | * |
| 4735 | */ |
| 4736 | static inline struct ib_device *rdma_device_to_ibdev(struct device *device) |
| 4737 | { |
Parav Pandit | cebe556 | 2019-02-26 13:56:11 +0200 | [diff] [blame] | 4738 | struct ib_core_device *coredev = |
| 4739 | container_of(device, struct ib_core_device, dev); |
| 4740 | |
| 4741 | return coredev->owner; |
Parav Pandit | 5474723 | 2018-12-18 14:15:56 +0200 | [diff] [blame] | 4742 | } |
| 4743 | |
| 4744 | /** |
Christoph Hellwig | 8ecfca6 | 2020-11-06 19:19:34 +0100 | [diff] [blame] | 4745 | * ibdev_to_node - return the NUMA node for a given ib_device |
| 4746 | * @dev: device to get the NUMA node for. |
| 4747 | */ |
| 4748 | static inline int ibdev_to_node(struct ib_device *ibdev) |
| 4749 | { |
| 4750 | struct device *parent = ibdev->dev.parent; |
| 4751 | |
| 4752 | if (!parent) |
| 4753 | return NUMA_NO_NODE; |
| 4754 | return dev_to_node(parent); |
| 4755 | } |
| 4756 | |
| 4757 | /** |
Parav Pandit | 5474723 | 2018-12-18 14:15:56 +0200 | [diff] [blame] | 4758 | * rdma_device_to_drv_device - Helper macro to reach back to driver's |
| 4759 | * ib_device holder structure from device pointer. |
| 4760 | * |
| 4761 | * NOTE: New drivers should not make use of this API; This API is only for |
| 4762 | * existing drivers who have exposed sysfs entries using |
Jason Gunthorpe | 915e4af | 2021-06-11 19:00:34 +0300 | [diff] [blame] | 4763 | * ops->device_group. |
Parav Pandit | 5474723 | 2018-12-18 14:15:56 +0200 | [diff] [blame] | 4764 | */ |
| 4765 | #define rdma_device_to_drv_device(dev, drv_dev_struct, ibdev_member) \ |
| 4766 | container_of(rdma_device_to_ibdev(dev), drv_dev_struct, ibdev_member) |
Parav Pandit | 41c6140 | 2019-02-26 14:01:46 +0200 | [diff] [blame] | 4767 | |
| 4768 | bool rdma_dev_access_netns(const struct ib_device *device, |
| 4769 | const struct net *net); |
Mark Zhang | d5665a2 | 2020-05-04 08:19:31 +0300 | [diff] [blame] | 4770 | |
| 4771 | #define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000) |
Weihang Li | 074bf2c | 2020-08-21 17:31:29 +0800 | [diff] [blame] | 4772 | #define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF) |
Mark Zhang | d5665a2 | 2020-05-04 08:19:31 +0300 | [diff] [blame] | 4773 | #define IB_GRH_FLOWLABEL_MASK (0x000FFFFF) |
| 4774 | |
| 4775 | /** |
| 4776 | * rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based |
| 4777 | * on the flow_label |
| 4778 | * |
| 4779 | * This function will convert the 20 bit flow_label input to a valid RoCE v2 |
| 4780 | * UDP src port 14 bit value. All RoCE V2 drivers should use this same |
| 4781 | * convention. |
| 4782 | */ |
| 4783 | static inline u16 rdma_flow_label_to_udp_sport(u32 fl) |
| 4784 | { |
| 4785 | u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000; |
| 4786 | |
| 4787 | fl_low ^= fl_high >> 14; |
| 4788 | return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN); |
| 4789 | } |
| 4790 | |
| 4791 | /** |
| 4792 | * rdma_calc_flow_label - generate a RDMA symmetric flow label value based on |
| 4793 | * local and remote qpn values |
| 4794 | * |
| 4795 | * This function folded the multiplication results of two qpns, 24 bit each, |
| 4796 | * fields, and converts it to a 20 bit results. |
| 4797 | * |
| 4798 | * This function will create symmetric flow_label value based on the local |
| 4799 | * and remote qpn values. this will allow both the requester and responder |
| 4800 | * to calculate the same flow_label for a given connection. |
| 4801 | * |
| 4802 | * This helper function should be used by driver in case the upper layer |
| 4803 | * provide a zero flow_label value. This is to improve entropy of RDMA |
| 4804 | * traffic in the network. |
| 4805 | */ |
| 4806 | static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn) |
| 4807 | { |
| 4808 | u64 v = (u64)lqpn * rqpn; |
| 4809 | |
| 4810 | v ^= v >> 20; |
| 4811 | v ^= v >> 40; |
| 4812 | |
| 4813 | return (u32)(v & IB_GRH_FLOWLABEL_MASK); |
| 4814 | } |
Parav Pandit | 7416790 | 2021-02-03 15:01:32 +0200 | [diff] [blame] | 4815 | |
Zhu Yanjun | 18451db | 2022-01-06 13:03:56 -0500 | [diff] [blame] | 4816 | /** |
| 4817 | * rdma_get_udp_sport - Calculate and set UDP source port based on the flow |
| 4818 | * label. If flow label is not defined in GRH then |
| 4819 | * calculate it based on lqpn/rqpn. |
| 4820 | * |
| 4821 | * @fl: flow label from GRH |
| 4822 | * @lqpn: local qp number |
| 4823 | * @rqpn: remote qp number |
| 4824 | */ |
| 4825 | static inline u16 rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn) |
| 4826 | { |
| 4827 | if (!fl) |
| 4828 | fl = rdma_calc_flow_label(lqpn, rqpn); |
| 4829 | |
| 4830 | return rdma_flow_label_to_udp_sport(fl); |
| 4831 | } |
| 4832 | |
Parav Pandit | 7416790 | 2021-02-03 15:01:32 +0200 | [diff] [blame] | 4833 | const struct ib_port_immutable* |
| 4834 | ib_port_immutable_read(struct ib_device *dev, unsigned int port); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4835 | #endif /* IB_VERBS_H */ |