blob: 67146624eb58b7946665026ee29ce57a84316888 [file] [log] [blame]
Roland Dreier225c7b12007-05-08 18:00:38 -07001/*
2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
Jack Morgenstein51a379d2008-07-25 10:32:52 -07003 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
Roland Dreier225c7b12007-05-08 18:00:38 -07004 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
Jack Morgenstein3ec65b22011-12-13 04:13:05 +000034
Roland Dreier225c7b12007-05-08 18:00:38 -070035#include <linux/mlx4/cmd.h>
Or Gerlitzfe66bb22013-10-15 16:55:21 +020036#include <linux/mlx4/srq.h>
Paul Gortmakeree40fa02011-05-27 16:14:23 -040037#include <linux/export.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090038#include <linux/gfp.h>
Roland Dreier225c7b12007-05-08 18:00:38 -070039
40#include "mlx4.h"
41#include "icm.h"
42
Roland Dreier225c7b12007-05-08 18:00:38 -070043void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type)
44{
45 struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
46 struct mlx4_srq *srq;
47
48 spin_lock(&srq_table->lock);
49
50 srq = radix_tree_lookup(&srq_table->tree, srqn & (dev->caps.num_srqs - 1));
51 if (srq)
52 atomic_inc(&srq->refcount);
53
54 spin_unlock(&srq_table->lock);
55
56 if (!srq) {
57 mlx4_warn(dev, "Async event for bogus SRQ %08x\n", srqn);
58 return;
59 }
60
61 srq->event(srq, event_type);
62
63 if (atomic_dec_and_test(&srq->refcount))
64 complete(&srq->free);
65}
66
67static int mlx4_SW2HW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
68 int srq_num)
69{
Marcel Apfelbaumeb410492012-01-19 09:45:19 +000070 return mlx4_cmd(dev, mailbox->dma, srq_num, 0,
Jack Morgenstein3ec65b22011-12-13 04:13:05 +000071 MLX4_CMD_SW2HW_SRQ, MLX4_CMD_TIME_CLASS_A,
72 MLX4_CMD_WRAPPED);
Roland Dreier225c7b12007-05-08 18:00:38 -070073}
74
75static int mlx4_HW2SW_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
76 int srq_num)
77{
78 return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, srq_num,
79 mailbox ? 0 : 1, MLX4_CMD_HW2SW_SRQ,
Jack Morgensteinf9baff52011-12-13 04:10:51 +000080 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
Roland Dreier225c7b12007-05-08 18:00:38 -070081}
82
83static int mlx4_ARM_SRQ(struct mlx4_dev *dev, int srq_num, int limit_watermark)
84{
85 return mlx4_cmd(dev, limit_watermark, srq_num, 0, MLX4_CMD_ARM_SRQ,
Jack Morgensteinf9baff52011-12-13 04:10:51 +000086 MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED);
Roland Dreier225c7b12007-05-08 18:00:38 -070087}
88
Jack Morgenstein65541cb2007-06-21 13:03:11 +030089static int mlx4_QUERY_SRQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
90 int srq_num)
91{
92 return mlx4_cmd_box(dev, 0, mailbox->dma, srq_num, 0, MLX4_CMD_QUERY_SRQ,
Jack Morgensteinf9baff52011-12-13 04:10:51 +000093 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
Jack Morgenstein65541cb2007-06-21 13:03:11 +030094}
95
Eli Cohenc82e9aa2011-12-13 04:15:24 +000096int __mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn)
Jack Morgenstein3ec65b22011-12-13 04:13:05 +000097{
98 struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
99 int err;
100
101
102 *srqn = mlx4_bitmap_alloc(&srq_table->bitmap);
103 if (*srqn == -1)
104 return -ENOMEM;
105
Jiri Kosina40f22872014-05-11 15:15:12 +0300106 err = mlx4_table_get(dev, &srq_table->table, *srqn, GFP_KERNEL);
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000107 if (err)
108 goto err_out;
109
Jiri Kosina40f22872014-05-11 15:15:12 +0300110 err = mlx4_table_get(dev, &srq_table->cmpt_table, *srqn, GFP_KERNEL);
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000111 if (err)
112 goto err_put;
113 return 0;
114
115err_put:
116 mlx4_table_put(dev, &srq_table->table, *srqn);
117
118err_out:
Jack Morgenstein7c6d74d2013-12-08 16:50:17 +0200119 mlx4_bitmap_free(&srq_table->bitmap, *srqn, MLX4_NO_RR);
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000120 return err;
121}
122
123static int mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn)
124{
125 u64 out_param;
126 int err;
127
128 if (mlx4_is_mfunc(dev)) {
129 err = mlx4_cmd_imm(dev, 0, &out_param, RES_SRQ,
130 RES_OP_RESERVE_AND_MAP,
131 MLX4_CMD_ALLOC_RES,
132 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
133 if (!err)
134 *srqn = get_param_l(&out_param);
135
136 return err;
137 }
138 return __mlx4_srq_alloc_icm(dev, srqn);
139}
140
Eli Cohenc82e9aa2011-12-13 04:15:24 +0000141void __mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn)
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000142{
143 struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
144
145 mlx4_table_put(dev, &srq_table->cmpt_table, srqn);
146 mlx4_table_put(dev, &srq_table->table, srqn);
Jack Morgenstein7c6d74d2013-12-08 16:50:17 +0200147 mlx4_bitmap_free(&srq_table->bitmap, srqn, MLX4_NO_RR);
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000148}
149
150static void mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn)
151{
Jack Morgensteine7dbeba2013-03-07 03:46:54 +0000152 u64 in_param = 0;
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000153
154 if (mlx4_is_mfunc(dev)) {
155 set_param_l(&in_param, srqn);
156 if (mlx4_cmd(dev, in_param, RES_SRQ, RES_OP_RESERVE_AND_MAP,
157 MLX4_CMD_FREE_RES,
158 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED))
159 mlx4_warn(dev, "Failed freeing cq:%d\n", srqn);
160 return;
161 }
162 __mlx4_srq_free_icm(dev, srqn);
163}
164
Sean Hefty18abd5e2011-06-02 10:43:26 -0700165int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd,
166 struct mlx4_mtt *mtt, u64 db_rec, struct mlx4_srq *srq)
Roland Dreier225c7b12007-05-08 18:00:38 -0700167{
168 struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
169 struct mlx4_cmd_mailbox *mailbox;
170 struct mlx4_srq_context *srq_context;
171 u64 mtt_addr;
172 int err;
173
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000174 err = mlx4_srq_alloc_icm(dev, &srq->srqn);
Roland Dreier225c7b12007-05-08 18:00:38 -0700175 if (err)
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000176 return err;
Roland Dreier225c7b12007-05-08 18:00:38 -0700177
178 spin_lock_irq(&srq_table->lock);
179 err = radix_tree_insert(&srq_table->tree, srq->srqn, srq);
180 spin_unlock_irq(&srq_table->lock);
181 if (err)
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000182 goto err_icm;
Roland Dreier225c7b12007-05-08 18:00:38 -0700183
184 mailbox = mlx4_alloc_cmd_mailbox(dev);
185 if (IS_ERR(mailbox)) {
186 err = PTR_ERR(mailbox);
187 goto err_radix;
188 }
189
190 srq_context = mailbox->buf;
Roland Dreier225c7b12007-05-08 18:00:38 -0700191 srq_context->state_logsize_srqn = cpu_to_be32((ilog2(srq->max) << 24) |
192 srq->srqn);
193 srq_context->logstride = srq->wqe_shift - 4;
Sean Hefty18abd5e2011-06-02 10:43:26 -0700194 srq_context->xrcd = cpu_to_be16(xrcd);
195 srq_context->pg_offset_cqn = cpu_to_be32(cqn & 0xffffff);
Roland Dreier225c7b12007-05-08 18:00:38 -0700196 srq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
197
198 mtt_addr = mlx4_mtt_addr(dev, mtt);
199 srq_context->mtt_base_addr_h = mtt_addr >> 32;
200 srq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
201 srq_context->pd = cpu_to_be32(pdn);
202 srq_context->db_rec_addr = cpu_to_be64(db_rec);
203
204 err = mlx4_SW2HW_SRQ(dev, mailbox, srq->srqn);
205 mlx4_free_cmd_mailbox(dev, mailbox);
206 if (err)
207 goto err_radix;
208
209 atomic_set(&srq->refcount, 1);
210 init_completion(&srq->free);
211
212 return 0;
213
214err_radix:
215 spin_lock_irq(&srq_table->lock);
216 radix_tree_delete(&srq_table->tree, srq->srqn);
217 spin_unlock_irq(&srq_table->lock);
218
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000219err_icm:
220 mlx4_srq_free_icm(dev, srq->srqn);
Roland Dreier225c7b12007-05-08 18:00:38 -0700221 return err;
222}
223EXPORT_SYMBOL_GPL(mlx4_srq_alloc);
224
225void mlx4_srq_free(struct mlx4_dev *dev, struct mlx4_srq *srq)
226{
227 struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
228 int err;
229
230 err = mlx4_HW2SW_SRQ(dev, NULL, srq->srqn);
231 if (err)
232 mlx4_warn(dev, "HW2SW_SRQ failed (%d) for SRQN %06x\n", err, srq->srqn);
233
234 spin_lock_irq(&srq_table->lock);
235 radix_tree_delete(&srq_table->tree, srq->srqn);
236 spin_unlock_irq(&srq_table->lock);
237
238 if (atomic_dec_and_test(&srq->refcount))
239 complete(&srq->free);
240 wait_for_completion(&srq->free);
241
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000242 mlx4_srq_free_icm(dev, srq->srqn);
Roland Dreier225c7b12007-05-08 18:00:38 -0700243}
244EXPORT_SYMBOL_GPL(mlx4_srq_free);
245
246int mlx4_srq_arm(struct mlx4_dev *dev, struct mlx4_srq *srq, int limit_watermark)
247{
248 return mlx4_ARM_SRQ(dev, srq->srqn, limit_watermark);
249}
250EXPORT_SYMBOL_GPL(mlx4_srq_arm);
251
Jack Morgenstein65541cb2007-06-21 13:03:11 +0300252int mlx4_srq_query(struct mlx4_dev *dev, struct mlx4_srq *srq, int *limit_watermark)
253{
254 struct mlx4_cmd_mailbox *mailbox;
255 struct mlx4_srq_context *srq_context;
256 int err;
257
258 mailbox = mlx4_alloc_cmd_mailbox(dev);
259 if (IS_ERR(mailbox))
260 return PTR_ERR(mailbox);
261
262 srq_context = mailbox->buf;
263
264 err = mlx4_QUERY_SRQ(dev, mailbox, srq->srqn);
265 if (err)
266 goto err_out;
Roland Dreierd7dc3cc2007-10-09 19:59:06 -0700267 *limit_watermark = be16_to_cpu(srq_context->limit_watermark);
Jack Morgenstein65541cb2007-06-21 13:03:11 +0300268
269err_out:
270 mlx4_free_cmd_mailbox(dev, mailbox);
271 return err;
272}
273EXPORT_SYMBOL_GPL(mlx4_srq_query);
274
Roland Dreier3d73c282007-10-10 15:43:54 -0700275int mlx4_init_srq_table(struct mlx4_dev *dev)
Roland Dreier225c7b12007-05-08 18:00:38 -0700276{
277 struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
278 int err;
279
280 spin_lock_init(&srq_table->lock);
281 INIT_RADIX_TREE(&srq_table->tree, GFP_ATOMIC);
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000282 if (mlx4_is_slave(dev))
283 return 0;
Roland Dreier225c7b12007-05-08 18:00:38 -0700284
285 err = mlx4_bitmap_init(&srq_table->bitmap, dev->caps.num_srqs,
Yevgeny Petrilin93fc9e12008-10-22 10:25:29 -0700286 dev->caps.num_srqs - 1, dev->caps.reserved_srqs, 0);
Roland Dreier225c7b12007-05-08 18:00:38 -0700287 if (err)
288 return err;
289
290 return 0;
291}
292
293void mlx4_cleanup_srq_table(struct mlx4_dev *dev)
294{
Jack Morgenstein3ec65b22011-12-13 04:13:05 +0000295 if (mlx4_is_slave(dev))
296 return;
Roland Dreier225c7b12007-05-08 18:00:38 -0700297 mlx4_bitmap_cleanup(&mlx4_priv(dev)->srq_table.bitmap);
298}
Shlomo Pongratz9f550552013-04-10 14:26:47 +0000299
300struct mlx4_srq *mlx4_srq_lookup(struct mlx4_dev *dev, u32 srqn)
301{
302 struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table;
303 struct mlx4_srq *srq;
304 unsigned long flags;
305
306 spin_lock_irqsave(&srq_table->lock, flags);
307 srq = radix_tree_lookup(&srq_table->tree,
308 srqn & (dev->caps.num_srqs - 1));
309 spin_unlock_irqrestore(&srq_table->lock, flags);
310
311 return srq;
312}
313EXPORT_SYMBOL_GPL(mlx4_srq_lookup);