blob: 833f3f1b87f5ee2ee0c279f30e26371b8580de70 [file] [log] [blame]
Andrew Scullb4b6d4a2019-01-02 15:54:55 +00001/*
2 * Copyright(c) 2016 - 2018 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47
48#include <linux/module.h>
49#include <linux/kernel.h>
50#include <linux/dma-mapping.h>
51#include "vt.h"
52#include "cq.h"
53#include "trace.h"
54
55#define RVT_UVERBS_ABI_VERSION 2
56
57MODULE_LICENSE("Dual BSD/GPL");
58MODULE_DESCRIPTION("RDMA Verbs Transport Library");
59
60static int rvt_init(void)
61{
62 int ret = rvt_driver_cq_init();
63
64 if (ret)
65 pr_err("Error in driver CQ init.\n");
66
67 return ret;
68}
69module_init(rvt_init);
70
71static void rvt_cleanup(void)
72{
73 rvt_cq_exit();
74}
75module_exit(rvt_cleanup);
76
77/**
78 * rvt_alloc_device - allocate rdi
79 * @size: how big of a structure to allocate
80 * @nports: number of ports to allocate array slots for
81 *
82 * Use IB core device alloc to allocate space for the rdi which is assumed to be
83 * inside of the ib_device. Any extra space that drivers require should be
84 * included in size.
85 *
86 * We also allocate a port array based on the number of ports.
87 *
88 * Return: pointer to allocated rdi
89 */
90struct rvt_dev_info *rvt_alloc_device(size_t size, int nports)
91{
92 struct rvt_dev_info *rdi;
93
David Brazdil0f672f62019-12-10 10:32:29 +000094 rdi = container_of(_ib_alloc_device(size), struct rvt_dev_info, ibdev);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000095 if (!rdi)
96 return rdi;
97
Olivier Deprez0e641232021-09-23 10:07:05 +020098 rdi->ports = kcalloc(nports, sizeof(*rdi->ports), GFP_KERNEL);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +000099 if (!rdi->ports)
100 ib_dealloc_device(&rdi->ibdev);
101
102 return rdi;
103}
104EXPORT_SYMBOL(rvt_alloc_device);
105
106/**
107 * rvt_dealloc_device - deallocate rdi
108 * @rdi: structure to free
109 *
110 * Free a structure allocated with rvt_alloc_device()
111 */
112void rvt_dealloc_device(struct rvt_dev_info *rdi)
113{
114 kfree(rdi->ports);
115 ib_dealloc_device(&rdi->ibdev);
116}
117EXPORT_SYMBOL(rvt_dealloc_device);
118
119static int rvt_query_device(struct ib_device *ibdev,
120 struct ib_device_attr *props,
121 struct ib_udata *uhw)
122{
123 struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
124
125 if (uhw->inlen || uhw->outlen)
126 return -EINVAL;
127 /*
128 * Return rvt_dev_info.dparms.props contents
129 */
130 *props = rdi->dparms.props;
131 return 0;
132}
133
134static int rvt_modify_device(struct ib_device *device,
135 int device_modify_mask,
136 struct ib_device_modify *device_modify)
137{
138 /*
139 * There is currently no need to supply this based on qib and hfi1.
140 * Future drivers may need to implement this though.
141 */
142
143 return -EOPNOTSUPP;
144}
145
146/**
147 * rvt_query_port: Passes the query port call to the driver
148 * @ibdev: Verbs IB dev
149 * @port_num: port number, 1 based from ib core
150 * @props: structure to hold returned properties
151 *
152 * Return: 0 on success
153 */
154static int rvt_query_port(struct ib_device *ibdev, u8 port_num,
155 struct ib_port_attr *props)
156{
157 struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
158 struct rvt_ibport *rvp;
159 int port_index = ibport_num_to_idx(ibdev, port_num);
160
161 if (port_index < 0)
162 return -EINVAL;
163
164 rvp = rdi->ports[port_index];
165 /* props being zeroed by the caller, avoid zeroing it here */
166 props->sm_lid = rvp->sm_lid;
167 props->sm_sl = rvp->sm_sl;
168 props->port_cap_flags = rvp->port_cap_flags;
169 props->max_msg_sz = 0x80000000;
170 props->pkey_tbl_len = rvt_get_npkeys(rdi);
171 props->bad_pkey_cntr = rvp->pkey_violations;
172 props->qkey_viol_cntr = rvp->qkey_violations;
173 props->subnet_timeout = rvp->subnet_timeout;
174 props->init_type_reply = 0;
175
176 /* Populate the remaining ib_port_attr elements */
177 return rdi->driver_f.query_port_state(rdi, port_num, props);
178}
179
180/**
181 * rvt_modify_port
182 * @ibdev: Verbs IB dev
183 * @port_num: Port number, 1 based from ib core
184 * @port_modify_mask: How to change the port
185 * @props: Structure to fill in
186 *
187 * Return: 0 on success
188 */
189static int rvt_modify_port(struct ib_device *ibdev, u8 port_num,
190 int port_modify_mask, struct ib_port_modify *props)
191{
192 struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
193 struct rvt_ibport *rvp;
194 int ret = 0;
195 int port_index = ibport_num_to_idx(ibdev, port_num);
196
197 if (port_index < 0)
198 return -EINVAL;
199
200 rvp = rdi->ports[port_index];
201 if (port_modify_mask & IB_PORT_OPA_MASK_CHG) {
202 rvp->port_cap3_flags |= props->set_port_cap_mask;
203 rvp->port_cap3_flags &= ~props->clr_port_cap_mask;
204 } else {
205 rvp->port_cap_flags |= props->set_port_cap_mask;
206 rvp->port_cap_flags &= ~props->clr_port_cap_mask;
207 }
208
209 if (props->set_port_cap_mask || props->clr_port_cap_mask)
210 rdi->driver_f.cap_mask_chg(rdi, port_num);
211 if (port_modify_mask & IB_PORT_SHUTDOWN)
212 ret = rdi->driver_f.shut_down_port(rdi, port_num);
213 if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR)
214 rvp->qkey_violations = 0;
215
216 return ret;
217}
218
219/**
220 * rvt_query_pkey - Return a pkey from the table at a given index
221 * @ibdev: Verbs IB dev
222 * @port_num: Port number, 1 based from ib core
223 * @index: Index into pkey table
224 * @pkey: returned pkey from the port pkey table
225 *
226 * Return: 0 on failure pkey otherwise
227 */
228static int rvt_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index,
229 u16 *pkey)
230{
231 /*
232 * Driver will be responsible for keeping rvt_dev_info.pkey_table up to
233 * date. This function will just return that value. There is no need to
234 * lock, if a stale value is read and sent to the user so be it there is
235 * no way to protect against that anyway.
236 */
237 struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
238 int port_index;
239
240 port_index = ibport_num_to_idx(ibdev, port_num);
241 if (port_index < 0)
242 return -EINVAL;
243
244 if (index >= rvt_get_npkeys(rdi))
245 return -EINVAL;
246
247 *pkey = rvt_get_pkey(rdi, port_index, index);
248 return 0;
249}
250
251/**
252 * rvt_query_gid - Return a gid from the table
253 * @ibdev: Verbs IB dev
254 * @port_num: Port number, 1 based from ib core
255 * @guid_index: Index in table
256 * @gid: Gid to return
257 *
258 * Return: 0 on success
259 */
260static int rvt_query_gid(struct ib_device *ibdev, u8 port_num,
261 int guid_index, union ib_gid *gid)
262{
263 struct rvt_dev_info *rdi;
264 struct rvt_ibport *rvp;
265 int port_index;
266
267 /*
268 * Driver is responsible for updating the guid table. Which will be used
269 * to craft the return value. This will work similar to how query_pkey()
270 * is being done.
271 */
272 port_index = ibport_num_to_idx(ibdev, port_num);
273 if (port_index < 0)
274 return -EINVAL;
275
276 rdi = ib_to_rvt(ibdev);
277 rvp = rdi->ports[port_index];
278
279 gid->global.subnet_prefix = rvp->gid_prefix;
280
281 return rdi->driver_f.get_guid_be(rdi, rvp, guid_index,
282 &gid->global.interface_id);
283}
284
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000285static inline struct rvt_ucontext *to_iucontext(struct ib_ucontext
286 *ibucontext)
287{
288 return container_of(ibucontext, struct rvt_ucontext, ibucontext);
289}
290
291/**
292 * rvt_alloc_ucontext - Allocate a user context
David Brazdil0f672f62019-12-10 10:32:29 +0000293 * @uctx: Verbs context
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000294 * @udata: User data allocated
295 */
David Brazdil0f672f62019-12-10 10:32:29 +0000296static int rvt_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000297{
David Brazdil0f672f62019-12-10 10:32:29 +0000298 return 0;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000299}
300
301/**
David Brazdil0f672f62019-12-10 10:32:29 +0000302 * rvt_dealloc_ucontext - Free a user context
303 * @context - Free this
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000304 */
David Brazdil0f672f62019-12-10 10:32:29 +0000305static void rvt_dealloc_ucontext(struct ib_ucontext *context)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000306{
David Brazdil0f672f62019-12-10 10:32:29 +0000307 return;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000308}
309
310static int rvt_get_port_immutable(struct ib_device *ibdev, u8 port_num,
311 struct ib_port_immutable *immutable)
312{
313 struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
314 struct ib_port_attr attr;
315 int err, port_index;
316
317 port_index = ibport_num_to_idx(ibdev, port_num);
318 if (port_index < 0)
319 return -EINVAL;
320
321 immutable->core_cap_flags = rdi->dparms.core_cap_flags;
322
323 err = ib_query_port(ibdev, port_num, &attr);
324 if (err)
325 return err;
326
327 immutable->pkey_tbl_len = attr.pkey_tbl_len;
328 immutable->gid_tbl_len = attr.gid_tbl_len;
329 immutable->max_mad_size = rdi->dparms.max_mad_size;
330
331 return 0;
332}
333
334enum {
335 MISC,
336 QUERY_DEVICE,
337 MODIFY_DEVICE,
338 QUERY_PORT,
339 MODIFY_PORT,
340 QUERY_PKEY,
341 QUERY_GID,
342 ALLOC_UCONTEXT,
343 DEALLOC_UCONTEXT,
344 GET_PORT_IMMUTABLE,
345 CREATE_QP,
346 MODIFY_QP,
347 DESTROY_QP,
348 QUERY_QP,
349 POST_SEND,
350 POST_RECV,
351 POST_SRQ_RECV,
352 CREATE_AH,
353 DESTROY_AH,
354 MODIFY_AH,
355 QUERY_AH,
356 CREATE_SRQ,
357 MODIFY_SRQ,
358 DESTROY_SRQ,
359 QUERY_SRQ,
360 ATTACH_MCAST,
361 DETACH_MCAST,
362 GET_DMA_MR,
363 REG_USER_MR,
364 DEREG_MR,
365 ALLOC_MR,
366 MAP_MR_SG,
367 ALLOC_FMR,
368 MAP_PHYS_FMR,
369 UNMAP_FMR,
370 DEALLOC_FMR,
371 MMAP,
372 CREATE_CQ,
373 DESTROY_CQ,
374 POLL_CQ,
375 REQ_NOTFIY_CQ,
376 RESIZE_CQ,
377 ALLOC_PD,
378 DEALLOC_PD,
379 _VERB_IDX_MAX /* Must always be last! */
380};
381
David Brazdil0f672f62019-12-10 10:32:29 +0000382static const struct ib_device_ops rvt_dev_ops = {
383 .uverbs_abi_ver = RVT_UVERBS_ABI_VERSION,
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000384
David Brazdil0f672f62019-12-10 10:32:29 +0000385 .alloc_fmr = rvt_alloc_fmr,
386 .alloc_mr = rvt_alloc_mr,
387 .alloc_pd = rvt_alloc_pd,
388 .alloc_ucontext = rvt_alloc_ucontext,
389 .attach_mcast = rvt_attach_mcast,
390 .create_ah = rvt_create_ah,
391 .create_cq = rvt_create_cq,
392 .create_qp = rvt_create_qp,
393 .create_srq = rvt_create_srq,
394 .dealloc_fmr = rvt_dealloc_fmr,
395 .dealloc_pd = rvt_dealloc_pd,
396 .dealloc_ucontext = rvt_dealloc_ucontext,
397 .dereg_mr = rvt_dereg_mr,
398 .destroy_ah = rvt_destroy_ah,
399 .destroy_cq = rvt_destroy_cq,
400 .destroy_qp = rvt_destroy_qp,
401 .destroy_srq = rvt_destroy_srq,
402 .detach_mcast = rvt_detach_mcast,
403 .get_dma_mr = rvt_get_dma_mr,
404 .get_port_immutable = rvt_get_port_immutable,
405 .map_mr_sg = rvt_map_mr_sg,
406 .map_phys_fmr = rvt_map_phys_fmr,
407 .mmap = rvt_mmap,
408 .modify_ah = rvt_modify_ah,
409 .modify_device = rvt_modify_device,
410 .modify_port = rvt_modify_port,
411 .modify_qp = rvt_modify_qp,
412 .modify_srq = rvt_modify_srq,
413 .poll_cq = rvt_poll_cq,
414 .post_recv = rvt_post_recv,
415 .post_send = rvt_post_send,
416 .post_srq_recv = rvt_post_srq_recv,
417 .query_ah = rvt_query_ah,
418 .query_device = rvt_query_device,
419 .query_gid = rvt_query_gid,
420 .query_pkey = rvt_query_pkey,
421 .query_port = rvt_query_port,
422 .query_qp = rvt_query_qp,
423 .query_srq = rvt_query_srq,
424 .reg_user_mr = rvt_reg_user_mr,
425 .req_notify_cq = rvt_req_notify_cq,
426 .resize_cq = rvt_resize_cq,
427 .unmap_fmr = rvt_unmap_fmr,
428
429 INIT_RDMA_OBJ_SIZE(ib_ah, rvt_ah, ibah),
430 INIT_RDMA_OBJ_SIZE(ib_cq, rvt_cq, ibcq),
431 INIT_RDMA_OBJ_SIZE(ib_pd, rvt_pd, ibpd),
432 INIT_RDMA_OBJ_SIZE(ib_srq, rvt_srq, ibsrq),
433 INIT_RDMA_OBJ_SIZE(ib_ucontext, rvt_ucontext, ibucontext),
434};
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000435
436static noinline int check_support(struct rvt_dev_info *rdi, int verb)
437{
438 switch (verb) {
439 case MISC:
440 /*
441 * These functions are not part of verbs specifically but are
442 * required for rdmavt to function.
443 */
David Brazdil0f672f62019-12-10 10:32:29 +0000444 if ((!rdi->ibdev.ops.init_port) ||
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000445 (!rdi->driver_f.get_pci_dev))
446 return -EINVAL;
447 break;
448
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000449 case MODIFY_DEVICE:
450 /*
451 * rdmavt does not support modify device currently drivers must
452 * provide.
453 */
David Brazdil0f672f62019-12-10 10:32:29 +0000454 if (!rdi->ibdev.ops.modify_device)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000455 return -EOPNOTSUPP;
456 break;
457
458 case QUERY_PORT:
David Brazdil0f672f62019-12-10 10:32:29 +0000459 if (!rdi->ibdev.ops.query_port)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000460 if (!rdi->driver_f.query_port_state)
461 return -EINVAL;
462 break;
463
464 case MODIFY_PORT:
David Brazdil0f672f62019-12-10 10:32:29 +0000465 if (!rdi->ibdev.ops.modify_port)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000466 if (!rdi->driver_f.cap_mask_chg ||
467 !rdi->driver_f.shut_down_port)
468 return -EINVAL;
469 break;
470
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000471 case QUERY_GID:
David Brazdil0f672f62019-12-10 10:32:29 +0000472 if (!rdi->ibdev.ops.query_gid)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000473 if (!rdi->driver_f.get_guid_be)
474 return -EINVAL;
475 break;
476
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000477 case CREATE_QP:
David Brazdil0f672f62019-12-10 10:32:29 +0000478 if (!rdi->ibdev.ops.create_qp)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000479 if (!rdi->driver_f.qp_priv_alloc ||
480 !rdi->driver_f.qp_priv_free ||
481 !rdi->driver_f.notify_qp_reset ||
482 !rdi->driver_f.flush_qp_waiters ||
483 !rdi->driver_f.stop_send_queue ||
484 !rdi->driver_f.quiesce_qp)
485 return -EINVAL;
486 break;
487
488 case MODIFY_QP:
David Brazdil0f672f62019-12-10 10:32:29 +0000489 if (!rdi->ibdev.ops.modify_qp)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000490 if (!rdi->driver_f.notify_qp_reset ||
491 !rdi->driver_f.schedule_send ||
492 !rdi->driver_f.get_pmtu_from_attr ||
493 !rdi->driver_f.flush_qp_waiters ||
494 !rdi->driver_f.stop_send_queue ||
495 !rdi->driver_f.quiesce_qp ||
496 !rdi->driver_f.notify_error_qp ||
497 !rdi->driver_f.mtu_from_qp ||
498 !rdi->driver_f.mtu_to_path_mtu)
499 return -EINVAL;
500 break;
501
502 case DESTROY_QP:
David Brazdil0f672f62019-12-10 10:32:29 +0000503 if (!rdi->ibdev.ops.destroy_qp)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000504 if (!rdi->driver_f.qp_priv_free ||
505 !rdi->driver_f.notify_qp_reset ||
506 !rdi->driver_f.flush_qp_waiters ||
507 !rdi->driver_f.stop_send_queue ||
508 !rdi->driver_f.quiesce_qp)
509 return -EINVAL;
510 break;
511
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000512 case POST_SEND:
David Brazdil0f672f62019-12-10 10:32:29 +0000513 if (!rdi->ibdev.ops.post_send)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000514 if (!rdi->driver_f.schedule_send ||
515 !rdi->driver_f.do_send ||
516 !rdi->post_parms)
517 return -EINVAL;
518 break;
519
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000520 }
521
522 return 0;
523}
524
525/**
526 * rvt_register_device - register a driver
527 * @rdi: main dev structure for all of rdmavt operations
528 *
529 * It is up to drivers to allocate the rdi and fill in the appropriate
530 * information.
531 *
532 * Return: 0 on success otherwise an errno.
533 */
David Brazdil0f672f62019-12-10 10:32:29 +0000534int rvt_register_device(struct rvt_dev_info *rdi)
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000535{
536 int ret = 0, i;
537
538 if (!rdi)
539 return -EINVAL;
540
541 /*
542 * Check to ensure drivers have setup the required helpers for the verbs
543 * they want rdmavt to handle
544 */
545 for (i = 0; i < _VERB_IDX_MAX; i++)
546 if (check_support(rdi, i)) {
547 pr_err("Driver support req not met at %d\n", i);
548 return -EINVAL;
549 }
550
David Brazdil0f672f62019-12-10 10:32:29 +0000551 ib_set_device_ops(&rdi->ibdev, &rvt_dev_ops);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000552
553 /* Once we get past here we can use rvt_pr macros and tracepoints */
554 trace_rvt_dbg(rdi, "Driver attempting registration");
555 rvt_mmap_init(rdi);
556
557 /* Queue Pairs */
558 ret = rvt_driver_qp_init(rdi);
559 if (ret) {
560 pr_err("Error in driver QP init.\n");
561 return -EINVAL;
562 }
563
564 /* Address Handle */
565 spin_lock_init(&rdi->n_ahs_lock);
566 rdi->n_ahs_allocated = 0;
567
568 /* Shared Receive Queue */
569 rvt_driver_srq_init(rdi);
570
571 /* Multicast */
572 rvt_driver_mcast_init(rdi);
573
574 /* Mem Region */
575 ret = rvt_driver_mr_init(rdi);
576 if (ret) {
577 pr_err("Error in driver MR init.\n");
578 goto bail_no_mr;
579 }
580
David Brazdil0f672f62019-12-10 10:32:29 +0000581 /* Memory Working Set Size */
582 ret = rvt_wss_init(rdi);
583 if (ret) {
584 rvt_pr_err(rdi, "Error in WSS init.\n");
585 goto bail_mr;
586 }
587
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000588 /* Completion queues */
589 spin_lock_init(&rdi->n_cqs_lock);
590
591 /* DMA Operations */
592 rdi->ibdev.dev.dma_ops = rdi->ibdev.dev.dma_ops ? : &dma_virt_ops;
593
594 /* Protection Domain */
595 spin_lock_init(&rdi->n_pds_lock);
596 rdi->n_pds_allocated = 0;
597
598 /*
599 * There are some things which could be set by underlying drivers but
600 * really should be up to rdmavt to set. For instance drivers can't know
601 * exactly which functions rdmavt supports, nor do they know the ABI
602 * version, so we do all of this sort of stuff here.
603 */
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000604 rdi->ibdev.uverbs_cmd_mask =
605 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
606 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
607 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
608 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
609 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
610 (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
611 (1ull << IB_USER_VERBS_CMD_MODIFY_AH) |
612 (1ull << IB_USER_VERBS_CMD_QUERY_AH) |
613 (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
614 (1ull << IB_USER_VERBS_CMD_REG_MR) |
615 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
616 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
617 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
618 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
619 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
620 (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
621 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
622 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
623 (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
624 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
625 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
626 (1ull << IB_USER_VERBS_CMD_POST_SEND) |
627 (1ull << IB_USER_VERBS_CMD_POST_RECV) |
628 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
629 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
630 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
631 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
632 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
633 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
634 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
635 rdi->ibdev.node_type = RDMA_NODE_IB_CA;
636 if (!rdi->ibdev.num_comp_vectors)
637 rdi->ibdev.num_comp_vectors = 1;
638
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000639 /* We are now good to announce we exist */
David Brazdil0f672f62019-12-10 10:32:29 +0000640 ret = ib_register_device(&rdi->ibdev, dev_name(&rdi->ibdev.dev));
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000641 if (ret) {
642 rvt_pr_err(rdi, "Failed to register driver with ib core.\n");
David Brazdil0f672f62019-12-10 10:32:29 +0000643 goto bail_wss;
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000644 }
645
646 rvt_create_mad_agents(rdi);
647
648 rvt_pr_info(rdi, "Registration with rdmavt done.\n");
649 return ret;
650
David Brazdil0f672f62019-12-10 10:32:29 +0000651bail_wss:
652 rvt_wss_exit(rdi);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000653bail_mr:
654 rvt_mr_exit(rdi);
655
656bail_no_mr:
657 rvt_qp_exit(rdi);
658
659 return ret;
660}
661EXPORT_SYMBOL(rvt_register_device);
662
663/**
664 * rvt_unregister_device - remove a driver
665 * @rdi: rvt dev struct
666 */
667void rvt_unregister_device(struct rvt_dev_info *rdi)
668{
669 trace_rvt_dbg(rdi, "Driver is unregistering.");
670 if (!rdi)
671 return;
672
673 rvt_free_mad_agents(rdi);
674
675 ib_unregister_device(&rdi->ibdev);
David Brazdil0f672f62019-12-10 10:32:29 +0000676 rvt_wss_exit(rdi);
Andrew Scullb4b6d4a2019-01-02 15:54:55 +0000677 rvt_mr_exit(rdi);
678 rvt_qp_exit(rdi);
679}
680EXPORT_SYMBOL(rvt_unregister_device);
681
682/**
683 * rvt_init_port - init internal data for driver port
684 * @rdi: rvt dev strut
685 * @port: rvt port
686 * @port_index: 0 based index of ports, different from IB core port num
687 *
688 * Keep track of a list of ports. No need to have a detach port.
689 * They persist until the driver goes away.
690 *
691 * Return: always 0
692 */
693int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
694 int port_index, u16 *pkey_table)
695{
696
697 rdi->ports[port_index] = port;
698 rdi->ports[port_index]->pkey_table = pkey_table;
699
700 return 0;
701}
702EXPORT_SYMBOL(rvt_init_port);