pvrdma_qp.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010
  1. /*
  2. * Copyright (c) 2012-2016 VMware, Inc. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of EITHER the GNU General Public License
  6. * version 2 as published by the Free Software Foundation or the BSD
  7. * 2-Clause License. This program is distributed in the hope that it
  8. * will be useful, but WITHOUT ANY WARRANTY; WITHOUT EVEN THE IMPLIED
  9. * WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
  10. * See the GNU General Public License version 2 for more details at
  11. * http://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html.
  12. *
  13. * You should have received a copy of the GNU General Public License
  14. * along with this program available in the file COPYING in the main
  15. * directory of this source tree.
  16. *
  17. * The BSD 2-Clause License
  18. *
  19. * Redistribution and use in source and binary forms, with or
  20. * without modification, are permitted provided that the following
  21. * conditions are met:
  22. *
  23. * - Redistributions of source code must retain the above
  24. * copyright notice, this list of conditions and the following
  25. * disclaimer.
  26. *
  27. * - Redistributions in binary form must reproduce the above
  28. * copyright notice, this list of conditions and the following
  29. * disclaimer in the documentation and/or other materials
  30. * provided with the distribution.
  31. *
  32. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  33. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  34. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  35. * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  36. * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
  37. * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  38. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  39. * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  40. * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  41. * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  42. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
  43. * OF THE POSSIBILITY OF SUCH DAMAGE.
  44. */
  45. #include <asm/page.h>
  46. #include <linux/io.h>
  47. #include <linux/wait.h>
  48. #include <rdma/ib_addr.h>
  49. #include <rdma/ib_smi.h>
  50. #include <rdma/ib_user_verbs.h>
  51. #include "pvrdma.h"
  52. static inline void get_cqs(struct pvrdma_qp *qp, struct pvrdma_cq **send_cq,
  53. struct pvrdma_cq **recv_cq)
  54. {
  55. *send_cq = to_vcq(qp->ibqp.send_cq);
  56. *recv_cq = to_vcq(qp->ibqp.recv_cq);
  57. }
  58. static void pvrdma_lock_cqs(struct pvrdma_cq *scq, struct pvrdma_cq *rcq,
  59. unsigned long *scq_flags,
  60. unsigned long *rcq_flags)
  61. __acquires(scq->cq_lock) __acquires(rcq->cq_lock)
  62. {
  63. if (scq == rcq) {
  64. spin_lock_irqsave(&scq->cq_lock, *scq_flags);
  65. __acquire(rcq->cq_lock);
  66. } else if (scq->cq_handle < rcq->cq_handle) {
  67. spin_lock_irqsave(&scq->cq_lock, *scq_flags);
  68. spin_lock_irqsave_nested(&rcq->cq_lock, *rcq_flags,
  69. SINGLE_DEPTH_NESTING);
  70. } else {
  71. spin_lock_irqsave(&rcq->cq_lock, *rcq_flags);
  72. spin_lock_irqsave_nested(&scq->cq_lock, *scq_flags,
  73. SINGLE_DEPTH_NESTING);
  74. }
  75. }
  76. static void pvrdma_unlock_cqs(struct pvrdma_cq *scq, struct pvrdma_cq *rcq,
  77. unsigned long *scq_flags,
  78. unsigned long *rcq_flags)
  79. __releases(scq->cq_lock) __releases(rcq->cq_lock)
  80. {
  81. if (scq == rcq) {
  82. __release(rcq->cq_lock);
  83. spin_unlock_irqrestore(&scq->cq_lock, *scq_flags);
  84. } else if (scq->cq_handle < rcq->cq_handle) {
  85. spin_unlock_irqrestore(&rcq->cq_lock, *rcq_flags);
  86. spin_unlock_irqrestore(&scq->cq_lock, *scq_flags);
  87. } else {
  88. spin_unlock_irqrestore(&scq->cq_lock, *scq_flags);
  89. spin_unlock_irqrestore(&rcq->cq_lock, *rcq_flags);
  90. }
  91. }
  92. static void pvrdma_reset_qp(struct pvrdma_qp *qp)
  93. {
  94. struct pvrdma_cq *scq, *rcq;
  95. unsigned long scq_flags, rcq_flags;
  96. /* Clean up cqes */
  97. get_cqs(qp, &scq, &rcq);
  98. pvrdma_lock_cqs(scq, rcq, &scq_flags, &rcq_flags);
  99. _pvrdma_flush_cqe(qp, scq);
  100. if (scq != rcq)
  101. _pvrdma_flush_cqe(qp, rcq);
  102. pvrdma_unlock_cqs(scq, rcq, &scq_flags, &rcq_flags);
  103. /*
  104. * Reset queuepair. The checks are because usermode queuepairs won't
  105. * have kernel ringstates.
  106. */
  107. if (qp->rq.ring) {
  108. atomic_set(&qp->rq.ring->cons_head, 0);
  109. atomic_set(&qp->rq.ring->prod_tail, 0);
  110. }
  111. if (qp->sq.ring) {
  112. atomic_set(&qp->sq.ring->cons_head, 0);
  113. atomic_set(&qp->sq.ring->prod_tail, 0);
  114. }
  115. }
  116. static int pvrdma_set_rq_size(struct pvrdma_dev *dev,
  117. struct ib_qp_cap *req_cap,
  118. struct pvrdma_qp *qp)
  119. {
  120. if (req_cap->max_recv_wr > dev->dsr->caps.max_qp_wr ||
  121. req_cap->max_recv_sge > dev->dsr->caps.max_sge) {
  122. dev_warn(&dev->pdev->dev, "recv queue size invalid\n");
  123. return -EINVAL;
  124. }
  125. qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, req_cap->max_recv_wr));
  126. qp->rq.max_sg = roundup_pow_of_two(max(1U, req_cap->max_recv_sge));
  127. /* Write back */
  128. req_cap->max_recv_wr = qp->rq.wqe_cnt;
  129. req_cap->max_recv_sge = qp->rq.max_sg;
  130. qp->rq.wqe_size = roundup_pow_of_two(sizeof(struct pvrdma_rq_wqe_hdr) +
  131. sizeof(struct pvrdma_sge) *
  132. qp->rq.max_sg);
  133. qp->npages_recv = (qp->rq.wqe_cnt * qp->rq.wqe_size + PAGE_SIZE - 1) /
  134. PAGE_SIZE;
  135. return 0;
  136. }
  137. static int pvrdma_set_sq_size(struct pvrdma_dev *dev, struct ib_qp_cap *req_cap,
  138. struct pvrdma_qp *qp)
  139. {
  140. if (req_cap->max_send_wr > dev->dsr->caps.max_qp_wr ||
  141. req_cap->max_send_sge > dev->dsr->caps.max_sge) {
  142. dev_warn(&dev->pdev->dev, "send queue size invalid\n");
  143. return -EINVAL;
  144. }
  145. qp->sq.wqe_cnt = roundup_pow_of_two(max(1U, req_cap->max_send_wr));
  146. qp->sq.max_sg = roundup_pow_of_two(max(1U, req_cap->max_send_sge));
  147. /* Write back */
  148. req_cap->max_send_wr = qp->sq.wqe_cnt;
  149. req_cap->max_send_sge = qp->sq.max_sg;
  150. qp->sq.wqe_size = roundup_pow_of_two(sizeof(struct pvrdma_sq_wqe_hdr) +
  151. sizeof(struct pvrdma_sge) *
  152. qp->sq.max_sg);
  153. /* Note: one extra page for the header. */
  154. qp->npages_send = PVRDMA_QP_NUM_HEADER_PAGES +
  155. (qp->sq.wqe_cnt * qp->sq.wqe_size + PAGE_SIZE - 1) /
  156. PAGE_SIZE;
  157. return 0;
  158. }
  159. /**
  160. * pvrdma_create_qp - create queue pair
  161. * @pd: protection domain
  162. * @init_attr: queue pair attributes
  163. * @udata: user data
  164. *
  165. * @return: the ib_qp pointer on success, otherwise returns an errno.
  166. */
  167. struct ib_qp *pvrdma_create_qp(struct ib_pd *pd,
  168. struct ib_qp_init_attr *init_attr,
  169. struct ib_udata *udata)
  170. {
  171. struct pvrdma_qp *qp = NULL;
  172. struct pvrdma_dev *dev = to_vdev(pd->device);
  173. union pvrdma_cmd_req req;
  174. union pvrdma_cmd_resp rsp;
  175. struct pvrdma_cmd_create_qp *cmd = &req.create_qp;
  176. struct pvrdma_cmd_create_qp_resp *resp = &rsp.create_qp_resp;
  177. struct pvrdma_create_qp ucmd;
  178. unsigned long flags;
  179. int ret;
  180. bool is_srq = !!init_attr->srq;
  181. if (init_attr->create_flags) {
  182. dev_warn(&dev->pdev->dev,
  183. "invalid create queuepair flags %#x\n",
  184. init_attr->create_flags);
  185. return ERR_PTR(-EINVAL);
  186. }
  187. if (init_attr->qp_type != IB_QPT_RC &&
  188. init_attr->qp_type != IB_QPT_UD &&
  189. init_attr->qp_type != IB_QPT_GSI) {
  190. dev_warn(&dev->pdev->dev, "queuepair type %d not supported\n",
  191. init_attr->qp_type);
  192. return ERR_PTR(-EINVAL);
  193. }
  194. if (is_srq && !dev->dsr->caps.max_srq) {
  195. dev_warn(&dev->pdev->dev,
  196. "SRQs not supported by device\n");
  197. return ERR_PTR(-EINVAL);
  198. }
  199. if (!atomic_add_unless(&dev->num_qps, 1, dev->dsr->caps.max_qp))
  200. return ERR_PTR(-ENOMEM);
  201. switch (init_attr->qp_type) {
  202. case IB_QPT_GSI:
  203. if (init_attr->port_num == 0 ||
  204. init_attr->port_num > pd->device->phys_port_cnt ||
  205. udata) {
  206. dev_warn(&dev->pdev->dev, "invalid queuepair attrs\n");
  207. ret = -EINVAL;
  208. goto err_qp;
  209. }
  210. /* fall through */
  211. case IB_QPT_RC:
  212. case IB_QPT_UD:
  213. qp = kzalloc(sizeof(*qp), GFP_KERNEL);
  214. if (!qp) {
  215. ret = -ENOMEM;
  216. goto err_qp;
  217. }
  218. spin_lock_init(&qp->sq.lock);
  219. spin_lock_init(&qp->rq.lock);
  220. mutex_init(&qp->mutex);
  221. refcount_set(&qp->refcnt, 1);
  222. init_completion(&qp->free);
  223. qp->state = IB_QPS_RESET;
  224. qp->is_kernel = !(pd->uobject && udata);
  225. if (!qp->is_kernel) {
  226. dev_dbg(&dev->pdev->dev,
  227. "create queuepair from user space\n");
  228. if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
  229. ret = -EFAULT;
  230. goto err_qp;
  231. }
  232. if (!is_srq) {
  233. /* set qp->sq.wqe_cnt, shift, buf_size.. */
  234. qp->rumem = ib_umem_get(pd->uobject->context,
  235. ucmd.rbuf_addr,
  236. ucmd.rbuf_size, 0, 0);
  237. if (IS_ERR(qp->rumem)) {
  238. ret = PTR_ERR(qp->rumem);
  239. goto err_qp;
  240. }
  241. qp->srq = NULL;
  242. } else {
  243. qp->rumem = NULL;
  244. qp->srq = to_vsrq(init_attr->srq);
  245. }
  246. qp->sumem = ib_umem_get(pd->uobject->context,
  247. ucmd.sbuf_addr,
  248. ucmd.sbuf_size, 0, 0);
  249. if (IS_ERR(qp->sumem)) {
  250. if (!is_srq)
  251. ib_umem_release(qp->rumem);
  252. ret = PTR_ERR(qp->sumem);
  253. goto err_qp;
  254. }
  255. qp->npages_send = ib_umem_page_count(qp->sumem);
  256. if (!is_srq)
  257. qp->npages_recv = ib_umem_page_count(qp->rumem);
  258. else
  259. qp->npages_recv = 0;
  260. qp->npages = qp->npages_send + qp->npages_recv;
  261. } else {
  262. ret = pvrdma_set_sq_size(to_vdev(pd->device),
  263. &init_attr->cap, qp);
  264. if (ret)
  265. goto err_qp;
  266. ret = pvrdma_set_rq_size(to_vdev(pd->device),
  267. &init_attr->cap, qp);
  268. if (ret)
  269. goto err_qp;
  270. qp->npages = qp->npages_send + qp->npages_recv;
  271. /* Skip header page. */
  272. qp->sq.offset = PVRDMA_QP_NUM_HEADER_PAGES * PAGE_SIZE;
  273. /* Recv queue pages are after send pages. */
  274. qp->rq.offset = qp->npages_send * PAGE_SIZE;
  275. }
  276. if (qp->npages < 0 || qp->npages > PVRDMA_PAGE_DIR_MAX_PAGES) {
  277. dev_warn(&dev->pdev->dev,
  278. "overflow pages in queuepair\n");
  279. ret = -EINVAL;
  280. goto err_umem;
  281. }
  282. ret = pvrdma_page_dir_init(dev, &qp->pdir, qp->npages,
  283. qp->is_kernel);
  284. if (ret) {
  285. dev_warn(&dev->pdev->dev,
  286. "could not allocate page directory\n");
  287. goto err_umem;
  288. }
  289. if (!qp->is_kernel) {
  290. pvrdma_page_dir_insert_umem(&qp->pdir, qp->sumem, 0);
  291. if (!is_srq)
  292. pvrdma_page_dir_insert_umem(&qp->pdir,
  293. qp->rumem,
  294. qp->npages_send);
  295. } else {
  296. /* Ring state is always the first page. */
  297. qp->sq.ring = qp->pdir.pages[0];
  298. qp->rq.ring = is_srq ? NULL : &qp->sq.ring[1];
  299. }
  300. break;
  301. default:
  302. ret = -EINVAL;
  303. goto err_qp;
  304. }
  305. /* Not supported */
  306. init_attr->cap.max_inline_data = 0;
  307. memset(cmd, 0, sizeof(*cmd));
  308. cmd->hdr.cmd = PVRDMA_CMD_CREATE_QP;
  309. cmd->pd_handle = to_vpd(pd)->pd_handle;
  310. cmd->send_cq_handle = to_vcq(init_attr->send_cq)->cq_handle;
  311. cmd->recv_cq_handle = to_vcq(init_attr->recv_cq)->cq_handle;
  312. if (is_srq)
  313. cmd->srq_handle = to_vsrq(init_attr->srq)->srq_handle;
  314. else
  315. cmd->srq_handle = 0;
  316. cmd->max_send_wr = init_attr->cap.max_send_wr;
  317. cmd->max_recv_wr = init_attr->cap.max_recv_wr;
  318. cmd->max_send_sge = init_attr->cap.max_send_sge;
  319. cmd->max_recv_sge = init_attr->cap.max_recv_sge;
  320. cmd->max_inline_data = init_attr->cap.max_inline_data;
  321. cmd->sq_sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;
  322. cmd->qp_type = ib_qp_type_to_pvrdma(init_attr->qp_type);
  323. cmd->is_srq = is_srq;
  324. cmd->lkey = 0;
  325. cmd->access_flags = IB_ACCESS_LOCAL_WRITE;
  326. cmd->total_chunks = qp->npages;
  327. cmd->send_chunks = qp->npages_send - PVRDMA_QP_NUM_HEADER_PAGES;
  328. cmd->pdir_dma = qp->pdir.dir_dma;
  329. dev_dbg(&dev->pdev->dev, "create queuepair with %d, %d, %d, %d\n",
  330. cmd->max_send_wr, cmd->max_recv_wr, cmd->max_send_sge,
  331. cmd->max_recv_sge);
  332. ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_QP_RESP);
  333. if (ret < 0) {
  334. dev_warn(&dev->pdev->dev,
  335. "could not create queuepair, error: %d\n", ret);
  336. goto err_pdir;
  337. }
  338. /* max_send_wr/_recv_wr/_send_sge/_recv_sge/_inline_data */
  339. qp->qp_handle = resp->qpn;
  340. qp->port = init_attr->port_num;
  341. qp->ibqp.qp_num = resp->qpn;
  342. spin_lock_irqsave(&dev->qp_tbl_lock, flags);
  343. dev->qp_tbl[qp->qp_handle % dev->dsr->caps.max_qp] = qp;
  344. spin_unlock_irqrestore(&dev->qp_tbl_lock, flags);
  345. return &qp->ibqp;
  346. err_pdir:
  347. pvrdma_page_dir_cleanup(dev, &qp->pdir);
  348. err_umem:
  349. if (!qp->is_kernel) {
  350. if (qp->rumem)
  351. ib_umem_release(qp->rumem);
  352. if (qp->sumem)
  353. ib_umem_release(qp->sumem);
  354. }
  355. err_qp:
  356. kfree(qp);
  357. atomic_dec(&dev->num_qps);
  358. return ERR_PTR(ret);
  359. }
  360. static void pvrdma_free_qp(struct pvrdma_qp *qp)
  361. {
  362. struct pvrdma_dev *dev = to_vdev(qp->ibqp.device);
  363. struct pvrdma_cq *scq;
  364. struct pvrdma_cq *rcq;
  365. unsigned long flags, scq_flags, rcq_flags;
  366. /* In case cq is polling */
  367. get_cqs(qp, &scq, &rcq);
  368. pvrdma_lock_cqs(scq, rcq, &scq_flags, &rcq_flags);
  369. _pvrdma_flush_cqe(qp, scq);
  370. if (scq != rcq)
  371. _pvrdma_flush_cqe(qp, rcq);
  372. spin_lock_irqsave(&dev->qp_tbl_lock, flags);
  373. dev->qp_tbl[qp->qp_handle] = NULL;
  374. spin_unlock_irqrestore(&dev->qp_tbl_lock, flags);
  375. pvrdma_unlock_cqs(scq, rcq, &scq_flags, &rcq_flags);
  376. if (refcount_dec_and_test(&qp->refcnt))
  377. complete(&qp->free);
  378. wait_for_completion(&qp->free);
  379. if (!qp->is_kernel) {
  380. if (qp->rumem)
  381. ib_umem_release(qp->rumem);
  382. if (qp->sumem)
  383. ib_umem_release(qp->sumem);
  384. }
  385. pvrdma_page_dir_cleanup(dev, &qp->pdir);
  386. kfree(qp);
  387. atomic_dec(&dev->num_qps);
  388. }
  389. /**
  390. * pvrdma_destroy_qp - destroy a queue pair
  391. * @qp: the queue pair to destroy
  392. *
  393. * @return: 0 on success.
  394. */
  395. int pvrdma_destroy_qp(struct ib_qp *qp)
  396. {
  397. struct pvrdma_qp *vqp = to_vqp(qp);
  398. union pvrdma_cmd_req req;
  399. struct pvrdma_cmd_destroy_qp *cmd = &req.destroy_qp;
  400. int ret;
  401. memset(cmd, 0, sizeof(*cmd));
  402. cmd->hdr.cmd = PVRDMA_CMD_DESTROY_QP;
  403. cmd->qp_handle = vqp->qp_handle;
  404. ret = pvrdma_cmd_post(to_vdev(qp->device), &req, NULL, 0);
  405. if (ret < 0)
  406. dev_warn(&to_vdev(qp->device)->pdev->dev,
  407. "destroy queuepair failed, error: %d\n", ret);
  408. pvrdma_free_qp(vqp);
  409. return 0;
  410. }
  411. /**
  412. * pvrdma_modify_qp - modify queue pair attributes
  413. * @ibqp: the queue pair
  414. * @attr: the new queue pair's attributes
  415. * @attr_mask: attributes mask
  416. * @udata: user data
  417. *
  418. * @returns 0 on success, otherwise returns an errno.
  419. */
  420. int pvrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
  421. int attr_mask, struct ib_udata *udata)
  422. {
  423. struct pvrdma_dev *dev = to_vdev(ibqp->device);
  424. struct pvrdma_qp *qp = to_vqp(ibqp);
  425. union pvrdma_cmd_req req;
  426. union pvrdma_cmd_resp rsp;
  427. struct pvrdma_cmd_modify_qp *cmd = &req.modify_qp;
  428. enum ib_qp_state cur_state, next_state;
  429. int ret;
  430. /* Sanity checking. Should need lock here */
  431. mutex_lock(&qp->mutex);
  432. cur_state = (attr_mask & IB_QP_CUR_STATE) ? attr->cur_qp_state :
  433. qp->state;
  434. next_state = (attr_mask & IB_QP_STATE) ? attr->qp_state : cur_state;
  435. if (!ib_modify_qp_is_ok(cur_state, next_state, ibqp->qp_type,
  436. attr_mask, IB_LINK_LAYER_ETHERNET)) {
  437. ret = -EINVAL;
  438. goto out;
  439. }
  440. if (attr_mask & IB_QP_PORT) {
  441. if (attr->port_num == 0 ||
  442. attr->port_num > ibqp->device->phys_port_cnt) {
  443. ret = -EINVAL;
  444. goto out;
  445. }
  446. }
  447. if (attr_mask & IB_QP_MIN_RNR_TIMER) {
  448. if (attr->min_rnr_timer > 31) {
  449. ret = -EINVAL;
  450. goto out;
  451. }
  452. }
  453. if (attr_mask & IB_QP_PKEY_INDEX) {
  454. if (attr->pkey_index >= dev->dsr->caps.max_pkeys) {
  455. ret = -EINVAL;
  456. goto out;
  457. }
  458. }
  459. if (attr_mask & IB_QP_QKEY)
  460. qp->qkey = attr->qkey;
  461. if (cur_state == next_state && cur_state == IB_QPS_RESET) {
  462. ret = 0;
  463. goto out;
  464. }
  465. qp->state = next_state;
  466. memset(cmd, 0, sizeof(*cmd));
  467. cmd->hdr.cmd = PVRDMA_CMD_MODIFY_QP;
  468. cmd->qp_handle = qp->qp_handle;
  469. cmd->attr_mask = ib_qp_attr_mask_to_pvrdma(attr_mask);
  470. cmd->attrs.qp_state = ib_qp_state_to_pvrdma(attr->qp_state);
  471. cmd->attrs.cur_qp_state =
  472. ib_qp_state_to_pvrdma(attr->cur_qp_state);
  473. cmd->attrs.path_mtu = ib_mtu_to_pvrdma(attr->path_mtu);
  474. cmd->attrs.path_mig_state =
  475. ib_mig_state_to_pvrdma(attr->path_mig_state);
  476. cmd->attrs.qkey = attr->qkey;
  477. cmd->attrs.rq_psn = attr->rq_psn;
  478. cmd->attrs.sq_psn = attr->sq_psn;
  479. cmd->attrs.dest_qp_num = attr->dest_qp_num;
  480. cmd->attrs.qp_access_flags =
  481. ib_access_flags_to_pvrdma(attr->qp_access_flags);
  482. cmd->attrs.pkey_index = attr->pkey_index;
  483. cmd->attrs.alt_pkey_index = attr->alt_pkey_index;
  484. cmd->attrs.en_sqd_async_notify = attr->en_sqd_async_notify;
  485. cmd->attrs.sq_draining = attr->sq_draining;
  486. cmd->attrs.max_rd_atomic = attr->max_rd_atomic;
  487. cmd->attrs.max_dest_rd_atomic = attr->max_dest_rd_atomic;
  488. cmd->attrs.min_rnr_timer = attr->min_rnr_timer;
  489. cmd->attrs.port_num = attr->port_num;
  490. cmd->attrs.timeout = attr->timeout;
  491. cmd->attrs.retry_cnt = attr->retry_cnt;
  492. cmd->attrs.rnr_retry = attr->rnr_retry;
  493. cmd->attrs.alt_port_num = attr->alt_port_num;
  494. cmd->attrs.alt_timeout = attr->alt_timeout;
  495. ib_qp_cap_to_pvrdma(&cmd->attrs.cap, &attr->cap);
  496. rdma_ah_attr_to_pvrdma(&cmd->attrs.ah_attr, &attr->ah_attr);
  497. rdma_ah_attr_to_pvrdma(&cmd->attrs.alt_ah_attr, &attr->alt_ah_attr);
  498. ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_MODIFY_QP_RESP);
  499. if (ret < 0) {
  500. dev_warn(&dev->pdev->dev,
  501. "could not modify queuepair, error: %d\n", ret);
  502. } else if (rsp.hdr.err > 0) {
  503. dev_warn(&dev->pdev->dev,
  504. "cannot modify queuepair, error: %d\n", rsp.hdr.err);
  505. ret = -EINVAL;
  506. }
  507. if (ret == 0 && next_state == IB_QPS_RESET)
  508. pvrdma_reset_qp(qp);
  509. out:
  510. mutex_unlock(&qp->mutex);
  511. return ret;
  512. }
  513. static inline void *get_sq_wqe(struct pvrdma_qp *qp, unsigned int n)
  514. {
  515. return pvrdma_page_dir_get_ptr(&qp->pdir,
  516. qp->sq.offset + n * qp->sq.wqe_size);
  517. }
  518. static inline void *get_rq_wqe(struct pvrdma_qp *qp, unsigned int n)
  519. {
  520. return pvrdma_page_dir_get_ptr(&qp->pdir,
  521. qp->rq.offset + n * qp->rq.wqe_size);
  522. }
  523. static int set_reg_seg(struct pvrdma_sq_wqe_hdr *wqe_hdr,
  524. const struct ib_reg_wr *wr)
  525. {
  526. struct pvrdma_user_mr *mr = to_vmr(wr->mr);
  527. wqe_hdr->wr.fast_reg.iova_start = mr->ibmr.iova;
  528. wqe_hdr->wr.fast_reg.pl_pdir_dma = mr->pdir.dir_dma;
  529. wqe_hdr->wr.fast_reg.page_shift = mr->page_shift;
  530. wqe_hdr->wr.fast_reg.page_list_len = mr->npages;
  531. wqe_hdr->wr.fast_reg.length = mr->ibmr.length;
  532. wqe_hdr->wr.fast_reg.access_flags = wr->access;
  533. wqe_hdr->wr.fast_reg.rkey = wr->key;
  534. return pvrdma_page_dir_insert_page_list(&mr->pdir, mr->pages,
  535. mr->npages);
  536. }
  537. /**
  538. * pvrdma_post_send - post send work request entries on a QP
  539. * @ibqp: the QP
  540. * @wr: work request list to post
  541. * @bad_wr: the first bad WR returned
  542. *
  543. * @return: 0 on success, otherwise errno returned.
  544. */
  545. int pvrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
  546. const struct ib_send_wr **bad_wr)
  547. {
  548. struct pvrdma_qp *qp = to_vqp(ibqp);
  549. struct pvrdma_dev *dev = to_vdev(ibqp->device);
  550. unsigned long flags;
  551. struct pvrdma_sq_wqe_hdr *wqe_hdr;
  552. struct pvrdma_sge *sge;
  553. int i, ret;
  554. /*
  555. * In states lower than RTS, we can fail immediately. In other states,
  556. * just post and let the device figure it out.
  557. */
  558. if (qp->state < IB_QPS_RTS) {
  559. *bad_wr = wr;
  560. return -EINVAL;
  561. }
  562. spin_lock_irqsave(&qp->sq.lock, flags);
  563. while (wr) {
  564. unsigned int tail = 0;
  565. if (unlikely(!pvrdma_idx_ring_has_space(
  566. qp->sq.ring, qp->sq.wqe_cnt, &tail))) {
  567. dev_warn_ratelimited(&dev->pdev->dev,
  568. "send queue is full\n");
  569. *bad_wr = wr;
  570. ret = -ENOMEM;
  571. goto out;
  572. }
  573. if (unlikely(wr->num_sge > qp->sq.max_sg || wr->num_sge < 0)) {
  574. dev_warn_ratelimited(&dev->pdev->dev,
  575. "send SGE overflow\n");
  576. *bad_wr = wr;
  577. ret = -EINVAL;
  578. goto out;
  579. }
  580. if (unlikely(wr->opcode < 0)) {
  581. dev_warn_ratelimited(&dev->pdev->dev,
  582. "invalid send opcode\n");
  583. *bad_wr = wr;
  584. ret = -EINVAL;
  585. goto out;
  586. }
  587. /*
  588. * Only support UD, RC.
  589. * Need to check opcode table for thorough checking.
  590. * opcode _UD _UC _RC
  591. * _SEND x x x
  592. * _SEND_WITH_IMM x x x
  593. * _RDMA_WRITE x x
  594. * _RDMA_WRITE_WITH_IMM x x
  595. * _LOCAL_INV x x
  596. * _SEND_WITH_INV x x
  597. * _RDMA_READ x
  598. * _ATOMIC_CMP_AND_SWP x
  599. * _ATOMIC_FETCH_AND_ADD x
  600. * _MASK_ATOMIC_CMP_AND_SWP x
  601. * _MASK_ATOMIC_FETCH_AND_ADD x
  602. * _REG_MR x
  603. *
  604. */
  605. if (qp->ibqp.qp_type != IB_QPT_UD &&
  606. qp->ibqp.qp_type != IB_QPT_RC &&
  607. wr->opcode != IB_WR_SEND) {
  608. dev_warn_ratelimited(&dev->pdev->dev,
  609. "unsupported queuepair type\n");
  610. *bad_wr = wr;
  611. ret = -EINVAL;
  612. goto out;
  613. } else if (qp->ibqp.qp_type == IB_QPT_UD ||
  614. qp->ibqp.qp_type == IB_QPT_GSI) {
  615. if (wr->opcode != IB_WR_SEND &&
  616. wr->opcode != IB_WR_SEND_WITH_IMM) {
  617. dev_warn_ratelimited(&dev->pdev->dev,
  618. "invalid send opcode\n");
  619. *bad_wr = wr;
  620. ret = -EINVAL;
  621. goto out;
  622. }
  623. }
  624. wqe_hdr = (struct pvrdma_sq_wqe_hdr *)get_sq_wqe(qp, tail);
  625. memset(wqe_hdr, 0, sizeof(*wqe_hdr));
  626. wqe_hdr->wr_id = wr->wr_id;
  627. wqe_hdr->num_sge = wr->num_sge;
  628. wqe_hdr->opcode = ib_wr_opcode_to_pvrdma(wr->opcode);
  629. wqe_hdr->send_flags = ib_send_flags_to_pvrdma(wr->send_flags);
  630. if (wr->opcode == IB_WR_SEND_WITH_IMM ||
  631. wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM)
  632. wqe_hdr->ex.imm_data = wr->ex.imm_data;
  633. if (unlikely(wqe_hdr->opcode == PVRDMA_WR_ERROR)) {
  634. *bad_wr = wr;
  635. ret = -EINVAL;
  636. goto out;
  637. }
  638. switch (qp->ibqp.qp_type) {
  639. case IB_QPT_GSI:
  640. case IB_QPT_UD:
  641. if (unlikely(!ud_wr(wr)->ah)) {
  642. dev_warn_ratelimited(&dev->pdev->dev,
  643. "invalid address handle\n");
  644. *bad_wr = wr;
  645. ret = -EINVAL;
  646. goto out;
  647. }
  648. /*
  649. * Use qkey from qp context if high order bit set,
  650. * otherwise from work request.
  651. */
  652. wqe_hdr->wr.ud.remote_qpn = ud_wr(wr)->remote_qpn;
  653. wqe_hdr->wr.ud.remote_qkey =
  654. ud_wr(wr)->remote_qkey & 0x80000000 ?
  655. qp->qkey : ud_wr(wr)->remote_qkey;
  656. wqe_hdr->wr.ud.av = to_vah(ud_wr(wr)->ah)->av;
  657. break;
  658. case IB_QPT_RC:
  659. switch (wr->opcode) {
  660. case IB_WR_RDMA_READ:
  661. case IB_WR_RDMA_WRITE:
  662. case IB_WR_RDMA_WRITE_WITH_IMM:
  663. wqe_hdr->wr.rdma.remote_addr =
  664. rdma_wr(wr)->remote_addr;
  665. wqe_hdr->wr.rdma.rkey = rdma_wr(wr)->rkey;
  666. break;
  667. case IB_WR_LOCAL_INV:
  668. case IB_WR_SEND_WITH_INV:
  669. wqe_hdr->ex.invalidate_rkey =
  670. wr->ex.invalidate_rkey;
  671. break;
  672. case IB_WR_ATOMIC_CMP_AND_SWP:
  673. case IB_WR_ATOMIC_FETCH_AND_ADD:
  674. wqe_hdr->wr.atomic.remote_addr =
  675. atomic_wr(wr)->remote_addr;
  676. wqe_hdr->wr.atomic.rkey = atomic_wr(wr)->rkey;
  677. wqe_hdr->wr.atomic.compare_add =
  678. atomic_wr(wr)->compare_add;
  679. if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP)
  680. wqe_hdr->wr.atomic.swap =
  681. atomic_wr(wr)->swap;
  682. break;
  683. case IB_WR_REG_MR:
  684. ret = set_reg_seg(wqe_hdr, reg_wr(wr));
  685. if (ret < 0) {
  686. dev_warn_ratelimited(&dev->pdev->dev,
  687. "Failed to set fast register work request\n");
  688. *bad_wr = wr;
  689. goto out;
  690. }
  691. break;
  692. default:
  693. break;
  694. }
  695. break;
  696. default:
  697. dev_warn_ratelimited(&dev->pdev->dev,
  698. "invalid queuepair type\n");
  699. ret = -EINVAL;
  700. *bad_wr = wr;
  701. goto out;
  702. }
  703. sge = (struct pvrdma_sge *)(wqe_hdr + 1);
  704. for (i = 0; i < wr->num_sge; i++) {
  705. /* Need to check wqe_size 0 or max size */
  706. sge->addr = wr->sg_list[i].addr;
  707. sge->length = wr->sg_list[i].length;
  708. sge->lkey = wr->sg_list[i].lkey;
  709. sge++;
  710. }
  711. /* Make sure wqe is written before index update */
  712. smp_wmb();
  713. /* Update shared sq ring */
  714. pvrdma_idx_ring_inc(&qp->sq.ring->prod_tail,
  715. qp->sq.wqe_cnt);
  716. wr = wr->next;
  717. }
  718. ret = 0;
  719. out:
  720. spin_unlock_irqrestore(&qp->sq.lock, flags);
  721. if (!ret)
  722. pvrdma_write_uar_qp(dev, PVRDMA_UAR_QP_SEND | qp->qp_handle);
  723. return ret;
  724. }
  725. /**
  726. * pvrdma_post_receive - post receive work request entries on a QP
  727. * @ibqp: the QP
  728. * @wr: the work request list to post
  729. * @bad_wr: the first bad WR returned
  730. *
  731. * @return: 0 on success, otherwise errno returned.
  732. */
  733. int pvrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
  734. const struct ib_recv_wr **bad_wr)
  735. {
  736. struct pvrdma_dev *dev = to_vdev(ibqp->device);
  737. unsigned long flags;
  738. struct pvrdma_qp *qp = to_vqp(ibqp);
  739. struct pvrdma_rq_wqe_hdr *wqe_hdr;
  740. struct pvrdma_sge *sge;
  741. int ret = 0;
  742. int i;
  743. /*
  744. * In the RESET state, we can fail immediately. For other states,
  745. * just post and let the device figure it out.
  746. */
  747. if (qp->state == IB_QPS_RESET) {
  748. *bad_wr = wr;
  749. return -EINVAL;
  750. }
  751. if (qp->srq) {
  752. dev_warn(&dev->pdev->dev, "QP associated with SRQ\n");
  753. *bad_wr = wr;
  754. return -EINVAL;
  755. }
  756. spin_lock_irqsave(&qp->rq.lock, flags);
  757. while (wr) {
  758. unsigned int tail = 0;
  759. if (unlikely(wr->num_sge > qp->rq.max_sg ||
  760. wr->num_sge < 0)) {
  761. ret = -EINVAL;
  762. *bad_wr = wr;
  763. dev_warn_ratelimited(&dev->pdev->dev,
  764. "recv SGE overflow\n");
  765. goto out;
  766. }
  767. if (unlikely(!pvrdma_idx_ring_has_space(
  768. qp->rq.ring, qp->rq.wqe_cnt, &tail))) {
  769. ret = -ENOMEM;
  770. *bad_wr = wr;
  771. dev_warn_ratelimited(&dev->pdev->dev,
  772. "recv queue full\n");
  773. goto out;
  774. }
  775. wqe_hdr = (struct pvrdma_rq_wqe_hdr *)get_rq_wqe(qp, tail);
  776. wqe_hdr->wr_id = wr->wr_id;
  777. wqe_hdr->num_sge = wr->num_sge;
  778. wqe_hdr->total_len = 0;
  779. sge = (struct pvrdma_sge *)(wqe_hdr + 1);
  780. for (i = 0; i < wr->num_sge; i++) {
  781. sge->addr = wr->sg_list[i].addr;
  782. sge->length = wr->sg_list[i].length;
  783. sge->lkey = wr->sg_list[i].lkey;
  784. sge++;
  785. }
  786. /* Make sure wqe is written before index update */
  787. smp_wmb();
  788. /* Update shared rq ring */
  789. pvrdma_idx_ring_inc(&qp->rq.ring->prod_tail,
  790. qp->rq.wqe_cnt);
  791. wr = wr->next;
  792. }
  793. spin_unlock_irqrestore(&qp->rq.lock, flags);
  794. pvrdma_write_uar_qp(dev, PVRDMA_UAR_QP_RECV | qp->qp_handle);
  795. return ret;
  796. out:
  797. spin_unlock_irqrestore(&qp->rq.lock, flags);
  798. return ret;
  799. }
  800. /**
  801. * pvrdma_query_qp - query a queue pair's attributes
  802. * @ibqp: the queue pair to query
  803. * @attr: the queue pair's attributes
  804. * @attr_mask: attributes mask
  805. * @init_attr: initial queue pair attributes
  806. *
  807. * @returns 0 on success, otherwise returns an errno.
  808. */
  809. int pvrdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
  810. int attr_mask, struct ib_qp_init_attr *init_attr)
  811. {
  812. struct pvrdma_dev *dev = to_vdev(ibqp->device);
  813. struct pvrdma_qp *qp = to_vqp(ibqp);
  814. union pvrdma_cmd_req req;
  815. union pvrdma_cmd_resp rsp;
  816. struct pvrdma_cmd_query_qp *cmd = &req.query_qp;
  817. struct pvrdma_cmd_query_qp_resp *resp = &rsp.query_qp_resp;
  818. int ret = 0;
  819. mutex_lock(&qp->mutex);
  820. if (qp->state == IB_QPS_RESET) {
  821. attr->qp_state = IB_QPS_RESET;
  822. goto out;
  823. }
  824. memset(cmd, 0, sizeof(*cmd));
  825. cmd->hdr.cmd = PVRDMA_CMD_QUERY_QP;
  826. cmd->qp_handle = qp->qp_handle;
  827. cmd->attr_mask = ib_qp_attr_mask_to_pvrdma(attr_mask);
  828. ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_QUERY_QP_RESP);
  829. if (ret < 0) {
  830. dev_warn(&dev->pdev->dev,
  831. "could not query queuepair, error: %d\n", ret);
  832. goto out;
  833. }
  834. attr->qp_state = pvrdma_qp_state_to_ib(resp->attrs.qp_state);
  835. attr->cur_qp_state =
  836. pvrdma_qp_state_to_ib(resp->attrs.cur_qp_state);
  837. attr->path_mtu = pvrdma_mtu_to_ib(resp->attrs.path_mtu);
  838. attr->path_mig_state =
  839. pvrdma_mig_state_to_ib(resp->attrs.path_mig_state);
  840. attr->qkey = resp->attrs.qkey;
  841. attr->rq_psn = resp->attrs.rq_psn;
  842. attr->sq_psn = resp->attrs.sq_psn;
  843. attr->dest_qp_num = resp->attrs.dest_qp_num;
  844. attr->qp_access_flags =
  845. pvrdma_access_flags_to_ib(resp->attrs.qp_access_flags);
  846. attr->pkey_index = resp->attrs.pkey_index;
  847. attr->alt_pkey_index = resp->attrs.alt_pkey_index;
  848. attr->en_sqd_async_notify = resp->attrs.en_sqd_async_notify;
  849. attr->sq_draining = resp->attrs.sq_draining;
  850. attr->max_rd_atomic = resp->attrs.max_rd_atomic;
  851. attr->max_dest_rd_atomic = resp->attrs.max_dest_rd_atomic;
  852. attr->min_rnr_timer = resp->attrs.min_rnr_timer;
  853. attr->port_num = resp->attrs.port_num;
  854. attr->timeout = resp->attrs.timeout;
  855. attr->retry_cnt = resp->attrs.retry_cnt;
  856. attr->rnr_retry = resp->attrs.rnr_retry;
  857. attr->alt_port_num = resp->attrs.alt_port_num;
  858. attr->alt_timeout = resp->attrs.alt_timeout;
  859. pvrdma_qp_cap_to_ib(&attr->cap, &resp->attrs.cap);
  860. pvrdma_ah_attr_to_rdma(&attr->ah_attr, &resp->attrs.ah_attr);
  861. pvrdma_ah_attr_to_rdma(&attr->alt_ah_attr, &resp->attrs.alt_ah_attr);
  862. qp->state = attr->qp_state;
  863. ret = 0;
  864. out:
  865. attr->cur_qp_state = attr->qp_state;
  866. init_attr->event_handler = qp->ibqp.event_handler;
  867. init_attr->qp_context = qp->ibqp.qp_context;
  868. init_attr->send_cq = qp->ibqp.send_cq;
  869. init_attr->recv_cq = qp->ibqp.recv_cq;
  870. init_attr->srq = qp->ibqp.srq;
  871. init_attr->xrcd = NULL;
  872. init_attr->cap = attr->cap;
  873. init_attr->sq_sig_type = 0;
  874. init_attr->qp_type = qp->ibqp.qp_type;
  875. init_attr->create_flags = 0;
  876. init_attr->port_num = qp->port;
  877. mutex_unlock(&qp->mutex);
  878. return ret;
  879. }