cq.c 30 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193
  1. /*
  2. * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. */
  32. #include "iw_cxgb4.h"
  33. static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
  34. struct c4iw_dev_ucontext *uctx, struct sk_buff *skb,
  35. struct c4iw_wr_wait *wr_waitp)
  36. {
  37. struct fw_ri_res_wr *res_wr;
  38. struct fw_ri_res *res;
  39. int wr_len;
  40. int ret;
  41. wr_len = sizeof *res_wr + sizeof *res;
  42. set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);
  43. res_wr = __skb_put_zero(skb, wr_len);
  44. res_wr->op_nres = cpu_to_be32(
  45. FW_WR_OP_V(FW_RI_RES_WR) |
  46. FW_RI_RES_WR_NRES_V(1) |
  47. FW_WR_COMPL_F);
  48. res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16));
  49. res_wr->cookie = (uintptr_t)wr_waitp;
  50. res = res_wr->res;
  51. res->u.cq.restype = FW_RI_RES_TYPE_CQ;
  52. res->u.cq.op = FW_RI_RES_OP_RESET;
  53. res->u.cq.iqid = cpu_to_be32(cq->cqid);
  54. c4iw_init_wr_wait(wr_waitp);
  55. ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__);
  56. kfree(cq->sw_queue);
  57. dma_free_coherent(&(rdev->lldi.pdev->dev),
  58. cq->memsize, cq->queue,
  59. dma_unmap_addr(cq, mapping));
  60. c4iw_put_cqid(rdev, cq->cqid, uctx);
  61. return ret;
  62. }
  63. static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
  64. struct c4iw_dev_ucontext *uctx,
  65. struct c4iw_wr_wait *wr_waitp)
  66. {
  67. struct fw_ri_res_wr *res_wr;
  68. struct fw_ri_res *res;
  69. int wr_len;
  70. int user = (uctx != &rdev->uctx);
  71. int ret;
  72. struct sk_buff *skb;
  73. struct c4iw_ucontext *ucontext = NULL;
  74. if (user)
  75. ucontext = container_of(uctx, struct c4iw_ucontext, uctx);
  76. cq->cqid = c4iw_get_cqid(rdev, uctx);
  77. if (!cq->cqid) {
  78. ret = -ENOMEM;
  79. goto err1;
  80. }
  81. if (!user) {
  82. cq->sw_queue = kzalloc(cq->memsize, GFP_KERNEL);
  83. if (!cq->sw_queue) {
  84. ret = -ENOMEM;
  85. goto err2;
  86. }
  87. }
  88. cq->queue = dma_alloc_coherent(&rdev->lldi.pdev->dev, cq->memsize,
  89. &cq->dma_addr, GFP_KERNEL);
  90. if (!cq->queue) {
  91. ret = -ENOMEM;
  92. goto err3;
  93. }
  94. dma_unmap_addr_set(cq, mapping, cq->dma_addr);
  95. memset(cq->queue, 0, cq->memsize);
  96. if (user && ucontext->is_32b_cqe) {
  97. cq->qp_errp = &((struct t4_status_page *)
  98. ((u8 *)cq->queue + (cq->size - 1) *
  99. (sizeof(*cq->queue) / 2)))->qp_err;
  100. } else {
  101. cq->qp_errp = &((struct t4_status_page *)
  102. ((u8 *)cq->queue + (cq->size - 1) *
  103. sizeof(*cq->queue)))->qp_err;
  104. }
  105. /* build fw_ri_res_wr */
  106. wr_len = sizeof *res_wr + sizeof *res;
  107. skb = alloc_skb(wr_len, GFP_KERNEL);
  108. if (!skb) {
  109. ret = -ENOMEM;
  110. goto err4;
  111. }
  112. set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);
  113. res_wr = __skb_put_zero(skb, wr_len);
  114. res_wr->op_nres = cpu_to_be32(
  115. FW_WR_OP_V(FW_RI_RES_WR) |
  116. FW_RI_RES_WR_NRES_V(1) |
  117. FW_WR_COMPL_F);
  118. res_wr->len16_pkd = cpu_to_be32(DIV_ROUND_UP(wr_len, 16));
  119. res_wr->cookie = (uintptr_t)wr_waitp;
  120. res = res_wr->res;
  121. res->u.cq.restype = FW_RI_RES_TYPE_CQ;
  122. res->u.cq.op = FW_RI_RES_OP_WRITE;
  123. res->u.cq.iqid = cpu_to_be32(cq->cqid);
  124. res->u.cq.iqandst_to_iqandstindex = cpu_to_be32(
  125. FW_RI_RES_WR_IQANUS_V(0) |
  126. FW_RI_RES_WR_IQANUD_V(1) |
  127. FW_RI_RES_WR_IQANDST_F |
  128. FW_RI_RES_WR_IQANDSTINDEX_V(
  129. rdev->lldi.ciq_ids[cq->vector]));
  130. res->u.cq.iqdroprss_to_iqesize = cpu_to_be16(
  131. FW_RI_RES_WR_IQDROPRSS_F |
  132. FW_RI_RES_WR_IQPCIECH_V(2) |
  133. FW_RI_RES_WR_IQINTCNTTHRESH_V(0) |
  134. FW_RI_RES_WR_IQO_F |
  135. ((user && ucontext->is_32b_cqe) ?
  136. FW_RI_RES_WR_IQESIZE_V(1) :
  137. FW_RI_RES_WR_IQESIZE_V(2)));
  138. res->u.cq.iqsize = cpu_to_be16(cq->size);
  139. res->u.cq.iqaddr = cpu_to_be64(cq->dma_addr);
  140. c4iw_init_wr_wait(wr_waitp);
  141. ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__);
  142. if (ret)
  143. goto err4;
  144. cq->gen = 1;
  145. cq->gts = rdev->lldi.gts_reg;
  146. cq->rdev = rdev;
  147. cq->bar2_va = c4iw_bar2_addrs(rdev, cq->cqid, CXGB4_BAR2_QTYPE_INGRESS,
  148. &cq->bar2_qid,
  149. user ? &cq->bar2_pa : NULL);
  150. if (user && !cq->bar2_pa) {
  151. pr_warn("%s: cqid %u not in BAR2 range\n",
  152. pci_name(rdev->lldi.pdev), cq->cqid);
  153. ret = -EINVAL;
  154. goto err4;
  155. }
  156. return 0;
  157. err4:
  158. dma_free_coherent(&rdev->lldi.pdev->dev, cq->memsize, cq->queue,
  159. dma_unmap_addr(cq, mapping));
  160. err3:
  161. kfree(cq->sw_queue);
  162. err2:
  163. c4iw_put_cqid(rdev, cq->cqid, uctx);
  164. err1:
  165. return ret;
  166. }
  167. static void insert_recv_cqe(struct t4_wq *wq, struct t4_cq *cq, u32 srqidx)
  168. {
  169. struct t4_cqe cqe;
  170. pr_debug("wq %p cq %p sw_cidx %u sw_pidx %u\n",
  171. wq, cq, cq->sw_cidx, cq->sw_pidx);
  172. memset(&cqe, 0, sizeof(cqe));
  173. cqe.header = cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH) |
  174. CQE_OPCODE_V(FW_RI_SEND) |
  175. CQE_TYPE_V(0) |
  176. CQE_SWCQE_V(1) |
  177. CQE_QPID_V(wq->sq.qid));
  178. cqe.bits_type_ts = cpu_to_be64(CQE_GENBIT_V((u64)cq->gen));
  179. if (srqidx)
  180. cqe.u.srcqe.abs_rqe_idx = cpu_to_be32(srqidx);
  181. cq->sw_queue[cq->sw_pidx] = cqe;
  182. t4_swcq_produce(cq);
  183. }
  184. int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count)
  185. {
  186. int flushed = 0;
  187. int in_use = wq->rq.in_use - count;
  188. pr_debug("wq %p cq %p rq.in_use %u skip count %u\n",
  189. wq, cq, wq->rq.in_use, count);
  190. while (in_use--) {
  191. insert_recv_cqe(wq, cq, 0);
  192. flushed++;
  193. }
  194. return flushed;
  195. }
  196. static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,
  197. struct t4_swsqe *swcqe)
  198. {
  199. struct t4_cqe cqe;
  200. pr_debug("wq %p cq %p sw_cidx %u sw_pidx %u\n",
  201. wq, cq, cq->sw_cidx, cq->sw_pidx);
  202. memset(&cqe, 0, sizeof(cqe));
  203. cqe.header = cpu_to_be32(CQE_STATUS_V(T4_ERR_SWFLUSH) |
  204. CQE_OPCODE_V(swcqe->opcode) |
  205. CQE_TYPE_V(1) |
  206. CQE_SWCQE_V(1) |
  207. CQE_QPID_V(wq->sq.qid));
  208. CQE_WRID_SQ_IDX(&cqe) = swcqe->idx;
  209. cqe.bits_type_ts = cpu_to_be64(CQE_GENBIT_V((u64)cq->gen));
  210. cq->sw_queue[cq->sw_pidx] = cqe;
  211. t4_swcq_produce(cq);
  212. }
  213. static void advance_oldest_read(struct t4_wq *wq);
  214. int c4iw_flush_sq(struct c4iw_qp *qhp)
  215. {
  216. int flushed = 0;
  217. struct t4_wq *wq = &qhp->wq;
  218. struct c4iw_cq *chp = to_c4iw_cq(qhp->ibqp.send_cq);
  219. struct t4_cq *cq = &chp->cq;
  220. int idx;
  221. struct t4_swsqe *swsqe;
  222. if (wq->sq.flush_cidx == -1)
  223. wq->sq.flush_cidx = wq->sq.cidx;
  224. idx = wq->sq.flush_cidx;
  225. while (idx != wq->sq.pidx) {
  226. swsqe = &wq->sq.sw_sq[idx];
  227. swsqe->flushed = 1;
  228. insert_sq_cqe(wq, cq, swsqe);
  229. if (wq->sq.oldest_read == swsqe) {
  230. advance_oldest_read(wq);
  231. }
  232. flushed++;
  233. if (++idx == wq->sq.size)
  234. idx = 0;
  235. }
  236. wq->sq.flush_cidx += flushed;
  237. if (wq->sq.flush_cidx >= wq->sq.size)
  238. wq->sq.flush_cidx -= wq->sq.size;
  239. return flushed;
  240. }
  241. static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)
  242. {
  243. struct t4_swsqe *swsqe;
  244. int cidx;
  245. if (wq->sq.flush_cidx == -1)
  246. wq->sq.flush_cidx = wq->sq.cidx;
  247. cidx = wq->sq.flush_cidx;
  248. while (cidx != wq->sq.pidx) {
  249. swsqe = &wq->sq.sw_sq[cidx];
  250. if (!swsqe->signaled) {
  251. if (++cidx == wq->sq.size)
  252. cidx = 0;
  253. } else if (swsqe->complete) {
  254. /*
  255. * Insert this completed cqe into the swcq.
  256. */
  257. pr_debug("moving cqe into swcq sq idx %u cq idx %u\n",
  258. cidx, cq->sw_pidx);
  259. swsqe->cqe.header |= htonl(CQE_SWCQE_V(1));
  260. cq->sw_queue[cq->sw_pidx] = swsqe->cqe;
  261. t4_swcq_produce(cq);
  262. swsqe->flushed = 1;
  263. if (++cidx == wq->sq.size)
  264. cidx = 0;
  265. wq->sq.flush_cidx = cidx;
  266. } else
  267. break;
  268. }
  269. }
  270. static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,
  271. struct t4_cqe *read_cqe)
  272. {
  273. read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;
  274. read_cqe->len = htonl(wq->sq.oldest_read->read_len);
  275. read_cqe->header = htonl(CQE_QPID_V(CQE_QPID(hw_cqe)) |
  276. CQE_SWCQE_V(SW_CQE(hw_cqe)) |
  277. CQE_OPCODE_V(FW_RI_READ_REQ) |
  278. CQE_TYPE_V(1));
  279. read_cqe->bits_type_ts = hw_cqe->bits_type_ts;
  280. }
  281. static void advance_oldest_read(struct t4_wq *wq)
  282. {
  283. u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;
  284. if (rptr == wq->sq.size)
  285. rptr = 0;
  286. while (rptr != wq->sq.pidx) {
  287. wq->sq.oldest_read = &wq->sq.sw_sq[rptr];
  288. if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)
  289. return;
  290. if (++rptr == wq->sq.size)
  291. rptr = 0;
  292. }
  293. wq->sq.oldest_read = NULL;
  294. }
  295. /*
  296. * Move all CQEs from the HWCQ into the SWCQ.
  297. * Deal with out-of-order and/or completions that complete
  298. * prior unsignalled WRs.
  299. */
  300. void c4iw_flush_hw_cq(struct c4iw_cq *chp, struct c4iw_qp *flush_qhp)
  301. {
  302. struct t4_cqe *hw_cqe, *swcqe, read_cqe;
  303. struct c4iw_qp *qhp;
  304. struct t4_swsqe *swsqe;
  305. int ret;
  306. pr_debug("cqid 0x%x\n", chp->cq.cqid);
  307. ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);
  308. /*
  309. * This logic is similar to poll_cq(), but not quite the same
  310. * unfortunately. Need to move pertinent HW CQEs to the SW CQ but
  311. * also do any translation magic that poll_cq() normally does.
  312. */
  313. while (!ret) {
  314. qhp = get_qhp(chp->rhp, CQE_QPID(hw_cqe));
  315. /*
  316. * drop CQEs with no associated QP
  317. */
  318. if (qhp == NULL)
  319. goto next_cqe;
  320. if (flush_qhp != qhp) {
  321. spin_lock(&qhp->lock);
  322. if (qhp->wq.flushed == 1)
  323. goto next_cqe;
  324. }
  325. if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE)
  326. goto next_cqe;
  327. if (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP) {
  328. /* If we have reached here because of async
  329. * event or other error, and have egress error
  330. * then drop
  331. */
  332. if (CQE_TYPE(hw_cqe) == 1)
  333. goto next_cqe;
  334. /* drop peer2peer RTR reads.
  335. */
  336. if (CQE_WRID_STAG(hw_cqe) == 1)
  337. goto next_cqe;
  338. /*
  339. * Eat completions for unsignaled read WRs.
  340. */
  341. if (!qhp->wq.sq.oldest_read->signaled) {
  342. advance_oldest_read(&qhp->wq);
  343. goto next_cqe;
  344. }
  345. /*
  346. * Don't write to the HWCQ, create a new read req CQE
  347. * in local memory and move it into the swcq.
  348. */
  349. create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe);
  350. hw_cqe = &read_cqe;
  351. advance_oldest_read(&qhp->wq);
  352. }
  353. /* if its a SQ completion, then do the magic to move all the
  354. * unsignaled and now in-order completions into the swcq.
  355. */
  356. if (SQ_TYPE(hw_cqe)) {
  357. swsqe = &qhp->wq.sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];
  358. swsqe->cqe = *hw_cqe;
  359. swsqe->complete = 1;
  360. flush_completed_wrs(&qhp->wq, &chp->cq);
  361. } else {
  362. swcqe = &chp->cq.sw_queue[chp->cq.sw_pidx];
  363. *swcqe = *hw_cqe;
  364. swcqe->header |= cpu_to_be32(CQE_SWCQE_V(1));
  365. t4_swcq_produce(&chp->cq);
  366. }
  367. next_cqe:
  368. t4_hwcq_consume(&chp->cq);
  369. ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);
  370. if (qhp && flush_qhp != qhp)
  371. spin_unlock(&qhp->lock);
  372. }
  373. }
  374. static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)
  375. {
  376. if (DRAIN_CQE(cqe)) {
  377. WARN_ONCE(1, "Unexpected DRAIN CQE qp id %u!\n", wq->sq.qid);
  378. return 0;
  379. }
  380. if (CQE_OPCODE(cqe) == FW_RI_TERMINATE)
  381. return 0;
  382. if ((CQE_OPCODE(cqe) == FW_RI_RDMA_WRITE) && RQ_TYPE(cqe))
  383. return 0;
  384. if ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) && SQ_TYPE(cqe))
  385. return 0;
  386. if (CQE_SEND_OPCODE(cqe) && RQ_TYPE(cqe) && t4_rq_empty(wq))
  387. return 0;
  388. return 1;
  389. }
  390. void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
  391. {
  392. struct t4_cqe *cqe;
  393. u32 ptr;
  394. *count = 0;
  395. pr_debug("count zero %d\n", *count);
  396. ptr = cq->sw_cidx;
  397. while (ptr != cq->sw_pidx) {
  398. cqe = &cq->sw_queue[ptr];
  399. if (RQ_TYPE(cqe) && (CQE_OPCODE(cqe) != FW_RI_READ_RESP) &&
  400. (CQE_QPID(cqe) == wq->sq.qid) && cqe_completes_wr(cqe, wq))
  401. (*count)++;
  402. if (++ptr == cq->size)
  403. ptr = 0;
  404. }
  405. pr_debug("cq %p count %d\n", cq, *count);
  406. }
  407. static void post_pending_srq_wrs(struct t4_srq *srq)
  408. {
  409. struct t4_srq_pending_wr *pwr;
  410. u16 idx = 0;
  411. while (srq->pending_in_use) {
  412. pwr = &srq->pending_wrs[srq->pending_cidx];
  413. srq->sw_rq[srq->pidx].wr_id = pwr->wr_id;
  414. srq->sw_rq[srq->pidx].valid = 1;
  415. pr_debug("%s posting pending cidx %u pidx %u wq_pidx %u in_use %u rq_size %u wr_id %llx\n",
  416. __func__,
  417. srq->cidx, srq->pidx, srq->wq_pidx,
  418. srq->in_use, srq->size,
  419. (unsigned long long)pwr->wr_id);
  420. c4iw_copy_wr_to_srq(srq, &pwr->wqe, pwr->len16);
  421. t4_srq_consume_pending_wr(srq);
  422. t4_srq_produce(srq, pwr->len16);
  423. idx += DIV_ROUND_UP(pwr->len16 * 16, T4_EQ_ENTRY_SIZE);
  424. }
  425. if (idx) {
  426. t4_ring_srq_db(srq, idx, pwr->len16, &pwr->wqe);
  427. srq->queue[srq->size].status.host_wq_pidx =
  428. srq->wq_pidx;
  429. }
  430. }
  431. static u64 reap_srq_cqe(struct t4_cqe *hw_cqe, struct t4_srq *srq)
  432. {
  433. int rel_idx = CQE_ABS_RQE_IDX(hw_cqe) - srq->rqt_abs_idx;
  434. u64 wr_id;
  435. srq->sw_rq[rel_idx].valid = 0;
  436. wr_id = srq->sw_rq[rel_idx].wr_id;
  437. if (rel_idx == srq->cidx) {
  438. pr_debug("%s in order cqe rel_idx %u cidx %u pidx %u wq_pidx %u in_use %u rq_size %u wr_id %llx\n",
  439. __func__, rel_idx, srq->cidx, srq->pidx,
  440. srq->wq_pidx, srq->in_use, srq->size,
  441. (unsigned long long)srq->sw_rq[rel_idx].wr_id);
  442. t4_srq_consume(srq);
  443. while (srq->ooo_count && !srq->sw_rq[srq->cidx].valid) {
  444. pr_debug("%s eat ooo cidx %u pidx %u wq_pidx %u in_use %u rq_size %u ooo_count %u wr_id %llx\n",
  445. __func__, srq->cidx, srq->pidx,
  446. srq->wq_pidx, srq->in_use,
  447. srq->size, srq->ooo_count,
  448. (unsigned long long)
  449. srq->sw_rq[srq->cidx].wr_id);
  450. t4_srq_consume_ooo(srq);
  451. }
  452. if (srq->ooo_count == 0 && srq->pending_in_use)
  453. post_pending_srq_wrs(srq);
  454. } else {
  455. pr_debug("%s ooo cqe rel_idx %u cidx %u pidx %u wq_pidx %u in_use %u rq_size %u ooo_count %u wr_id %llx\n",
  456. __func__, rel_idx, srq->cidx,
  457. srq->pidx, srq->wq_pidx,
  458. srq->in_use, srq->size,
  459. srq->ooo_count,
  460. (unsigned long long)srq->sw_rq[rel_idx].wr_id);
  461. t4_srq_produce_ooo(srq);
  462. }
  463. return wr_id;
  464. }
  465. /*
  466. * poll_cq
  467. *
  468. * Caller must:
  469. * check the validity of the first CQE,
  470. * supply the wq assicated with the qpid.
  471. *
  472. * credit: cq credit to return to sge.
  473. * cqe_flushed: 1 iff the CQE is flushed.
  474. * cqe: copy of the polled CQE.
  475. *
  476. * return value:
  477. * 0 CQE returned ok.
  478. * -EAGAIN CQE skipped, try again.
  479. * -EOVERFLOW CQ overflow detected.
  480. */
  481. static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
  482. u8 *cqe_flushed, u64 *cookie, u32 *credit,
  483. struct t4_srq *srq)
  484. {
  485. int ret = 0;
  486. struct t4_cqe *hw_cqe, read_cqe;
  487. *cqe_flushed = 0;
  488. *credit = 0;
  489. ret = t4_next_cqe(cq, &hw_cqe);
  490. if (ret)
  491. return ret;
  492. pr_debug("CQE OVF %u qpid 0x%0x genbit %u type %u status 0x%0x opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x\n",
  493. CQE_OVFBIT(hw_cqe), CQE_QPID(hw_cqe),
  494. CQE_GENBIT(hw_cqe), CQE_TYPE(hw_cqe), CQE_STATUS(hw_cqe),
  495. CQE_OPCODE(hw_cqe), CQE_LEN(hw_cqe), CQE_WRID_HI(hw_cqe),
  496. CQE_WRID_LOW(hw_cqe));
  497. /*
  498. * skip cqe's not affiliated with a QP.
  499. */
  500. if (wq == NULL) {
  501. ret = -EAGAIN;
  502. goto skip_cqe;
  503. }
  504. /*
  505. * skip hw cqe's if the wq is flushed.
  506. */
  507. if (wq->flushed && !SW_CQE(hw_cqe)) {
  508. ret = -EAGAIN;
  509. goto skip_cqe;
  510. }
  511. /*
  512. * skip TERMINATE cqes...
  513. */
  514. if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) {
  515. ret = -EAGAIN;
  516. goto skip_cqe;
  517. }
  518. /*
  519. * Special cqe for drain WR completions...
  520. */
  521. if (DRAIN_CQE(hw_cqe)) {
  522. *cookie = CQE_DRAIN_COOKIE(hw_cqe);
  523. *cqe = *hw_cqe;
  524. goto skip_cqe;
  525. }
  526. /*
  527. * Gotta tweak READ completions:
  528. * 1) the cqe doesn't contain the sq_wptr from the wr.
  529. * 2) opcode not reflected from the wr.
  530. * 3) read_len not reflected from the wr.
  531. * 4) cq_type is RQ_TYPE not SQ_TYPE.
  532. */
  533. if (RQ_TYPE(hw_cqe) && (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP)) {
  534. /* If we have reached here because of async
  535. * event or other error, and have egress error
  536. * then drop
  537. */
  538. if (CQE_TYPE(hw_cqe) == 1) {
  539. if (CQE_STATUS(hw_cqe))
  540. t4_set_wq_in_error(wq, 0);
  541. ret = -EAGAIN;
  542. goto skip_cqe;
  543. }
  544. /* If this is an unsolicited read response, then the read
  545. * was generated by the kernel driver as part of peer-2-peer
  546. * connection setup. So ignore the completion.
  547. */
  548. if (CQE_WRID_STAG(hw_cqe) == 1) {
  549. if (CQE_STATUS(hw_cqe))
  550. t4_set_wq_in_error(wq, 0);
  551. ret = -EAGAIN;
  552. goto skip_cqe;
  553. }
  554. /*
  555. * Eat completions for unsignaled read WRs.
  556. */
  557. if (!wq->sq.oldest_read->signaled) {
  558. advance_oldest_read(wq);
  559. ret = -EAGAIN;
  560. goto skip_cqe;
  561. }
  562. /*
  563. * Don't write to the HWCQ, so create a new read req CQE
  564. * in local memory.
  565. */
  566. create_read_req_cqe(wq, hw_cqe, &read_cqe);
  567. hw_cqe = &read_cqe;
  568. advance_oldest_read(wq);
  569. }
  570. if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) {
  571. *cqe_flushed = (CQE_STATUS(hw_cqe) == T4_ERR_SWFLUSH);
  572. t4_set_wq_in_error(wq, 0);
  573. }
  574. /*
  575. * RECV completion.
  576. */
  577. if (RQ_TYPE(hw_cqe)) {
  578. /*
  579. * HW only validates 4 bits of MSN. So we must validate that
  580. * the MSN in the SEND is the next expected MSN. If its not,
  581. * then we complete this with T4_ERR_MSN and mark the wq in
  582. * error.
  583. */
  584. if (unlikely(!CQE_STATUS(hw_cqe) &&
  585. CQE_WRID_MSN(hw_cqe) != wq->rq.msn)) {
  586. t4_set_wq_in_error(wq, 0);
  587. hw_cqe->header |= cpu_to_be32(CQE_STATUS_V(T4_ERR_MSN));
  588. }
  589. goto proc_cqe;
  590. }
  591. /*
  592. * If we get here its a send completion.
  593. *
  594. * Handle out of order completion. These get stuffed
  595. * in the SW SQ. Then the SW SQ is walked to move any
  596. * now in-order completions into the SW CQ. This handles
  597. * 2 cases:
  598. * 1) reaping unsignaled WRs when the first subsequent
  599. * signaled WR is completed.
  600. * 2) out of order read completions.
  601. */
  602. if (!SW_CQE(hw_cqe) && (CQE_WRID_SQ_IDX(hw_cqe) != wq->sq.cidx)) {
  603. struct t4_swsqe *swsqe;
  604. pr_debug("out of order completion going in sw_sq at idx %u\n",
  605. CQE_WRID_SQ_IDX(hw_cqe));
  606. swsqe = &wq->sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];
  607. swsqe->cqe = *hw_cqe;
  608. swsqe->complete = 1;
  609. ret = -EAGAIN;
  610. goto flush_wq;
  611. }
  612. proc_cqe:
  613. *cqe = *hw_cqe;
  614. /*
  615. * Reap the associated WR(s) that are freed up with this
  616. * completion.
  617. */
  618. if (SQ_TYPE(hw_cqe)) {
  619. int idx = CQE_WRID_SQ_IDX(hw_cqe);
  620. /*
  621. * Account for any unsignaled completions completed by
  622. * this signaled completion. In this case, cidx points
  623. * to the first unsignaled one, and idx points to the
  624. * signaled one. So adjust in_use based on this delta.
  625. * if this is not completing any unsigned wrs, then the
  626. * delta will be 0. Handle wrapping also!
  627. */
  628. if (idx < wq->sq.cidx)
  629. wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx;
  630. else
  631. wq->sq.in_use -= idx - wq->sq.cidx;
  632. wq->sq.cidx = (uint16_t)idx;
  633. pr_debug("completing sq idx %u\n", wq->sq.cidx);
  634. *cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id;
  635. if (c4iw_wr_log)
  636. c4iw_log_wr_stats(wq, hw_cqe);
  637. t4_sq_consume(wq);
  638. } else {
  639. if (!srq) {
  640. pr_debug("completing rq idx %u\n", wq->rq.cidx);
  641. *cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id;
  642. if (c4iw_wr_log)
  643. c4iw_log_wr_stats(wq, hw_cqe);
  644. t4_rq_consume(wq);
  645. } else {
  646. *cookie = reap_srq_cqe(hw_cqe, srq);
  647. }
  648. wq->rq.msn++;
  649. goto skip_cqe;
  650. }
  651. flush_wq:
  652. /*
  653. * Flush any completed cqes that are now in-order.
  654. */
  655. flush_completed_wrs(wq, cq);
  656. skip_cqe:
  657. if (SW_CQE(hw_cqe)) {
  658. pr_debug("cq %p cqid 0x%x skip sw cqe cidx %u\n",
  659. cq, cq->cqid, cq->sw_cidx);
  660. t4_swcq_consume(cq);
  661. } else {
  662. pr_debug("cq %p cqid 0x%x skip hw cqe cidx %u\n",
  663. cq, cq->cqid, cq->cidx);
  664. t4_hwcq_consume(cq);
  665. }
  666. return ret;
  667. }
  668. static int __c4iw_poll_cq_one(struct c4iw_cq *chp, struct c4iw_qp *qhp,
  669. struct ib_wc *wc, struct c4iw_srq *srq)
  670. {
  671. struct t4_cqe uninitialized_var(cqe);
  672. struct t4_wq *wq = qhp ? &qhp->wq : NULL;
  673. u32 credit = 0;
  674. u8 cqe_flushed;
  675. u64 cookie = 0;
  676. int ret;
  677. ret = poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, &credit,
  678. srq ? &srq->wq : NULL);
  679. if (ret)
  680. goto out;
  681. wc->wr_id = cookie;
  682. wc->qp = qhp ? &qhp->ibqp : NULL;
  683. wc->vendor_err = CQE_STATUS(&cqe);
  684. wc->wc_flags = 0;
  685. /*
  686. * Simulate a SRQ_LIMIT_REACHED HW notification if required.
  687. */
  688. if (srq && !(srq->flags & T4_SRQ_LIMIT_SUPPORT) && srq->armed &&
  689. srq->wq.in_use < srq->srq_limit)
  690. c4iw_dispatch_srq_limit_reached_event(srq);
  691. pr_debug("qpid 0x%x type %d opcode %d status 0x%x len %u wrid hi 0x%x lo 0x%x cookie 0x%llx\n",
  692. CQE_QPID(&cqe),
  693. CQE_TYPE(&cqe), CQE_OPCODE(&cqe),
  694. CQE_STATUS(&cqe), CQE_LEN(&cqe),
  695. CQE_WRID_HI(&cqe), CQE_WRID_LOW(&cqe),
  696. (unsigned long long)cookie);
  697. if (CQE_TYPE(&cqe) == 0) {
  698. if (!CQE_STATUS(&cqe))
  699. wc->byte_len = CQE_LEN(&cqe);
  700. else
  701. wc->byte_len = 0;
  702. switch (CQE_OPCODE(&cqe)) {
  703. case FW_RI_SEND:
  704. wc->opcode = IB_WC_RECV;
  705. break;
  706. case FW_RI_SEND_WITH_INV:
  707. case FW_RI_SEND_WITH_SE_INV:
  708. wc->opcode = IB_WC_RECV;
  709. wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe);
  710. wc->wc_flags |= IB_WC_WITH_INVALIDATE;
  711. c4iw_invalidate_mr(qhp->rhp, wc->ex.invalidate_rkey);
  712. break;
  713. case FW_RI_WRITE_IMMEDIATE:
  714. wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
  715. wc->ex.imm_data = CQE_IMM_DATA(&cqe);
  716. wc->wc_flags |= IB_WC_WITH_IMM;
  717. break;
  718. default:
  719. pr_err("Unexpected opcode %d in the CQE received for QPID=0x%0x\n",
  720. CQE_OPCODE(&cqe), CQE_QPID(&cqe));
  721. ret = -EINVAL;
  722. goto out;
  723. }
  724. } else {
  725. switch (CQE_OPCODE(&cqe)) {
  726. case FW_RI_WRITE_IMMEDIATE:
  727. case FW_RI_RDMA_WRITE:
  728. wc->opcode = IB_WC_RDMA_WRITE;
  729. break;
  730. case FW_RI_READ_REQ:
  731. wc->opcode = IB_WC_RDMA_READ;
  732. wc->byte_len = CQE_LEN(&cqe);
  733. break;
  734. case FW_RI_SEND_WITH_INV:
  735. case FW_RI_SEND_WITH_SE_INV:
  736. wc->opcode = IB_WC_SEND;
  737. wc->wc_flags |= IB_WC_WITH_INVALIDATE;
  738. break;
  739. case FW_RI_SEND:
  740. case FW_RI_SEND_WITH_SE:
  741. wc->opcode = IB_WC_SEND;
  742. break;
  743. case FW_RI_LOCAL_INV:
  744. wc->opcode = IB_WC_LOCAL_INV;
  745. break;
  746. case FW_RI_FAST_REGISTER:
  747. wc->opcode = IB_WC_REG_MR;
  748. /* Invalidate the MR if the fastreg failed */
  749. if (CQE_STATUS(&cqe) != T4_ERR_SUCCESS)
  750. c4iw_invalidate_mr(qhp->rhp,
  751. CQE_WRID_FR_STAG(&cqe));
  752. break;
  753. default:
  754. pr_err("Unexpected opcode %d in the CQE received for QPID=0x%0x\n",
  755. CQE_OPCODE(&cqe), CQE_QPID(&cqe));
  756. ret = -EINVAL;
  757. goto out;
  758. }
  759. }
  760. if (cqe_flushed)
  761. wc->status = IB_WC_WR_FLUSH_ERR;
  762. else {
  763. switch (CQE_STATUS(&cqe)) {
  764. case T4_ERR_SUCCESS:
  765. wc->status = IB_WC_SUCCESS;
  766. break;
  767. case T4_ERR_STAG:
  768. wc->status = IB_WC_LOC_ACCESS_ERR;
  769. break;
  770. case T4_ERR_PDID:
  771. wc->status = IB_WC_LOC_PROT_ERR;
  772. break;
  773. case T4_ERR_QPID:
  774. case T4_ERR_ACCESS:
  775. wc->status = IB_WC_LOC_ACCESS_ERR;
  776. break;
  777. case T4_ERR_WRAP:
  778. wc->status = IB_WC_GENERAL_ERR;
  779. break;
  780. case T4_ERR_BOUND:
  781. wc->status = IB_WC_LOC_LEN_ERR;
  782. break;
  783. case T4_ERR_INVALIDATE_SHARED_MR:
  784. case T4_ERR_INVALIDATE_MR_WITH_MW_BOUND:
  785. wc->status = IB_WC_MW_BIND_ERR;
  786. break;
  787. case T4_ERR_CRC:
  788. case T4_ERR_MARKER:
  789. case T4_ERR_PDU_LEN_ERR:
  790. case T4_ERR_OUT_OF_RQE:
  791. case T4_ERR_DDP_VERSION:
  792. case T4_ERR_RDMA_VERSION:
  793. case T4_ERR_DDP_QUEUE_NUM:
  794. case T4_ERR_MSN:
  795. case T4_ERR_TBIT:
  796. case T4_ERR_MO:
  797. case T4_ERR_MSN_RANGE:
  798. case T4_ERR_IRD_OVERFLOW:
  799. case T4_ERR_OPCODE:
  800. case T4_ERR_INTERNAL_ERR:
  801. wc->status = IB_WC_FATAL_ERR;
  802. break;
  803. case T4_ERR_SWFLUSH:
  804. wc->status = IB_WC_WR_FLUSH_ERR;
  805. break;
  806. default:
  807. pr_err("Unexpected cqe_status 0x%x for QPID=0x%0x\n",
  808. CQE_STATUS(&cqe), CQE_QPID(&cqe));
  809. wc->status = IB_WC_FATAL_ERR;
  810. }
  811. }
  812. out:
  813. return ret;
  814. }
  815. /*
  816. * Get one cq entry from c4iw and map it to openib.
  817. *
  818. * Returns:
  819. * 0 cqe returned
  820. * -ENODATA EMPTY;
  821. * -EAGAIN caller must try again
  822. * any other -errno fatal error
  823. */
  824. static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
  825. {
  826. struct c4iw_srq *srq = NULL;
  827. struct c4iw_qp *qhp = NULL;
  828. struct t4_cqe *rd_cqe;
  829. int ret;
  830. ret = t4_next_cqe(&chp->cq, &rd_cqe);
  831. if (ret)
  832. return ret;
  833. qhp = get_qhp(chp->rhp, CQE_QPID(rd_cqe));
  834. if (qhp) {
  835. spin_lock(&qhp->lock);
  836. srq = qhp->srq;
  837. if (srq)
  838. spin_lock(&srq->lock);
  839. ret = __c4iw_poll_cq_one(chp, qhp, wc, srq);
  840. spin_unlock(&qhp->lock);
  841. if (srq)
  842. spin_unlock(&srq->lock);
  843. } else {
  844. ret = __c4iw_poll_cq_one(chp, NULL, wc, NULL);
  845. }
  846. return ret;
  847. }
  848. int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
  849. {
  850. struct c4iw_cq *chp;
  851. unsigned long flags;
  852. int npolled;
  853. int err = 0;
  854. chp = to_c4iw_cq(ibcq);
  855. spin_lock_irqsave(&chp->lock, flags);
  856. for (npolled = 0; npolled < num_entries; ++npolled) {
  857. do {
  858. err = c4iw_poll_cq_one(chp, wc + npolled);
  859. } while (err == -EAGAIN);
  860. if (err)
  861. break;
  862. }
  863. spin_unlock_irqrestore(&chp->lock, flags);
  864. return !err || err == -ENODATA ? npolled : err;
  865. }
  866. int c4iw_destroy_cq(struct ib_cq *ib_cq)
  867. {
  868. struct c4iw_cq *chp;
  869. struct c4iw_ucontext *ucontext;
  870. pr_debug("ib_cq %p\n", ib_cq);
  871. chp = to_c4iw_cq(ib_cq);
  872. remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
  873. atomic_dec(&chp->refcnt);
  874. wait_event(chp->wait, !atomic_read(&chp->refcnt));
  875. ucontext = ib_cq->uobject ? to_c4iw_ucontext(ib_cq->uobject->context)
  876. : NULL;
  877. destroy_cq(&chp->rhp->rdev, &chp->cq,
  878. ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx,
  879. chp->destroy_skb, chp->wr_waitp);
  880. c4iw_put_wr_wait(chp->wr_waitp);
  881. kfree(chp);
  882. return 0;
  883. }
  884. struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
  885. const struct ib_cq_init_attr *attr,
  886. struct ib_ucontext *ib_context,
  887. struct ib_udata *udata)
  888. {
  889. int entries = attr->cqe;
  890. int vector = attr->comp_vector;
  891. struct c4iw_dev *rhp;
  892. struct c4iw_cq *chp;
  893. struct c4iw_create_cq ucmd;
  894. struct c4iw_create_cq_resp uresp;
  895. struct c4iw_ucontext *ucontext = NULL;
  896. int ret, wr_len;
  897. size_t memsize, hwentries;
  898. struct c4iw_mm_entry *mm, *mm2;
  899. pr_debug("ib_dev %p entries %d\n", ibdev, entries);
  900. if (attr->flags)
  901. return ERR_PTR(-EINVAL);
  902. rhp = to_c4iw_dev(ibdev);
  903. if (entries < 1 || entries > ibdev->attrs.max_cqe)
  904. return ERR_PTR(-EINVAL);
  905. if (vector >= rhp->rdev.lldi.nciq)
  906. return ERR_PTR(-EINVAL);
  907. if (ib_context) {
  908. ucontext = to_c4iw_ucontext(ib_context);
  909. if (udata->inlen < sizeof(ucmd))
  910. ucontext->is_32b_cqe = 1;
  911. }
  912. chp = kzalloc(sizeof(*chp), GFP_KERNEL);
  913. if (!chp)
  914. return ERR_PTR(-ENOMEM);
  915. chp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);
  916. if (!chp->wr_waitp) {
  917. ret = -ENOMEM;
  918. goto err_free_chp;
  919. }
  920. c4iw_init_wr_wait(chp->wr_waitp);
  921. wr_len = sizeof(struct fw_ri_res_wr) + sizeof(struct fw_ri_res);
  922. chp->destroy_skb = alloc_skb(wr_len, GFP_KERNEL);
  923. if (!chp->destroy_skb) {
  924. ret = -ENOMEM;
  925. goto err_free_wr_wait;
  926. }
  927. /* account for the status page. */
  928. entries++;
  929. /* IQ needs one extra entry to differentiate full vs empty. */
  930. entries++;
  931. /*
  932. * entries must be multiple of 16 for HW.
  933. */
  934. entries = roundup(entries, 16);
  935. /*
  936. * Make actual HW queue 2x to avoid cdix_inc overflows.
  937. */
  938. hwentries = min(entries * 2, rhp->rdev.hw_queue.t4_max_iq_size);
  939. /*
  940. * Make HW queue at least 64 entries so GTS updates aren't too
  941. * frequent.
  942. */
  943. if (hwentries < 64)
  944. hwentries = 64;
  945. memsize = hwentries * ((ucontext && ucontext->is_32b_cqe) ?
  946. (sizeof(*chp->cq.queue) / 2) : sizeof(*chp->cq.queue));
  947. /*
  948. * memsize must be a multiple of the page size if its a user cq.
  949. */
  950. if (ucontext)
  951. memsize = roundup(memsize, PAGE_SIZE);
  952. chp->cq.size = hwentries;
  953. chp->cq.memsize = memsize;
  954. chp->cq.vector = vector;
  955. ret = create_cq(&rhp->rdev, &chp->cq,
  956. ucontext ? &ucontext->uctx : &rhp->rdev.uctx,
  957. chp->wr_waitp);
  958. if (ret)
  959. goto err_free_skb;
  960. chp->rhp = rhp;
  961. chp->cq.size--; /* status page */
  962. chp->ibcq.cqe = entries - 2;
  963. spin_lock_init(&chp->lock);
  964. spin_lock_init(&chp->comp_handler_lock);
  965. atomic_set(&chp->refcnt, 1);
  966. init_waitqueue_head(&chp->wait);
  967. ret = insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid);
  968. if (ret)
  969. goto err_destroy_cq;
  970. if (ucontext) {
  971. ret = -ENOMEM;
  972. mm = kmalloc(sizeof *mm, GFP_KERNEL);
  973. if (!mm)
  974. goto err_remove_handle;
  975. mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
  976. if (!mm2)
  977. goto err_free_mm;
  978. memset(&uresp, 0, sizeof(uresp));
  979. uresp.qid_mask = rhp->rdev.cqmask;
  980. uresp.cqid = chp->cq.cqid;
  981. uresp.size = chp->cq.size;
  982. uresp.memsize = chp->cq.memsize;
  983. spin_lock(&ucontext->mmap_lock);
  984. uresp.key = ucontext->key;
  985. ucontext->key += PAGE_SIZE;
  986. uresp.gts_key = ucontext->key;
  987. ucontext->key += PAGE_SIZE;
  988. /* communicate to the userspace that
  989. * kernel driver supports 64B CQE
  990. */
  991. uresp.flags |= C4IW_64B_CQE;
  992. spin_unlock(&ucontext->mmap_lock);
  993. ret = ib_copy_to_udata(udata, &uresp,
  994. ucontext->is_32b_cqe ?
  995. sizeof(uresp) - sizeof(uresp.flags) :
  996. sizeof(uresp));
  997. if (ret)
  998. goto err_free_mm2;
  999. mm->key = uresp.key;
  1000. mm->addr = virt_to_phys(chp->cq.queue);
  1001. mm->len = chp->cq.memsize;
  1002. insert_mmap(ucontext, mm);
  1003. mm2->key = uresp.gts_key;
  1004. mm2->addr = chp->cq.bar2_pa;
  1005. mm2->len = PAGE_SIZE;
  1006. insert_mmap(ucontext, mm2);
  1007. }
  1008. pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx\n",
  1009. chp->cq.cqid, chp, chp->cq.size,
  1010. chp->cq.memsize, (unsigned long long)chp->cq.dma_addr);
  1011. return &chp->ibcq;
  1012. err_free_mm2:
  1013. kfree(mm2);
  1014. err_free_mm:
  1015. kfree(mm);
  1016. err_remove_handle:
  1017. remove_handle(rhp, &rhp->cqidr, chp->cq.cqid);
  1018. err_destroy_cq:
  1019. destroy_cq(&chp->rhp->rdev, &chp->cq,
  1020. ucontext ? &ucontext->uctx : &rhp->rdev.uctx,
  1021. chp->destroy_skb, chp->wr_waitp);
  1022. err_free_skb:
  1023. kfree_skb(chp->destroy_skb);
  1024. err_free_wr_wait:
  1025. c4iw_put_wr_wait(chp->wr_waitp);
  1026. err_free_chp:
  1027. kfree(chp);
  1028. return ERR_PTR(ret);
  1029. }
  1030. int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
  1031. {
  1032. struct c4iw_cq *chp;
  1033. int ret = 0;
  1034. unsigned long flag;
  1035. chp = to_c4iw_cq(ibcq);
  1036. spin_lock_irqsave(&chp->lock, flag);
  1037. t4_arm_cq(&chp->cq,
  1038. (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);
  1039. if (flags & IB_CQ_REPORT_MISSED_EVENTS)
  1040. ret = t4_cq_notempty(&chp->cq);
  1041. spin_unlock_irqrestore(&chp->lock, flag);
  1042. return ret;
  1043. }
  1044. void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx)
  1045. {
  1046. struct c4iw_cq *rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
  1047. unsigned long flag;
  1048. /* locking heirarchy: cq lock first, then qp lock. */
  1049. spin_lock_irqsave(&rchp->lock, flag);
  1050. spin_lock(&qhp->lock);
  1051. /* create a SRQ RECV CQE for srqidx */
  1052. insert_recv_cqe(&qhp->wq, &rchp->cq, srqidx);
  1053. spin_unlock(&qhp->lock);
  1054. spin_unlock_irqrestore(&rchp->lock, flag);
  1055. }