ib_isert.c 72 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735
  1. /*******************************************************************************
  2. * This file contains iSCSI extentions for RDMA (iSER) Verbs
  3. *
  4. * (c) Copyright 2013 Datera, Inc.
  5. *
  6. * Nicholas A. Bellinger <nab@linux-iscsi.org>
  7. *
  8. * This program is free software; you can redistribute it and/or modify
  9. * it under the terms of the GNU General Public License as published by
  10. * the Free Software Foundation; either version 2 of the License, or
  11. * (at your option) any later version.
  12. *
  13. * This program is distributed in the hope that it will be useful,
  14. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. * GNU General Public License for more details.
  17. ****************************************************************************/
  18. #include <linux/string.h>
  19. #include <linux/module.h>
  20. #include <linux/scatterlist.h>
  21. #include <linux/socket.h>
  22. #include <linux/in.h>
  23. #include <linux/in6.h>
  24. #include <rdma/ib_verbs.h>
  25. #include <rdma/rdma_cm.h>
  26. #include <target/target_core_base.h>
  27. #include <target/target_core_fabric.h>
  28. #include <target/iscsi/iscsi_transport.h>
  29. #include <linux/semaphore.h>
  30. #include "ib_isert.h"
  31. #define ISERT_MAX_CONN 8
  32. #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
  33. #define ISER_MAX_TX_CQ_LEN \
  34. ((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN)
  35. #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
  36. ISERT_MAX_CONN)
  37. static int isert_debug_level;
  38. module_param_named(debug_level, isert_debug_level, int, 0644);
  39. MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");
  40. static DEFINE_MUTEX(device_list_mutex);
  41. static LIST_HEAD(device_list);
  42. static struct workqueue_struct *isert_comp_wq;
  43. static struct workqueue_struct *isert_release_wq;
  44. static int
  45. isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd);
  46. static int
  47. isert_login_post_recv(struct isert_conn *isert_conn);
  48. static int
  49. isert_rdma_accept(struct isert_conn *isert_conn);
  50. struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np);
  51. static void isert_release_work(struct work_struct *work);
  52. static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc);
  53. static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc);
  54. static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc);
  55. static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc);
  56. static inline bool
  57. isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
  58. {
  59. return (conn->pi_support &&
  60. cmd->prot_op != TARGET_PROT_NORMAL);
  61. }
  62. static void
  63. isert_qp_event_callback(struct ib_event *e, void *context)
  64. {
  65. struct isert_conn *isert_conn = context;
  66. isert_err("%s (%d): conn %p\n",
  67. ib_event_msg(e->event), e->event, isert_conn);
  68. switch (e->event) {
  69. case IB_EVENT_COMM_EST:
  70. rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST);
  71. break;
  72. case IB_EVENT_QP_LAST_WQE_REACHED:
  73. isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n");
  74. break;
  75. default:
  76. break;
  77. }
  78. }
  79. static struct isert_comp *
  80. isert_comp_get(struct isert_conn *isert_conn)
  81. {
  82. struct isert_device *device = isert_conn->device;
  83. struct isert_comp *comp;
  84. int i, min = 0;
  85. mutex_lock(&device_list_mutex);
  86. for (i = 0; i < device->comps_used; i++)
  87. if (device->comps[i].active_qps <
  88. device->comps[min].active_qps)
  89. min = i;
  90. comp = &device->comps[min];
  91. comp->active_qps++;
  92. mutex_unlock(&device_list_mutex);
  93. isert_info("conn %p, using comp %p min_index: %d\n",
  94. isert_conn, comp, min);
  95. return comp;
  96. }
  97. static void
  98. isert_comp_put(struct isert_comp *comp)
  99. {
  100. mutex_lock(&device_list_mutex);
  101. comp->active_qps--;
  102. mutex_unlock(&device_list_mutex);
  103. }
  104. static struct ib_qp *
  105. isert_create_qp(struct isert_conn *isert_conn,
  106. struct isert_comp *comp,
  107. struct rdma_cm_id *cma_id)
  108. {
  109. struct isert_device *device = isert_conn->device;
  110. struct ib_qp_init_attr attr;
  111. int ret;
  112. memset(&attr, 0, sizeof(struct ib_qp_init_attr));
  113. attr.event_handler = isert_qp_event_callback;
  114. attr.qp_context = isert_conn;
  115. attr.send_cq = comp->cq;
  116. attr.recv_cq = comp->cq;
  117. attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
  118. attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
  119. attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
  120. attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
  121. attr.cap.max_recv_sge = 1;
  122. attr.sq_sig_type = IB_SIGNAL_REQ_WR;
  123. attr.qp_type = IB_QPT_RC;
  124. if (device->pi_capable)
  125. attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
  126. ret = rdma_create_qp(cma_id, device->pd, &attr);
  127. if (ret) {
  128. isert_err("rdma_create_qp failed for cma_id %d\n", ret);
  129. return ERR_PTR(ret);
  130. }
  131. return cma_id->qp;
  132. }
  133. static int
  134. isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
  135. {
  136. struct isert_comp *comp;
  137. int ret;
  138. comp = isert_comp_get(isert_conn);
  139. isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id);
  140. if (IS_ERR(isert_conn->qp)) {
  141. ret = PTR_ERR(isert_conn->qp);
  142. goto err;
  143. }
  144. return 0;
  145. err:
  146. isert_comp_put(comp);
  147. return ret;
  148. }
  149. static int
  150. isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
  151. {
  152. struct isert_device *device = isert_conn->device;
  153. struct ib_device *ib_dev = device->ib_device;
  154. struct iser_rx_desc *rx_desc;
  155. struct ib_sge *rx_sg;
  156. u64 dma_addr;
  157. int i, j;
  158. isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS,
  159. sizeof(struct iser_rx_desc),
  160. GFP_KERNEL);
  161. if (!isert_conn->rx_descs)
  162. return -ENOMEM;
  163. rx_desc = isert_conn->rx_descs;
  164. for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
  165. dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
  166. ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
  167. if (ib_dma_mapping_error(ib_dev, dma_addr))
  168. goto dma_map_fail;
  169. rx_desc->dma_addr = dma_addr;
  170. rx_sg = &rx_desc->rx_sg;
  171. rx_sg->addr = rx_desc->dma_addr;
  172. rx_sg->length = ISER_RX_PAYLOAD_SIZE;
  173. rx_sg->lkey = device->pd->local_dma_lkey;
  174. rx_desc->rx_cqe.done = isert_recv_done;
  175. }
  176. return 0;
  177. dma_map_fail:
  178. rx_desc = isert_conn->rx_descs;
  179. for (j = 0; j < i; j++, rx_desc++) {
  180. ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
  181. ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
  182. }
  183. kfree(isert_conn->rx_descs);
  184. isert_conn->rx_descs = NULL;
  185. isert_err("conn %p failed to allocate rx descriptors\n", isert_conn);
  186. return -ENOMEM;
  187. }
  188. static void
  189. isert_free_rx_descriptors(struct isert_conn *isert_conn)
  190. {
  191. struct ib_device *ib_dev = isert_conn->device->ib_device;
  192. struct iser_rx_desc *rx_desc;
  193. int i;
  194. if (!isert_conn->rx_descs)
  195. return;
  196. rx_desc = isert_conn->rx_descs;
  197. for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
  198. ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
  199. ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
  200. }
  201. kfree(isert_conn->rx_descs);
  202. isert_conn->rx_descs = NULL;
  203. }
  204. static void
  205. isert_free_comps(struct isert_device *device)
  206. {
  207. int i;
  208. for (i = 0; i < device->comps_used; i++) {
  209. struct isert_comp *comp = &device->comps[i];
  210. if (comp->cq)
  211. ib_free_cq(comp->cq);
  212. }
  213. kfree(device->comps);
  214. }
  215. static int
  216. isert_alloc_comps(struct isert_device *device)
  217. {
  218. int i, max_cqe, ret = 0;
  219. device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(),
  220. device->ib_device->num_comp_vectors));
  221. isert_info("Using %d CQs, %s supports %d vectors support "
  222. "pi_capable %d\n",
  223. device->comps_used, device->ib_device->name,
  224. device->ib_device->num_comp_vectors,
  225. device->pi_capable);
  226. device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
  227. GFP_KERNEL);
  228. if (!device->comps)
  229. return -ENOMEM;
  230. max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe);
  231. for (i = 0; i < device->comps_used; i++) {
  232. struct isert_comp *comp = &device->comps[i];
  233. comp->device = device;
  234. comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i,
  235. IB_POLL_WORKQUEUE);
  236. if (IS_ERR(comp->cq)) {
  237. isert_err("Unable to allocate cq\n");
  238. ret = PTR_ERR(comp->cq);
  239. comp->cq = NULL;
  240. goto out_cq;
  241. }
  242. }
  243. return 0;
  244. out_cq:
  245. isert_free_comps(device);
  246. return ret;
  247. }
  248. static int
  249. isert_create_device_ib_res(struct isert_device *device)
  250. {
  251. struct ib_device *ib_dev = device->ib_device;
  252. int ret;
  253. isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n",
  254. ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
  255. isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
  256. ret = isert_alloc_comps(device);
  257. if (ret)
  258. goto out;
  259. device->pd = ib_alloc_pd(ib_dev, 0);
  260. if (IS_ERR(device->pd)) {
  261. ret = PTR_ERR(device->pd);
  262. isert_err("failed to allocate pd, device %p, ret=%d\n",
  263. device, ret);
  264. goto out_cq;
  265. }
  266. /* Check signature cap */
  267. device->pi_capable = ib_dev->attrs.device_cap_flags &
  268. IB_DEVICE_SIGNATURE_HANDOVER ? true : false;
  269. return 0;
  270. out_cq:
  271. isert_free_comps(device);
  272. out:
  273. if (ret > 0)
  274. ret = -EINVAL;
  275. return ret;
  276. }
  277. static void
  278. isert_free_device_ib_res(struct isert_device *device)
  279. {
  280. isert_info("device %p\n", device);
  281. ib_dealloc_pd(device->pd);
  282. isert_free_comps(device);
  283. }
  284. static void
  285. isert_device_put(struct isert_device *device)
  286. {
  287. mutex_lock(&device_list_mutex);
  288. device->refcount--;
  289. isert_info("device %p refcount %d\n", device, device->refcount);
  290. if (!device->refcount) {
  291. isert_free_device_ib_res(device);
  292. list_del(&device->dev_node);
  293. kfree(device);
  294. }
  295. mutex_unlock(&device_list_mutex);
  296. }
  297. static struct isert_device *
  298. isert_device_get(struct rdma_cm_id *cma_id)
  299. {
  300. struct isert_device *device;
  301. int ret;
  302. mutex_lock(&device_list_mutex);
  303. list_for_each_entry(device, &device_list, dev_node) {
  304. if (device->ib_device->node_guid == cma_id->device->node_guid) {
  305. device->refcount++;
  306. isert_info("Found iser device %p refcount %d\n",
  307. device, device->refcount);
  308. mutex_unlock(&device_list_mutex);
  309. return device;
  310. }
  311. }
  312. device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
  313. if (!device) {
  314. mutex_unlock(&device_list_mutex);
  315. return ERR_PTR(-ENOMEM);
  316. }
  317. INIT_LIST_HEAD(&device->dev_node);
  318. device->ib_device = cma_id->device;
  319. ret = isert_create_device_ib_res(device);
  320. if (ret) {
  321. kfree(device);
  322. mutex_unlock(&device_list_mutex);
  323. return ERR_PTR(ret);
  324. }
  325. device->refcount++;
  326. list_add_tail(&device->dev_node, &device_list);
  327. isert_info("Created a new iser device %p refcount %d\n",
  328. device, device->refcount);
  329. mutex_unlock(&device_list_mutex);
  330. return device;
  331. }
  332. static void
  333. isert_init_conn(struct isert_conn *isert_conn)
  334. {
  335. isert_conn->state = ISER_CONN_INIT;
  336. INIT_LIST_HEAD(&isert_conn->node);
  337. init_completion(&isert_conn->login_comp);
  338. init_completion(&isert_conn->login_req_comp);
  339. init_waitqueue_head(&isert_conn->rem_wait);
  340. kref_init(&isert_conn->kref);
  341. mutex_init(&isert_conn->mutex);
  342. INIT_WORK(&isert_conn->release_work, isert_release_work);
  343. }
  344. static void
  345. isert_free_login_buf(struct isert_conn *isert_conn)
  346. {
  347. struct ib_device *ib_dev = isert_conn->device->ib_device;
  348. ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
  349. ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
  350. kfree(isert_conn->login_rsp_buf);
  351. ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
  352. ISER_RX_PAYLOAD_SIZE,
  353. DMA_FROM_DEVICE);
  354. kfree(isert_conn->login_req_buf);
  355. }
  356. static int
  357. isert_alloc_login_buf(struct isert_conn *isert_conn,
  358. struct ib_device *ib_dev)
  359. {
  360. int ret;
  361. isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf),
  362. GFP_KERNEL);
  363. if (!isert_conn->login_req_buf)
  364. return -ENOMEM;
  365. isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
  366. isert_conn->login_req_buf,
  367. ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
  368. ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
  369. if (ret) {
  370. isert_err("login_req_dma mapping error: %d\n", ret);
  371. isert_conn->login_req_dma = 0;
  372. goto out_free_login_req_buf;
  373. }
  374. isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL);
  375. if (!isert_conn->login_rsp_buf) {
  376. ret = -ENOMEM;
  377. goto out_unmap_login_req_buf;
  378. }
  379. isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
  380. isert_conn->login_rsp_buf,
  381. ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
  382. ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
  383. if (ret) {
  384. isert_err("login_rsp_dma mapping error: %d\n", ret);
  385. isert_conn->login_rsp_dma = 0;
  386. goto out_free_login_rsp_buf;
  387. }
  388. return 0;
  389. out_free_login_rsp_buf:
  390. kfree(isert_conn->login_rsp_buf);
  391. out_unmap_login_req_buf:
  392. ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
  393. ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
  394. out_free_login_req_buf:
  395. kfree(isert_conn->login_req_buf);
  396. return ret;
  397. }
  398. static void
  399. isert_set_nego_params(struct isert_conn *isert_conn,
  400. struct rdma_conn_param *param)
  401. {
  402. struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs;
  403. /* Set max inflight RDMA READ requests */
  404. isert_conn->initiator_depth = min_t(u8, param->initiator_depth,
  405. attr->max_qp_init_rd_atom);
  406. isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
  407. if (param->private_data) {
  408. u8 flags = *(u8 *)param->private_data;
  409. /*
  410. * use remote invalidation if the both initiator
  411. * and the HCA support it
  412. */
  413. isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) &&
  414. (attr->device_cap_flags &
  415. IB_DEVICE_MEM_MGT_EXTENSIONS);
  416. if (isert_conn->snd_w_inv)
  417. isert_info("Using remote invalidation\n");
  418. }
  419. }
  420. static int
  421. isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
  422. {
  423. struct isert_np *isert_np = cma_id->context;
  424. struct iscsi_np *np = isert_np->np;
  425. struct isert_conn *isert_conn;
  426. struct isert_device *device;
  427. int ret = 0;
  428. spin_lock_bh(&np->np_thread_lock);
  429. if (!np->enabled) {
  430. spin_unlock_bh(&np->np_thread_lock);
  431. isert_dbg("iscsi_np is not enabled, reject connect request\n");
  432. return rdma_reject(cma_id, NULL, 0);
  433. }
  434. spin_unlock_bh(&np->np_thread_lock);
  435. isert_dbg("cma_id: %p, portal: %p\n",
  436. cma_id, cma_id->context);
  437. isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
  438. if (!isert_conn)
  439. return -ENOMEM;
  440. isert_init_conn(isert_conn);
  441. isert_conn->cm_id = cma_id;
  442. ret = isert_alloc_login_buf(isert_conn, cma_id->device);
  443. if (ret)
  444. goto out;
  445. device = isert_device_get(cma_id);
  446. if (IS_ERR(device)) {
  447. ret = PTR_ERR(device);
  448. goto out_rsp_dma_map;
  449. }
  450. isert_conn->device = device;
  451. isert_set_nego_params(isert_conn, &event->param.conn);
  452. ret = isert_conn_setup_qp(isert_conn, cma_id);
  453. if (ret)
  454. goto out_conn_dev;
  455. ret = isert_login_post_recv(isert_conn);
  456. if (ret)
  457. goto out_conn_dev;
  458. ret = isert_rdma_accept(isert_conn);
  459. if (ret)
  460. goto out_conn_dev;
  461. mutex_lock(&isert_np->mutex);
  462. list_add_tail(&isert_conn->node, &isert_np->accepted);
  463. mutex_unlock(&isert_np->mutex);
  464. return 0;
  465. out_conn_dev:
  466. isert_device_put(device);
  467. out_rsp_dma_map:
  468. isert_free_login_buf(isert_conn);
  469. out:
  470. kfree(isert_conn);
  471. rdma_reject(cma_id, NULL, 0);
  472. return ret;
  473. }
  474. static void
  475. isert_connect_release(struct isert_conn *isert_conn)
  476. {
  477. struct isert_device *device = isert_conn->device;
  478. isert_dbg("conn %p\n", isert_conn);
  479. BUG_ON(!device);
  480. isert_free_rx_descriptors(isert_conn);
  481. if (isert_conn->cm_id &&
  482. !isert_conn->dev_removed)
  483. rdma_destroy_id(isert_conn->cm_id);
  484. if (isert_conn->qp) {
  485. struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context;
  486. isert_comp_put(comp);
  487. ib_destroy_qp(isert_conn->qp);
  488. }
  489. if (isert_conn->login_req_buf)
  490. isert_free_login_buf(isert_conn);
  491. isert_device_put(device);
  492. if (isert_conn->dev_removed)
  493. wake_up_interruptible(&isert_conn->rem_wait);
  494. else
  495. kfree(isert_conn);
  496. }
  497. static void
  498. isert_connected_handler(struct rdma_cm_id *cma_id)
  499. {
  500. struct isert_conn *isert_conn = cma_id->qp->qp_context;
  501. struct isert_np *isert_np = cma_id->context;
  502. isert_info("conn %p\n", isert_conn);
  503. mutex_lock(&isert_conn->mutex);
  504. isert_conn->state = ISER_CONN_UP;
  505. kref_get(&isert_conn->kref);
  506. mutex_unlock(&isert_conn->mutex);
  507. mutex_lock(&isert_np->mutex);
  508. list_move_tail(&isert_conn->node, &isert_np->pending);
  509. mutex_unlock(&isert_np->mutex);
  510. isert_info("np %p: Allow accept_np to continue\n", isert_np);
  511. up(&isert_np->sem);
  512. }
  513. static void
  514. isert_release_kref(struct kref *kref)
  515. {
  516. struct isert_conn *isert_conn = container_of(kref,
  517. struct isert_conn, kref);
  518. isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm,
  519. current->pid);
  520. isert_connect_release(isert_conn);
  521. }
  522. static void
  523. isert_put_conn(struct isert_conn *isert_conn)
  524. {
  525. kref_put(&isert_conn->kref, isert_release_kref);
  526. }
  527. static void
  528. isert_handle_unbound_conn(struct isert_conn *isert_conn)
  529. {
  530. struct isert_np *isert_np = isert_conn->cm_id->context;
  531. mutex_lock(&isert_np->mutex);
  532. if (!list_empty(&isert_conn->node)) {
  533. /*
  534. * This means iscsi doesn't know this connection
  535. * so schedule a cleanup ourselves
  536. */
  537. list_del_init(&isert_conn->node);
  538. isert_put_conn(isert_conn);
  539. queue_work(isert_release_wq, &isert_conn->release_work);
  540. }
  541. mutex_unlock(&isert_np->mutex);
  542. }
  543. /**
  544. * isert_conn_terminate() - Initiate connection termination
  545. * @isert_conn: isert connection struct
  546. *
  547. * Notes:
  548. * In case the connection state is BOUND, move state
  549. * to TEMINATING and start teardown sequence (rdma_disconnect).
  550. * In case the connection state is UP, complete flush as well.
  551. *
  552. * This routine must be called with mutex held. Thus it is
  553. * safe to call multiple times.
  554. */
  555. static void
  556. isert_conn_terminate(struct isert_conn *isert_conn)
  557. {
  558. int err;
  559. if (isert_conn->state >= ISER_CONN_TERMINATING)
  560. return;
  561. isert_info("Terminating conn %p state %d\n",
  562. isert_conn, isert_conn->state);
  563. isert_conn->state = ISER_CONN_TERMINATING;
  564. err = rdma_disconnect(isert_conn->cm_id);
  565. if (err)
  566. isert_warn("Failed rdma_disconnect isert_conn %p\n",
  567. isert_conn);
  568. }
  569. static int
  570. isert_np_cma_handler(struct isert_np *isert_np,
  571. enum rdma_cm_event_type event)
  572. {
  573. isert_dbg("%s (%d): isert np %p\n",
  574. rdma_event_msg(event), event, isert_np);
  575. switch (event) {
  576. case RDMA_CM_EVENT_DEVICE_REMOVAL:
  577. isert_np->cm_id = NULL;
  578. break;
  579. case RDMA_CM_EVENT_ADDR_CHANGE:
  580. isert_np->cm_id = isert_setup_id(isert_np);
  581. if (IS_ERR(isert_np->cm_id)) {
  582. isert_err("isert np %p setup id failed: %ld\n",
  583. isert_np, PTR_ERR(isert_np->cm_id));
  584. isert_np->cm_id = NULL;
  585. }
  586. break;
  587. default:
  588. isert_err("isert np %p Unexpected event %d\n",
  589. isert_np, event);
  590. }
  591. return -1;
  592. }
  593. static int
  594. isert_disconnected_handler(struct rdma_cm_id *cma_id,
  595. enum rdma_cm_event_type event)
  596. {
  597. struct isert_conn *isert_conn = cma_id->qp->qp_context;
  598. mutex_lock(&isert_conn->mutex);
  599. switch (isert_conn->state) {
  600. case ISER_CONN_TERMINATING:
  601. break;
  602. case ISER_CONN_UP:
  603. isert_conn_terminate(isert_conn);
  604. ib_drain_qp(isert_conn->qp);
  605. isert_handle_unbound_conn(isert_conn);
  606. break;
  607. case ISER_CONN_BOUND:
  608. case ISER_CONN_FULL_FEATURE: /* FALLTHRU */
  609. iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
  610. break;
  611. default:
  612. isert_warn("conn %p terminating in state %d\n",
  613. isert_conn, isert_conn->state);
  614. }
  615. mutex_unlock(&isert_conn->mutex);
  616. return 0;
  617. }
  618. static int
  619. isert_connect_error(struct rdma_cm_id *cma_id)
  620. {
  621. struct isert_conn *isert_conn = cma_id->qp->qp_context;
  622. ib_drain_qp(isert_conn->qp);
  623. list_del_init(&isert_conn->node);
  624. isert_conn->cm_id = NULL;
  625. isert_put_conn(isert_conn);
  626. return -1;
  627. }
  628. static int
  629. isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
  630. {
  631. struct isert_np *isert_np = cma_id->context;
  632. struct isert_conn *isert_conn;
  633. int ret = 0;
  634. isert_info("%s (%d): status %d id %p np %p\n",
  635. rdma_event_msg(event->event), event->event,
  636. event->status, cma_id, cma_id->context);
  637. if (isert_np->cm_id == cma_id)
  638. return isert_np_cma_handler(cma_id->context, event->event);
  639. switch (event->event) {
  640. case RDMA_CM_EVENT_CONNECT_REQUEST:
  641. ret = isert_connect_request(cma_id, event);
  642. if (ret)
  643. isert_err("failed handle connect request %d\n", ret);
  644. break;
  645. case RDMA_CM_EVENT_ESTABLISHED:
  646. isert_connected_handler(cma_id);
  647. break;
  648. case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */
  649. case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */
  650. case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */
  651. ret = isert_disconnected_handler(cma_id, event->event);
  652. break;
  653. case RDMA_CM_EVENT_DEVICE_REMOVAL:
  654. isert_conn = cma_id->qp->qp_context;
  655. isert_conn->dev_removed = true;
  656. isert_disconnected_handler(cma_id, event->event);
  657. wait_event_interruptible(isert_conn->rem_wait,
  658. isert_conn->state == ISER_CONN_DOWN);
  659. kfree(isert_conn);
  660. /*
  661. * return non-zero from the callback to destroy
  662. * the rdma cm id
  663. */
  664. return 1;
  665. case RDMA_CM_EVENT_REJECTED:
  666. isert_info("Connection rejected: %s\n",
  667. rdma_reject_msg(cma_id, event->status));
  668. /* fall through */
  669. case RDMA_CM_EVENT_UNREACHABLE:
  670. case RDMA_CM_EVENT_CONNECT_ERROR:
  671. ret = isert_connect_error(cma_id);
  672. break;
  673. default:
  674. isert_err("Unhandled RDMA CMA event: %d\n", event->event);
  675. break;
  676. }
  677. return ret;
  678. }
  679. static int
  680. isert_post_recvm(struct isert_conn *isert_conn, u32 count)
  681. {
  682. struct ib_recv_wr *rx_wr;
  683. int i, ret;
  684. struct iser_rx_desc *rx_desc;
  685. for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
  686. rx_desc = &isert_conn->rx_descs[i];
  687. rx_wr->wr_cqe = &rx_desc->rx_cqe;
  688. rx_wr->sg_list = &rx_desc->rx_sg;
  689. rx_wr->num_sge = 1;
  690. rx_wr->next = rx_wr + 1;
  691. rx_desc->in_use = false;
  692. }
  693. rx_wr--;
  694. rx_wr->next = NULL; /* mark end of work requests list */
  695. ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL);
  696. if (ret)
  697. isert_err("ib_post_recv() failed with ret: %d\n", ret);
  698. return ret;
  699. }
  700. static int
  701. isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc)
  702. {
  703. struct ib_recv_wr rx_wr;
  704. int ret;
  705. if (!rx_desc->in_use) {
  706. /*
  707. * if the descriptor is not in-use we already reposted it
  708. * for recv, so just silently return
  709. */
  710. return 0;
  711. }
  712. rx_desc->in_use = false;
  713. rx_wr.wr_cqe = &rx_desc->rx_cqe;
  714. rx_wr.sg_list = &rx_desc->rx_sg;
  715. rx_wr.num_sge = 1;
  716. rx_wr.next = NULL;
  717. ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
  718. if (ret)
  719. isert_err("ib_post_recv() failed with ret: %d\n", ret);
  720. return ret;
  721. }
  722. static int
  723. isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
  724. {
  725. struct ib_device *ib_dev = isert_conn->cm_id->device;
  726. struct ib_send_wr send_wr;
  727. int ret;
  728. ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
  729. ISER_HEADERS_LEN, DMA_TO_DEVICE);
  730. tx_desc->tx_cqe.done = isert_login_send_done;
  731. send_wr.next = NULL;
  732. send_wr.wr_cqe = &tx_desc->tx_cqe;
  733. send_wr.sg_list = tx_desc->tx_sg;
  734. send_wr.num_sge = tx_desc->num_sge;
  735. send_wr.opcode = IB_WR_SEND;
  736. send_wr.send_flags = IB_SEND_SIGNALED;
  737. ret = ib_post_send(isert_conn->qp, &send_wr, NULL);
  738. if (ret)
  739. isert_err("ib_post_send() failed, ret: %d\n", ret);
  740. return ret;
  741. }
  742. static void
  743. __isert_create_send_desc(struct isert_device *device,
  744. struct iser_tx_desc *tx_desc)
  745. {
  746. memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl));
  747. tx_desc->iser_header.flags = ISCSI_CTRL;
  748. tx_desc->num_sge = 1;
  749. if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) {
  750. tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
  751. isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc);
  752. }
  753. }
  754. static void
  755. isert_create_send_desc(struct isert_conn *isert_conn,
  756. struct isert_cmd *isert_cmd,
  757. struct iser_tx_desc *tx_desc)
  758. {
  759. struct isert_device *device = isert_conn->device;
  760. struct ib_device *ib_dev = device->ib_device;
  761. ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
  762. ISER_HEADERS_LEN, DMA_TO_DEVICE);
  763. __isert_create_send_desc(device, tx_desc);
  764. }
  765. static int
  766. isert_init_tx_hdrs(struct isert_conn *isert_conn,
  767. struct iser_tx_desc *tx_desc)
  768. {
  769. struct isert_device *device = isert_conn->device;
  770. struct ib_device *ib_dev = device->ib_device;
  771. u64 dma_addr;
  772. dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
  773. ISER_HEADERS_LEN, DMA_TO_DEVICE);
  774. if (ib_dma_mapping_error(ib_dev, dma_addr)) {
  775. isert_err("ib_dma_mapping_error() failed\n");
  776. return -ENOMEM;
  777. }
  778. tx_desc->dma_addr = dma_addr;
  779. tx_desc->tx_sg[0].addr = tx_desc->dma_addr;
  780. tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
  781. tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
  782. isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n",
  783. tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length,
  784. tx_desc->tx_sg[0].lkey);
  785. return 0;
  786. }
  787. static void
  788. isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
  789. struct ib_send_wr *send_wr)
  790. {
  791. struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;
  792. tx_desc->tx_cqe.done = isert_send_done;
  793. send_wr->wr_cqe = &tx_desc->tx_cqe;
  794. if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) {
  795. send_wr->opcode = IB_WR_SEND_WITH_INV;
  796. send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey;
  797. } else {
  798. send_wr->opcode = IB_WR_SEND;
  799. }
  800. send_wr->sg_list = &tx_desc->tx_sg[0];
  801. send_wr->num_sge = isert_cmd->tx_desc.num_sge;
  802. send_wr->send_flags = IB_SEND_SIGNALED;
  803. }
  804. static int
  805. isert_login_post_recv(struct isert_conn *isert_conn)
  806. {
  807. struct ib_recv_wr rx_wr;
  808. struct ib_sge sge;
  809. int ret;
  810. memset(&sge, 0, sizeof(struct ib_sge));
  811. sge.addr = isert_conn->login_req_dma;
  812. sge.length = ISER_RX_PAYLOAD_SIZE;
  813. sge.lkey = isert_conn->device->pd->local_dma_lkey;
  814. isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n",
  815. sge.addr, sge.length, sge.lkey);
  816. isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done;
  817. memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
  818. rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe;
  819. rx_wr.sg_list = &sge;
  820. rx_wr.num_sge = 1;
  821. ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
  822. if (ret)
  823. isert_err("ib_post_recv() failed: %d\n", ret);
  824. return ret;
  825. }
  826. static int
  827. isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
  828. u32 length)
  829. {
  830. struct isert_conn *isert_conn = conn->context;
  831. struct isert_device *device = isert_conn->device;
  832. struct ib_device *ib_dev = device->ib_device;
  833. struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc;
  834. int ret;
  835. __isert_create_send_desc(device, tx_desc);
  836. memcpy(&tx_desc->iscsi_header, &login->rsp[0],
  837. sizeof(struct iscsi_hdr));
  838. isert_init_tx_hdrs(isert_conn, tx_desc);
  839. if (length > 0) {
  840. struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
  841. ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
  842. length, DMA_TO_DEVICE);
  843. memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
  844. ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
  845. length, DMA_TO_DEVICE);
  846. tx_dsg->addr = isert_conn->login_rsp_dma;
  847. tx_dsg->length = length;
  848. tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey;
  849. tx_desc->num_sge = 2;
  850. }
  851. if (!login->login_failed) {
  852. if (login->login_complete) {
  853. ret = isert_alloc_rx_descriptors(isert_conn);
  854. if (ret)
  855. return ret;
  856. ret = isert_post_recvm(isert_conn,
  857. ISERT_QP_MAX_RECV_DTOS);
  858. if (ret)
  859. return ret;
  860. /* Now we are in FULL_FEATURE phase */
  861. mutex_lock(&isert_conn->mutex);
  862. isert_conn->state = ISER_CONN_FULL_FEATURE;
  863. mutex_unlock(&isert_conn->mutex);
  864. goto post_send;
  865. }
  866. ret = isert_login_post_recv(isert_conn);
  867. if (ret)
  868. return ret;
  869. }
  870. post_send:
  871. ret = isert_login_post_send(isert_conn, tx_desc);
  872. if (ret)
  873. return ret;
  874. return 0;
  875. }
  876. static void
  877. isert_rx_login_req(struct isert_conn *isert_conn)
  878. {
  879. struct iser_rx_desc *rx_desc = isert_conn->login_req_buf;
  880. int rx_buflen = isert_conn->login_req_len;
  881. struct iscsi_conn *conn = isert_conn->conn;
  882. struct iscsi_login *login = conn->conn_login;
  883. int size;
  884. isert_info("conn %p\n", isert_conn);
  885. WARN_ON_ONCE(!login);
  886. if (login->first_request) {
  887. struct iscsi_login_req *login_req =
  888. (struct iscsi_login_req *)&rx_desc->iscsi_header;
  889. /*
  890. * Setup the initial iscsi_login values from the leading
  891. * login request PDU.
  892. */
  893. login->leading_connection = (!login_req->tsih) ? 1 : 0;
  894. login->current_stage =
  895. (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
  896. >> 2;
  897. login->version_min = login_req->min_version;
  898. login->version_max = login_req->max_version;
  899. memcpy(login->isid, login_req->isid, 6);
  900. login->cmd_sn = be32_to_cpu(login_req->cmdsn);
  901. login->init_task_tag = login_req->itt;
  902. login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
  903. login->cid = be16_to_cpu(login_req->cid);
  904. login->tsih = be16_to_cpu(login_req->tsih);
  905. }
  906. memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);
  907. size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
  908. isert_dbg("Using login payload size: %d, rx_buflen: %d "
  909. "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen,
  910. MAX_KEY_VALUE_PAIRS);
  911. memcpy(login->req_buf, &rx_desc->data[0], size);
  912. if (login->first_request) {
  913. complete(&isert_conn->login_comp);
  914. return;
  915. }
  916. schedule_delayed_work(&conn->login_work, 0);
  917. }
  918. static struct iscsi_cmd
  919. *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc)
  920. {
  921. struct isert_conn *isert_conn = conn->context;
  922. struct isert_cmd *isert_cmd;
  923. struct iscsi_cmd *cmd;
  924. cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
  925. if (!cmd) {
  926. isert_err("Unable to allocate iscsi_cmd + isert_cmd\n");
  927. return NULL;
  928. }
  929. isert_cmd = iscsit_priv_cmd(cmd);
  930. isert_cmd->conn = isert_conn;
  931. isert_cmd->iscsi_cmd = cmd;
  932. isert_cmd->rx_desc = rx_desc;
  933. return cmd;
  934. }
  935. static int
  936. isert_handle_scsi_cmd(struct isert_conn *isert_conn,
  937. struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd,
  938. struct iser_rx_desc *rx_desc, unsigned char *buf)
  939. {
  940. struct iscsi_conn *conn = isert_conn->conn;
  941. struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
  942. int imm_data, imm_data_len, unsol_data, sg_nents, rc;
  943. bool dump_payload = false;
  944. unsigned int data_len;
  945. rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
  946. if (rc < 0)
  947. return rc;
  948. imm_data = cmd->immediate_data;
  949. imm_data_len = cmd->first_burst_len;
  950. unsol_data = cmd->unsolicited_data;
  951. data_len = cmd->se_cmd.data_length;
  952. if (imm_data && imm_data_len == data_len)
  953. cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
  954. rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
  955. if (rc < 0) {
  956. return 0;
  957. } else if (rc > 0) {
  958. dump_payload = true;
  959. goto sequence_cmd;
  960. }
  961. if (!imm_data)
  962. return 0;
  963. if (imm_data_len != data_len) {
  964. sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
  965. sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents,
  966. &rx_desc->data[0], imm_data_len);
  967. isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n",
  968. sg_nents, imm_data_len);
  969. } else {
  970. sg_init_table(&isert_cmd->sg, 1);
  971. cmd->se_cmd.t_data_sg = &isert_cmd->sg;
  972. cmd->se_cmd.t_data_nents = 1;
  973. sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len);
  974. isert_dbg("Transfer Immediate imm_data_len: %d\n",
  975. imm_data_len);
  976. }
  977. cmd->write_data_done += imm_data_len;
  978. if (cmd->write_data_done == cmd->se_cmd.data_length) {
  979. spin_lock_bh(&cmd->istate_lock);
  980. cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
  981. cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
  982. spin_unlock_bh(&cmd->istate_lock);
  983. }
  984. sequence_cmd:
  985. rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
  986. if (!rc && dump_payload == false && unsol_data)
  987. iscsit_set_unsoliticed_dataout(cmd);
  988. else if (dump_payload && imm_data)
  989. target_put_sess_cmd(&cmd->se_cmd);
  990. return 0;
  991. }
  992. static int
  993. isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
  994. struct iser_rx_desc *rx_desc, unsigned char *buf)
  995. {
  996. struct scatterlist *sg_start;
  997. struct iscsi_conn *conn = isert_conn->conn;
  998. struct iscsi_cmd *cmd = NULL;
  999. struct iscsi_data *hdr = (struct iscsi_data *)buf;
  1000. u32 unsol_data_len = ntoh24(hdr->dlength);
  1001. int rc, sg_nents, sg_off, page_off;
  1002. rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
  1003. if (rc < 0)
  1004. return rc;
  1005. else if (!cmd)
  1006. return 0;
  1007. /*
  1008. * FIXME: Unexpected unsolicited_data out
  1009. */
  1010. if (!cmd->unsolicited_data) {
  1011. isert_err("Received unexpected solicited data payload\n");
  1012. dump_stack();
  1013. return -1;
  1014. }
  1015. isert_dbg("Unsolicited DataOut unsol_data_len: %u, "
  1016. "write_data_done: %u, data_length: %u\n",
  1017. unsol_data_len, cmd->write_data_done,
  1018. cmd->se_cmd.data_length);
  1019. sg_off = cmd->write_data_done / PAGE_SIZE;
  1020. sg_start = &cmd->se_cmd.t_data_sg[sg_off];
  1021. sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
  1022. page_off = cmd->write_data_done % PAGE_SIZE;
  1023. /*
  1024. * FIXME: Non page-aligned unsolicited_data out
  1025. */
  1026. if (page_off) {
  1027. isert_err("unexpected non-page aligned data payload\n");
  1028. dump_stack();
  1029. return -1;
  1030. }
  1031. isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u "
  1032. "sg_nents: %u from %p %u\n", sg_start, sg_off,
  1033. sg_nents, &rx_desc->data[0], unsol_data_len);
  1034. sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
  1035. unsol_data_len);
  1036. rc = iscsit_check_dataout_payload(cmd, hdr, false);
  1037. if (rc < 0)
  1038. return rc;
  1039. /*
  1040. * multiple data-outs on the same command can arrive -
  1041. * so post the buffer before hand
  1042. */
  1043. rc = isert_post_recv(isert_conn, rx_desc);
  1044. if (rc) {
  1045. isert_err("ib_post_recv failed with %d\n", rc);
  1046. return rc;
  1047. }
  1048. return 0;
  1049. }
  1050. static int
  1051. isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
  1052. struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
  1053. unsigned char *buf)
  1054. {
  1055. struct iscsi_conn *conn = isert_conn->conn;
  1056. struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
  1057. int rc;
  1058. rc = iscsit_setup_nop_out(conn, cmd, hdr);
  1059. if (rc < 0)
  1060. return rc;
  1061. /*
  1062. * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
  1063. */
  1064. return iscsit_process_nop_out(conn, cmd, hdr);
  1065. }
  1066. static int
  1067. isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
  1068. struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
  1069. struct iscsi_text *hdr)
  1070. {
  1071. struct iscsi_conn *conn = isert_conn->conn;
  1072. u32 payload_length = ntoh24(hdr->dlength);
  1073. int rc;
  1074. unsigned char *text_in = NULL;
  1075. rc = iscsit_setup_text_cmd(conn, cmd, hdr);
  1076. if (rc < 0)
  1077. return rc;
  1078. if (payload_length) {
  1079. text_in = kzalloc(payload_length, GFP_KERNEL);
  1080. if (!text_in)
  1081. return -ENOMEM;
  1082. }
  1083. cmd->text_in_ptr = text_in;
  1084. memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length);
  1085. return iscsit_process_text_cmd(conn, cmd, hdr);
  1086. }
  1087. static int
  1088. isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
  1089. uint32_t read_stag, uint64_t read_va,
  1090. uint32_t write_stag, uint64_t write_va)
  1091. {
  1092. struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
  1093. struct iscsi_conn *conn = isert_conn->conn;
  1094. struct iscsi_cmd *cmd;
  1095. struct isert_cmd *isert_cmd;
  1096. int ret = -EINVAL;
  1097. u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
  1098. if (conn->sess->sess_ops->SessionType &&
  1099. (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) {
  1100. isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery,"
  1101. " ignoring\n", opcode);
  1102. return 0;
  1103. }
  1104. switch (opcode) {
  1105. case ISCSI_OP_SCSI_CMD:
  1106. cmd = isert_allocate_cmd(conn, rx_desc);
  1107. if (!cmd)
  1108. break;
  1109. isert_cmd = iscsit_priv_cmd(cmd);
  1110. isert_cmd->read_stag = read_stag;
  1111. isert_cmd->read_va = read_va;
  1112. isert_cmd->write_stag = write_stag;
  1113. isert_cmd->write_va = write_va;
  1114. isert_cmd->inv_rkey = read_stag ? read_stag : write_stag;
  1115. ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,
  1116. rx_desc, (unsigned char *)hdr);
  1117. break;
  1118. case ISCSI_OP_NOOP_OUT:
  1119. cmd = isert_allocate_cmd(conn, rx_desc);
  1120. if (!cmd)
  1121. break;
  1122. isert_cmd = iscsit_priv_cmd(cmd);
  1123. ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,
  1124. rx_desc, (unsigned char *)hdr);
  1125. break;
  1126. case ISCSI_OP_SCSI_DATA_OUT:
  1127. ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
  1128. (unsigned char *)hdr);
  1129. break;
  1130. case ISCSI_OP_SCSI_TMFUNC:
  1131. cmd = isert_allocate_cmd(conn, rx_desc);
  1132. if (!cmd)
  1133. break;
  1134. ret = iscsit_handle_task_mgt_cmd(conn, cmd,
  1135. (unsigned char *)hdr);
  1136. break;
  1137. case ISCSI_OP_LOGOUT:
  1138. cmd = isert_allocate_cmd(conn, rx_desc);
  1139. if (!cmd)
  1140. break;
  1141. ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
  1142. break;
  1143. case ISCSI_OP_TEXT:
  1144. if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF)
  1145. cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
  1146. else
  1147. cmd = isert_allocate_cmd(conn, rx_desc);
  1148. if (!cmd)
  1149. break;
  1150. isert_cmd = iscsit_priv_cmd(cmd);
  1151. ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,
  1152. rx_desc, (struct iscsi_text *)hdr);
  1153. break;
  1154. default:
  1155. isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
  1156. dump_stack();
  1157. break;
  1158. }
  1159. return ret;
  1160. }
  1161. static void
  1162. isert_print_wc(struct ib_wc *wc, const char *type)
  1163. {
  1164. if (wc->status != IB_WC_WR_FLUSH_ERR)
  1165. isert_err("%s failure: %s (%d) vend_err %x\n", type,
  1166. ib_wc_status_msg(wc->status), wc->status,
  1167. wc->vendor_err);
  1168. else
  1169. isert_dbg("%s failure: %s (%d)\n", type,
  1170. ib_wc_status_msg(wc->status), wc->status);
  1171. }
  1172. static void
  1173. isert_recv_done(struct ib_cq *cq, struct ib_wc *wc)
  1174. {
  1175. struct isert_conn *isert_conn = wc->qp->qp_context;
  1176. struct ib_device *ib_dev = isert_conn->cm_id->device;
  1177. struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe);
  1178. struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
  1179. struct iser_ctrl *iser_ctrl = &rx_desc->iser_header;
  1180. uint64_t read_va = 0, write_va = 0;
  1181. uint32_t read_stag = 0, write_stag = 0;
  1182. if (unlikely(wc->status != IB_WC_SUCCESS)) {
  1183. isert_print_wc(wc, "recv");
  1184. if (wc->status != IB_WC_WR_FLUSH_ERR)
  1185. iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
  1186. return;
  1187. }
  1188. rx_desc->in_use = true;
  1189. ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr,
  1190. ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
  1191. isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
  1192. rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags,
  1193. (int)(wc->byte_len - ISER_HEADERS_LEN));
  1194. switch (iser_ctrl->flags & 0xF0) {
  1195. case ISCSI_CTRL:
  1196. if (iser_ctrl->flags & ISER_RSV) {
  1197. read_stag = be32_to_cpu(iser_ctrl->read_stag);
  1198. read_va = be64_to_cpu(iser_ctrl->read_va);
  1199. isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n",
  1200. read_stag, (unsigned long long)read_va);
  1201. }
  1202. if (iser_ctrl->flags & ISER_WSV) {
  1203. write_stag = be32_to_cpu(iser_ctrl->write_stag);
  1204. write_va = be64_to_cpu(iser_ctrl->write_va);
  1205. isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n",
  1206. write_stag, (unsigned long long)write_va);
  1207. }
  1208. isert_dbg("ISER ISCSI_CTRL PDU\n");
  1209. break;
  1210. case ISER_HELLO:
  1211. isert_err("iSER Hello message\n");
  1212. break;
  1213. default:
  1214. isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags);
  1215. break;
  1216. }
  1217. isert_rx_opcode(isert_conn, rx_desc,
  1218. read_stag, read_va, write_stag, write_va);
  1219. ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr,
  1220. ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
  1221. }
  1222. static void
  1223. isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc)
  1224. {
  1225. struct isert_conn *isert_conn = wc->qp->qp_context;
  1226. struct ib_device *ib_dev = isert_conn->device->ib_device;
  1227. if (unlikely(wc->status != IB_WC_SUCCESS)) {
  1228. isert_print_wc(wc, "login recv");
  1229. return;
  1230. }
  1231. ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma,
  1232. ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
  1233. isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN;
  1234. if (isert_conn->conn) {
  1235. struct iscsi_login *login = isert_conn->conn->conn_login;
  1236. if (login && !login->first_request)
  1237. isert_rx_login_req(isert_conn);
  1238. }
  1239. mutex_lock(&isert_conn->mutex);
  1240. complete(&isert_conn->login_req_comp);
  1241. mutex_unlock(&isert_conn->mutex);
  1242. ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma,
  1243. ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
  1244. }
  1245. static void
  1246. isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn)
  1247. {
  1248. struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd;
  1249. enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
  1250. if (!cmd->rw.nr_ops)
  1251. return;
  1252. if (isert_prot_cmd(conn, se_cmd)) {
  1253. rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp,
  1254. conn->cm_id->port_num, se_cmd->t_data_sg,
  1255. se_cmd->t_data_nents, se_cmd->t_prot_sg,
  1256. se_cmd->t_prot_nents, dir);
  1257. } else {
  1258. rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num,
  1259. se_cmd->t_data_sg, se_cmd->t_data_nents, dir);
  1260. }
  1261. cmd->rw.nr_ops = 0;
  1262. }
  1263. static void
  1264. isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
  1265. {
  1266. struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
  1267. struct isert_conn *isert_conn = isert_cmd->conn;
  1268. struct iscsi_conn *conn = isert_conn->conn;
  1269. struct iscsi_text_rsp *hdr;
  1270. isert_dbg("Cmd %p\n", isert_cmd);
  1271. switch (cmd->iscsi_opcode) {
  1272. case ISCSI_OP_SCSI_CMD:
  1273. spin_lock_bh(&conn->cmd_lock);
  1274. if (!list_empty(&cmd->i_conn_node))
  1275. list_del_init(&cmd->i_conn_node);
  1276. spin_unlock_bh(&conn->cmd_lock);
  1277. if (cmd->data_direction == DMA_TO_DEVICE) {
  1278. iscsit_stop_dataout_timer(cmd);
  1279. /*
  1280. * Check for special case during comp_err where
  1281. * WRITE_PENDING has been handed off from core,
  1282. * but requires an extra target_put_sess_cmd()
  1283. * before transport_generic_free_cmd() below.
  1284. */
  1285. if (comp_err &&
  1286. cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
  1287. struct se_cmd *se_cmd = &cmd->se_cmd;
  1288. target_put_sess_cmd(se_cmd);
  1289. }
  1290. }
  1291. isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
  1292. transport_generic_free_cmd(&cmd->se_cmd, 0);
  1293. break;
  1294. case ISCSI_OP_SCSI_TMFUNC:
  1295. spin_lock_bh(&conn->cmd_lock);
  1296. if (!list_empty(&cmd->i_conn_node))
  1297. list_del_init(&cmd->i_conn_node);
  1298. spin_unlock_bh(&conn->cmd_lock);
  1299. transport_generic_free_cmd(&cmd->se_cmd, 0);
  1300. break;
  1301. case ISCSI_OP_REJECT:
  1302. case ISCSI_OP_NOOP_OUT:
  1303. case ISCSI_OP_TEXT:
  1304. hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
  1305. /* If the continue bit is on, keep the command alive */
  1306. if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)
  1307. break;
  1308. spin_lock_bh(&conn->cmd_lock);
  1309. if (!list_empty(&cmd->i_conn_node))
  1310. list_del_init(&cmd->i_conn_node);
  1311. spin_unlock_bh(&conn->cmd_lock);
  1312. /*
  1313. * Handle special case for REJECT when iscsi_add_reject*() has
  1314. * overwritten the original iscsi_opcode assignment, and the
  1315. * associated cmd->se_cmd needs to be released.
  1316. */
  1317. if (cmd->se_cmd.se_tfo != NULL) {
  1318. isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n",
  1319. cmd->iscsi_opcode);
  1320. transport_generic_free_cmd(&cmd->se_cmd, 0);
  1321. break;
  1322. }
  1323. /* fall through */
  1324. default:
  1325. iscsit_release_cmd(cmd);
  1326. break;
  1327. }
  1328. }
  1329. static void
  1330. isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
  1331. {
  1332. if (tx_desc->dma_addr != 0) {
  1333. isert_dbg("unmap single for tx_desc->dma_addr\n");
  1334. ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
  1335. ISER_HEADERS_LEN, DMA_TO_DEVICE);
  1336. tx_desc->dma_addr = 0;
  1337. }
  1338. }
  1339. static void
  1340. isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
  1341. struct ib_device *ib_dev, bool comp_err)
  1342. {
  1343. if (isert_cmd->pdu_buf_dma != 0) {
  1344. isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n");
  1345. ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma,
  1346. isert_cmd->pdu_buf_len, DMA_TO_DEVICE);
  1347. isert_cmd->pdu_buf_dma = 0;
  1348. }
  1349. isert_unmap_tx_desc(tx_desc, ib_dev);
  1350. isert_put_cmd(isert_cmd, comp_err);
  1351. }
  1352. static int
  1353. isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr)
  1354. {
  1355. struct ib_mr_status mr_status;
  1356. int ret;
  1357. ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
  1358. if (ret) {
  1359. isert_err("ib_check_mr_status failed, ret %d\n", ret);
  1360. goto fail_mr_status;
  1361. }
  1362. if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
  1363. u64 sec_offset_err;
  1364. u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8;
  1365. switch (mr_status.sig_err.err_type) {
  1366. case IB_SIG_BAD_GUARD:
  1367. se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED;
  1368. break;
  1369. case IB_SIG_BAD_REFTAG:
  1370. se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED;
  1371. break;
  1372. case IB_SIG_BAD_APPTAG:
  1373. se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED;
  1374. break;
  1375. }
  1376. sec_offset_err = mr_status.sig_err.sig_err_offset;
  1377. do_div(sec_offset_err, block_size);
  1378. se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba;
  1379. isert_err("PI error found type %d at sector 0x%llx "
  1380. "expected 0x%x vs actual 0x%x\n",
  1381. mr_status.sig_err.err_type,
  1382. (unsigned long long)se_cmd->bad_sector,
  1383. mr_status.sig_err.expected,
  1384. mr_status.sig_err.actual);
  1385. ret = 1;
  1386. }
  1387. fail_mr_status:
  1388. return ret;
  1389. }
  1390. static void
  1391. isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
  1392. {
  1393. struct isert_conn *isert_conn = wc->qp->qp_context;
  1394. struct isert_device *device = isert_conn->device;
  1395. struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
  1396. struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
  1397. struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd;
  1398. int ret = 0;
  1399. if (unlikely(wc->status != IB_WC_SUCCESS)) {
  1400. isert_print_wc(wc, "rdma write");
  1401. if (wc->status != IB_WC_WR_FLUSH_ERR)
  1402. iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
  1403. isert_completion_put(desc, isert_cmd, device->ib_device, true);
  1404. return;
  1405. }
  1406. isert_dbg("Cmd %p\n", isert_cmd);
  1407. ret = isert_check_pi_status(cmd, isert_cmd->rw.sig->sig_mr);
  1408. isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
  1409. if (ret) {
  1410. /*
  1411. * transport_generic_request_failure() expects to have
  1412. * plus two references to handle queue-full, so re-add
  1413. * one here as target-core will have already dropped
  1414. * it after the first isert_put_datain() callback.
  1415. */
  1416. kref_get(&cmd->cmd_kref);
  1417. transport_generic_request_failure(cmd, cmd->pi_err);
  1418. } else {
  1419. /*
  1420. * XXX: isert_put_response() failure is not retried.
  1421. */
  1422. ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd);
  1423. if (ret)
  1424. pr_warn_ratelimited("isert_put_response() ret: %d\n", ret);
  1425. }
  1426. }
  1427. static void
  1428. isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
  1429. {
  1430. struct isert_conn *isert_conn = wc->qp->qp_context;
  1431. struct isert_device *device = isert_conn->device;
  1432. struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
  1433. struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
  1434. struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
  1435. struct se_cmd *se_cmd = &cmd->se_cmd;
  1436. int ret = 0;
  1437. if (unlikely(wc->status != IB_WC_SUCCESS)) {
  1438. isert_print_wc(wc, "rdma read");
  1439. if (wc->status != IB_WC_WR_FLUSH_ERR)
  1440. iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
  1441. isert_completion_put(desc, isert_cmd, device->ib_device, true);
  1442. return;
  1443. }
  1444. isert_dbg("Cmd %p\n", isert_cmd);
  1445. iscsit_stop_dataout_timer(cmd);
  1446. if (isert_prot_cmd(isert_conn, se_cmd))
  1447. ret = isert_check_pi_status(se_cmd, isert_cmd->rw.sig->sig_mr);
  1448. isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
  1449. cmd->write_data_done = 0;
  1450. isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd);
  1451. spin_lock_bh(&cmd->istate_lock);
  1452. cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
  1453. cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
  1454. spin_unlock_bh(&cmd->istate_lock);
  1455. /*
  1456. * transport_generic_request_failure() will drop the extra
  1457. * se_cmd->cmd_kref reference after T10-PI error, and handle
  1458. * any non-zero ->queue_status() callback error retries.
  1459. */
  1460. if (ret)
  1461. transport_generic_request_failure(se_cmd, se_cmd->pi_err);
  1462. else
  1463. target_execute_cmd(se_cmd);
  1464. }
  1465. static void
  1466. isert_do_control_comp(struct work_struct *work)
  1467. {
  1468. struct isert_cmd *isert_cmd = container_of(work,
  1469. struct isert_cmd, comp_work);
  1470. struct isert_conn *isert_conn = isert_cmd->conn;
  1471. struct ib_device *ib_dev = isert_conn->cm_id->device;
  1472. struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
  1473. isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);
  1474. switch (cmd->i_state) {
  1475. case ISTATE_SEND_TASKMGTRSP:
  1476. iscsit_tmr_post_handler(cmd, cmd->conn);
  1477. /* fall through */
  1478. case ISTATE_SEND_REJECT:
  1479. case ISTATE_SEND_TEXTRSP:
  1480. cmd->i_state = ISTATE_SENT_STATUS;
  1481. isert_completion_put(&isert_cmd->tx_desc, isert_cmd,
  1482. ib_dev, false);
  1483. break;
  1484. case ISTATE_SEND_LOGOUTRSP:
  1485. iscsit_logout_post_handler(cmd, cmd->conn);
  1486. break;
  1487. default:
  1488. isert_err("Unknown i_state %d\n", cmd->i_state);
  1489. dump_stack();
  1490. break;
  1491. }
  1492. }
  1493. static void
  1494. isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc)
  1495. {
  1496. struct isert_conn *isert_conn = wc->qp->qp_context;
  1497. struct ib_device *ib_dev = isert_conn->cm_id->device;
  1498. struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
  1499. if (unlikely(wc->status != IB_WC_SUCCESS)) {
  1500. isert_print_wc(wc, "login send");
  1501. if (wc->status != IB_WC_WR_FLUSH_ERR)
  1502. iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
  1503. }
  1504. isert_unmap_tx_desc(tx_desc, ib_dev);
  1505. }
  1506. static void
  1507. isert_send_done(struct ib_cq *cq, struct ib_wc *wc)
  1508. {
  1509. struct isert_conn *isert_conn = wc->qp->qp_context;
  1510. struct ib_device *ib_dev = isert_conn->cm_id->device;
  1511. struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
  1512. struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc);
  1513. if (unlikely(wc->status != IB_WC_SUCCESS)) {
  1514. isert_print_wc(wc, "send");
  1515. if (wc->status != IB_WC_WR_FLUSH_ERR)
  1516. iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
  1517. isert_completion_put(tx_desc, isert_cmd, ib_dev, true);
  1518. return;
  1519. }
  1520. isert_dbg("Cmd %p\n", isert_cmd);
  1521. switch (isert_cmd->iscsi_cmd->i_state) {
  1522. case ISTATE_SEND_TASKMGTRSP:
  1523. case ISTATE_SEND_LOGOUTRSP:
  1524. case ISTATE_SEND_REJECT:
  1525. case ISTATE_SEND_TEXTRSP:
  1526. isert_unmap_tx_desc(tx_desc, ib_dev);
  1527. INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
  1528. queue_work(isert_comp_wq, &isert_cmd->comp_work);
  1529. return;
  1530. default:
  1531. isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS;
  1532. isert_completion_put(tx_desc, isert_cmd, ib_dev, false);
  1533. break;
  1534. }
  1535. }
  1536. static int
  1537. isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
  1538. {
  1539. int ret;
  1540. ret = isert_post_recv(isert_conn, isert_cmd->rx_desc);
  1541. if (ret) {
  1542. isert_err("ib_post_recv failed with %d\n", ret);
  1543. return ret;
  1544. }
  1545. ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL);
  1546. if (ret) {
  1547. isert_err("ib_post_send failed with %d\n", ret);
  1548. return ret;
  1549. }
  1550. return ret;
  1551. }
  1552. static int
  1553. isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
  1554. {
  1555. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1556. struct isert_conn *isert_conn = conn->context;
  1557. struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
  1558. struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
  1559. &isert_cmd->tx_desc.iscsi_header;
  1560. isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
  1561. iscsit_build_rsp_pdu(cmd, conn, true, hdr);
  1562. isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
  1563. /*
  1564. * Attach SENSE DATA payload to iSCSI Response PDU
  1565. */
  1566. if (cmd->se_cmd.sense_buffer &&
  1567. ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
  1568. (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
  1569. struct isert_device *device = isert_conn->device;
  1570. struct ib_device *ib_dev = device->ib_device;
  1571. struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
  1572. u32 padding, pdu_len;
  1573. put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
  1574. cmd->sense_buffer);
  1575. cmd->se_cmd.scsi_sense_length += sizeof(__be16);
  1576. padding = -(cmd->se_cmd.scsi_sense_length) & 3;
  1577. hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
  1578. pdu_len = cmd->se_cmd.scsi_sense_length + padding;
  1579. isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
  1580. (void *)cmd->sense_buffer, pdu_len,
  1581. DMA_TO_DEVICE);
  1582. if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
  1583. return -ENOMEM;
  1584. isert_cmd->pdu_buf_len = pdu_len;
  1585. tx_dsg->addr = isert_cmd->pdu_buf_dma;
  1586. tx_dsg->length = pdu_len;
  1587. tx_dsg->lkey = device->pd->local_dma_lkey;
  1588. isert_cmd->tx_desc.num_sge = 2;
  1589. }
  1590. isert_init_send_wr(isert_conn, isert_cmd, send_wr);
  1591. isert_dbg("Posting SCSI Response\n");
  1592. return isert_post_response(isert_conn, isert_cmd);
  1593. }
  1594. static void
  1595. isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
  1596. {
  1597. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1598. struct isert_conn *isert_conn = conn->context;
  1599. spin_lock_bh(&conn->cmd_lock);
  1600. if (!list_empty(&cmd->i_conn_node))
  1601. list_del_init(&cmd->i_conn_node);
  1602. spin_unlock_bh(&conn->cmd_lock);
  1603. if (cmd->data_direction == DMA_TO_DEVICE)
  1604. iscsit_stop_dataout_timer(cmd);
  1605. isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
  1606. }
  1607. static enum target_prot_op
  1608. isert_get_sup_prot_ops(struct iscsi_conn *conn)
  1609. {
  1610. struct isert_conn *isert_conn = conn->context;
  1611. struct isert_device *device = isert_conn->device;
  1612. if (conn->tpg->tpg_attrib.t10_pi) {
  1613. if (device->pi_capable) {
  1614. isert_info("conn %p PI offload enabled\n", isert_conn);
  1615. isert_conn->pi_support = true;
  1616. return TARGET_PROT_ALL;
  1617. }
  1618. }
  1619. isert_info("conn %p PI offload disabled\n", isert_conn);
  1620. isert_conn->pi_support = false;
  1621. return TARGET_PROT_NORMAL;
  1622. }
  1623. static int
  1624. isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
  1625. bool nopout_response)
  1626. {
  1627. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1628. struct isert_conn *isert_conn = conn->context;
  1629. struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
  1630. isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
  1631. iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
  1632. &isert_cmd->tx_desc.iscsi_header,
  1633. nopout_response);
  1634. isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
  1635. isert_init_send_wr(isert_conn, isert_cmd, send_wr);
  1636. isert_dbg("conn %p Posting NOPIN Response\n", isert_conn);
  1637. return isert_post_response(isert_conn, isert_cmd);
  1638. }
  1639. static int
  1640. isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  1641. {
  1642. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1643. struct isert_conn *isert_conn = conn->context;
  1644. struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
  1645. isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
  1646. iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
  1647. &isert_cmd->tx_desc.iscsi_header);
  1648. isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
  1649. isert_init_send_wr(isert_conn, isert_cmd, send_wr);
  1650. isert_dbg("conn %p Posting Logout Response\n", isert_conn);
  1651. return isert_post_response(isert_conn, isert_cmd);
  1652. }
  1653. static int
  1654. isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  1655. {
  1656. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1657. struct isert_conn *isert_conn = conn->context;
  1658. struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
  1659. isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
  1660. iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
  1661. &isert_cmd->tx_desc.iscsi_header);
  1662. isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
  1663. isert_init_send_wr(isert_conn, isert_cmd, send_wr);
  1664. isert_dbg("conn %p Posting Task Management Response\n", isert_conn);
  1665. return isert_post_response(isert_conn, isert_cmd);
  1666. }
  1667. static int
  1668. isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  1669. {
  1670. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1671. struct isert_conn *isert_conn = conn->context;
  1672. struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
  1673. struct isert_device *device = isert_conn->device;
  1674. struct ib_device *ib_dev = device->ib_device;
  1675. struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
  1676. struct iscsi_reject *hdr =
  1677. (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
  1678. isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
  1679. iscsit_build_reject(cmd, conn, hdr);
  1680. isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
  1681. hton24(hdr->dlength, ISCSI_HDR_LEN);
  1682. isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
  1683. (void *)cmd->buf_ptr, ISCSI_HDR_LEN,
  1684. DMA_TO_DEVICE);
  1685. if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
  1686. return -ENOMEM;
  1687. isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;
  1688. tx_dsg->addr = isert_cmd->pdu_buf_dma;
  1689. tx_dsg->length = ISCSI_HDR_LEN;
  1690. tx_dsg->lkey = device->pd->local_dma_lkey;
  1691. isert_cmd->tx_desc.num_sge = 2;
  1692. isert_init_send_wr(isert_conn, isert_cmd, send_wr);
  1693. isert_dbg("conn %p Posting Reject\n", isert_conn);
  1694. return isert_post_response(isert_conn, isert_cmd);
  1695. }
  1696. static int
  1697. isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  1698. {
  1699. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1700. struct isert_conn *isert_conn = conn->context;
  1701. struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
  1702. struct iscsi_text_rsp *hdr =
  1703. (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
  1704. u32 txt_rsp_len;
  1705. int rc;
  1706. isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
  1707. rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND);
  1708. if (rc < 0)
  1709. return rc;
  1710. txt_rsp_len = rc;
  1711. isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
  1712. if (txt_rsp_len) {
  1713. struct isert_device *device = isert_conn->device;
  1714. struct ib_device *ib_dev = device->ib_device;
  1715. struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
  1716. void *txt_rsp_buf = cmd->buf_ptr;
  1717. isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
  1718. txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE);
  1719. if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
  1720. return -ENOMEM;
  1721. isert_cmd->pdu_buf_len = txt_rsp_len;
  1722. tx_dsg->addr = isert_cmd->pdu_buf_dma;
  1723. tx_dsg->length = txt_rsp_len;
  1724. tx_dsg->lkey = device->pd->local_dma_lkey;
  1725. isert_cmd->tx_desc.num_sge = 2;
  1726. }
  1727. isert_init_send_wr(isert_conn, isert_cmd, send_wr);
  1728. isert_dbg("conn %p Text Response\n", isert_conn);
  1729. return isert_post_response(isert_conn, isert_cmd);
  1730. }
  1731. static inline void
  1732. isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs,
  1733. struct ib_sig_domain *domain)
  1734. {
  1735. domain->sig_type = IB_SIG_TYPE_T10_DIF;
  1736. domain->sig.dif.bg_type = IB_T10DIF_CRC;
  1737. domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size;
  1738. domain->sig.dif.ref_tag = se_cmd->reftag_seed;
  1739. /*
  1740. * At the moment we hard code those, but if in the future
  1741. * the target core would like to use it, we will take it
  1742. * from se_cmd.
  1743. */
  1744. domain->sig.dif.apptag_check_mask = 0xffff;
  1745. domain->sig.dif.app_escape = true;
  1746. domain->sig.dif.ref_escape = true;
  1747. if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT ||
  1748. se_cmd->prot_type == TARGET_DIF_TYPE2_PROT)
  1749. domain->sig.dif.ref_remap = true;
  1750. };
  1751. static int
  1752. isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
  1753. {
  1754. memset(sig_attrs, 0, sizeof(*sig_attrs));
  1755. switch (se_cmd->prot_op) {
  1756. case TARGET_PROT_DIN_INSERT:
  1757. case TARGET_PROT_DOUT_STRIP:
  1758. sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
  1759. isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
  1760. break;
  1761. case TARGET_PROT_DOUT_INSERT:
  1762. case TARGET_PROT_DIN_STRIP:
  1763. sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
  1764. isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
  1765. break;
  1766. case TARGET_PROT_DIN_PASS:
  1767. case TARGET_PROT_DOUT_PASS:
  1768. isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
  1769. isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
  1770. break;
  1771. default:
  1772. isert_err("Unsupported PI operation %d\n", se_cmd->prot_op);
  1773. return -EINVAL;
  1774. }
  1775. if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD)
  1776. sig_attrs->check_mask |= IB_SIG_CHECK_GUARD;
  1777. if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG)
  1778. sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG;
  1779. if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG)
  1780. sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG;
  1781. return 0;
  1782. }
  1783. static int
  1784. isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn,
  1785. struct ib_cqe *cqe, struct ib_send_wr *chain_wr)
  1786. {
  1787. struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd;
  1788. enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
  1789. u8 port_num = conn->cm_id->port_num;
  1790. u64 addr;
  1791. u32 rkey, offset;
  1792. int ret;
  1793. if (cmd->ctx_init_done)
  1794. goto rdma_ctx_post;
  1795. if (dir == DMA_FROM_DEVICE) {
  1796. addr = cmd->write_va;
  1797. rkey = cmd->write_stag;
  1798. offset = cmd->iscsi_cmd->write_data_done;
  1799. } else {
  1800. addr = cmd->read_va;
  1801. rkey = cmd->read_stag;
  1802. offset = 0;
  1803. }
  1804. if (isert_prot_cmd(conn, se_cmd)) {
  1805. struct ib_sig_attrs sig_attrs;
  1806. ret = isert_set_sig_attrs(se_cmd, &sig_attrs);
  1807. if (ret)
  1808. return ret;
  1809. WARN_ON_ONCE(offset);
  1810. ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num,
  1811. se_cmd->t_data_sg, se_cmd->t_data_nents,
  1812. se_cmd->t_prot_sg, se_cmd->t_prot_nents,
  1813. &sig_attrs, addr, rkey, dir);
  1814. } else {
  1815. ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num,
  1816. se_cmd->t_data_sg, se_cmd->t_data_nents,
  1817. offset, addr, rkey, dir);
  1818. }
  1819. if (ret < 0) {
  1820. isert_err("Cmd: %p failed to prepare RDMA res\n", cmd);
  1821. return ret;
  1822. }
  1823. cmd->ctx_init_done = true;
  1824. rdma_ctx_post:
  1825. ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr);
  1826. if (ret < 0)
  1827. isert_err("Cmd: %p failed to post RDMA res\n", cmd);
  1828. return ret;
  1829. }
  1830. static int
  1831. isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
  1832. {
  1833. struct se_cmd *se_cmd = &cmd->se_cmd;
  1834. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1835. struct isert_conn *isert_conn = conn->context;
  1836. struct ib_cqe *cqe = NULL;
  1837. struct ib_send_wr *chain_wr = NULL;
  1838. int rc;
  1839. isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n",
  1840. isert_cmd, se_cmd->data_length);
  1841. if (isert_prot_cmd(isert_conn, se_cmd)) {
  1842. isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done;
  1843. cqe = &isert_cmd->tx_desc.tx_cqe;
  1844. } else {
  1845. /*
  1846. * Build isert_conn->tx_desc for iSCSI response PDU and attach
  1847. */
  1848. isert_create_send_desc(isert_conn, isert_cmd,
  1849. &isert_cmd->tx_desc);
  1850. iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *)
  1851. &isert_cmd->tx_desc.iscsi_header);
  1852. isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
  1853. isert_init_send_wr(isert_conn, isert_cmd,
  1854. &isert_cmd->tx_desc.send_wr);
  1855. rc = isert_post_recv(isert_conn, isert_cmd->rx_desc);
  1856. if (rc) {
  1857. isert_err("ib_post_recv failed with %d\n", rc);
  1858. return rc;
  1859. }
  1860. chain_wr = &isert_cmd->tx_desc.send_wr;
  1861. }
  1862. rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr);
  1863. isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n",
  1864. isert_cmd, rc);
  1865. return rc;
  1866. }
  1867. static int
  1868. isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
  1869. {
  1870. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1871. int ret;
  1872. isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n",
  1873. isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done);
  1874. isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done;
  1875. ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context,
  1876. &isert_cmd->tx_desc.tx_cqe, NULL);
  1877. isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n",
  1878. isert_cmd, ret);
  1879. return ret;
  1880. }
  1881. static int
  1882. isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
  1883. {
  1884. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  1885. int ret = 0;
  1886. switch (state) {
  1887. case ISTATE_REMOVE:
  1888. spin_lock_bh(&conn->cmd_lock);
  1889. list_del_init(&cmd->i_conn_node);
  1890. spin_unlock_bh(&conn->cmd_lock);
  1891. isert_put_cmd(isert_cmd, true);
  1892. break;
  1893. case ISTATE_SEND_NOPIN_WANT_RESPONSE:
  1894. ret = isert_put_nopin(cmd, conn, false);
  1895. break;
  1896. default:
  1897. isert_err("Unknown immediate state: 0x%02x\n", state);
  1898. ret = -EINVAL;
  1899. break;
  1900. }
  1901. return ret;
  1902. }
  1903. static int
  1904. isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
  1905. {
  1906. struct isert_conn *isert_conn = conn->context;
  1907. int ret;
  1908. switch (state) {
  1909. case ISTATE_SEND_LOGOUTRSP:
  1910. ret = isert_put_logout_rsp(cmd, conn);
  1911. if (!ret)
  1912. isert_conn->logout_posted = true;
  1913. break;
  1914. case ISTATE_SEND_NOPIN:
  1915. ret = isert_put_nopin(cmd, conn, true);
  1916. break;
  1917. case ISTATE_SEND_TASKMGTRSP:
  1918. ret = isert_put_tm_rsp(cmd, conn);
  1919. break;
  1920. case ISTATE_SEND_REJECT:
  1921. ret = isert_put_reject(cmd, conn);
  1922. break;
  1923. case ISTATE_SEND_TEXTRSP:
  1924. ret = isert_put_text_rsp(cmd, conn);
  1925. break;
  1926. case ISTATE_SEND_STATUS:
  1927. /*
  1928. * Special case for sending non GOOD SCSI status from TX thread
  1929. * context during pre se_cmd excecution failure.
  1930. */
  1931. ret = isert_put_response(conn, cmd);
  1932. break;
  1933. default:
  1934. isert_err("Unknown response state: 0x%02x\n", state);
  1935. ret = -EINVAL;
  1936. break;
  1937. }
  1938. return ret;
  1939. }
  1940. struct rdma_cm_id *
  1941. isert_setup_id(struct isert_np *isert_np)
  1942. {
  1943. struct iscsi_np *np = isert_np->np;
  1944. struct rdma_cm_id *id;
  1945. struct sockaddr *sa;
  1946. int ret;
  1947. sa = (struct sockaddr *)&np->np_sockaddr;
  1948. isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa);
  1949. id = rdma_create_id(&init_net, isert_cma_handler, isert_np,
  1950. RDMA_PS_TCP, IB_QPT_RC);
  1951. if (IS_ERR(id)) {
  1952. isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id));
  1953. ret = PTR_ERR(id);
  1954. goto out;
  1955. }
  1956. isert_dbg("id %p context %p\n", id, id->context);
  1957. ret = rdma_bind_addr(id, sa);
  1958. if (ret) {
  1959. isert_err("rdma_bind_addr() failed: %d\n", ret);
  1960. goto out_id;
  1961. }
  1962. ret = rdma_listen(id, 0);
  1963. if (ret) {
  1964. isert_err("rdma_listen() failed: %d\n", ret);
  1965. goto out_id;
  1966. }
  1967. return id;
  1968. out_id:
  1969. rdma_destroy_id(id);
  1970. out:
  1971. return ERR_PTR(ret);
  1972. }
  1973. static int
  1974. isert_setup_np(struct iscsi_np *np,
  1975. struct sockaddr_storage *ksockaddr)
  1976. {
  1977. struct isert_np *isert_np;
  1978. struct rdma_cm_id *isert_lid;
  1979. int ret;
  1980. isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
  1981. if (!isert_np)
  1982. return -ENOMEM;
  1983. sema_init(&isert_np->sem, 0);
  1984. mutex_init(&isert_np->mutex);
  1985. INIT_LIST_HEAD(&isert_np->accepted);
  1986. INIT_LIST_HEAD(&isert_np->pending);
  1987. isert_np->np = np;
  1988. /*
  1989. * Setup the np->np_sockaddr from the passed sockaddr setup
  1990. * in iscsi_target_configfs.c code..
  1991. */
  1992. memcpy(&np->np_sockaddr, ksockaddr,
  1993. sizeof(struct sockaddr_storage));
  1994. isert_lid = isert_setup_id(isert_np);
  1995. if (IS_ERR(isert_lid)) {
  1996. ret = PTR_ERR(isert_lid);
  1997. goto out;
  1998. }
  1999. isert_np->cm_id = isert_lid;
  2000. np->np_context = isert_np;
  2001. return 0;
  2002. out:
  2003. kfree(isert_np);
  2004. return ret;
  2005. }
  2006. static int
  2007. isert_rdma_accept(struct isert_conn *isert_conn)
  2008. {
  2009. struct rdma_cm_id *cm_id = isert_conn->cm_id;
  2010. struct rdma_conn_param cp;
  2011. int ret;
  2012. struct iser_cm_hdr rsp_hdr;
  2013. memset(&cp, 0, sizeof(struct rdma_conn_param));
  2014. cp.initiator_depth = isert_conn->initiator_depth;
  2015. cp.retry_count = 7;
  2016. cp.rnr_retry_count = 7;
  2017. memset(&rsp_hdr, 0, sizeof(rsp_hdr));
  2018. rsp_hdr.flags = ISERT_ZBVA_NOT_USED;
  2019. if (!isert_conn->snd_w_inv)
  2020. rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED;
  2021. cp.private_data = (void *)&rsp_hdr;
  2022. cp.private_data_len = sizeof(rsp_hdr);
  2023. ret = rdma_accept(cm_id, &cp);
  2024. if (ret) {
  2025. isert_err("rdma_accept() failed with: %d\n", ret);
  2026. return ret;
  2027. }
  2028. return 0;
  2029. }
  2030. static int
  2031. isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
  2032. {
  2033. struct isert_conn *isert_conn = conn->context;
  2034. int ret;
  2035. isert_info("before login_req comp conn: %p\n", isert_conn);
  2036. ret = wait_for_completion_interruptible(&isert_conn->login_req_comp);
  2037. if (ret) {
  2038. isert_err("isert_conn %p interrupted before got login req\n",
  2039. isert_conn);
  2040. return ret;
  2041. }
  2042. reinit_completion(&isert_conn->login_req_comp);
  2043. /*
  2044. * For login requests after the first PDU, isert_rx_login_req() will
  2045. * kick schedule_delayed_work(&conn->login_work) as the packet is
  2046. * received, which turns this callback from iscsi_target_do_login_rx()
  2047. * into a NOP.
  2048. */
  2049. if (!login->first_request)
  2050. return 0;
  2051. isert_rx_login_req(isert_conn);
  2052. isert_info("before login_comp conn: %p\n", conn);
  2053. ret = wait_for_completion_interruptible(&isert_conn->login_comp);
  2054. if (ret)
  2055. return ret;
  2056. isert_info("processing login->req: %p\n", login->req);
  2057. return 0;
  2058. }
  2059. static void
  2060. isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
  2061. struct isert_conn *isert_conn)
  2062. {
  2063. struct rdma_cm_id *cm_id = isert_conn->cm_id;
  2064. struct rdma_route *cm_route = &cm_id->route;
  2065. conn->login_family = np->np_sockaddr.ss_family;
  2066. conn->login_sockaddr = cm_route->addr.dst_addr;
  2067. conn->local_sockaddr = cm_route->addr.src_addr;
  2068. }
  2069. static int
  2070. isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
  2071. {
  2072. struct isert_np *isert_np = np->np_context;
  2073. struct isert_conn *isert_conn;
  2074. int ret;
  2075. accept_wait:
  2076. ret = down_interruptible(&isert_np->sem);
  2077. if (ret)
  2078. return -ENODEV;
  2079. spin_lock_bh(&np->np_thread_lock);
  2080. if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
  2081. spin_unlock_bh(&np->np_thread_lock);
  2082. isert_dbg("np_thread_state %d\n",
  2083. np->np_thread_state);
  2084. /**
  2085. * No point in stalling here when np_thread
  2086. * is in state RESET/SHUTDOWN/EXIT - bail
  2087. **/
  2088. return -ENODEV;
  2089. }
  2090. spin_unlock_bh(&np->np_thread_lock);
  2091. mutex_lock(&isert_np->mutex);
  2092. if (list_empty(&isert_np->pending)) {
  2093. mutex_unlock(&isert_np->mutex);
  2094. goto accept_wait;
  2095. }
  2096. isert_conn = list_first_entry(&isert_np->pending,
  2097. struct isert_conn, node);
  2098. list_del_init(&isert_conn->node);
  2099. mutex_unlock(&isert_np->mutex);
  2100. conn->context = isert_conn;
  2101. isert_conn->conn = conn;
  2102. isert_conn->state = ISER_CONN_BOUND;
  2103. isert_set_conn_info(np, conn, isert_conn);
  2104. isert_dbg("Processing isert_conn: %p\n", isert_conn);
  2105. return 0;
  2106. }
  2107. static void
  2108. isert_free_np(struct iscsi_np *np)
  2109. {
  2110. struct isert_np *isert_np = np->np_context;
  2111. struct isert_conn *isert_conn, *n;
  2112. if (isert_np->cm_id)
  2113. rdma_destroy_id(isert_np->cm_id);
  2114. /*
  2115. * FIXME: At this point we don't have a good way to insure
  2116. * that at this point we don't have hanging connections that
  2117. * completed RDMA establishment but didn't start iscsi login
  2118. * process. So work-around this by cleaning up what ever piled
  2119. * up in accepted and pending lists.
  2120. */
  2121. mutex_lock(&isert_np->mutex);
  2122. if (!list_empty(&isert_np->pending)) {
  2123. isert_info("Still have isert pending connections\n");
  2124. list_for_each_entry_safe(isert_conn, n,
  2125. &isert_np->pending,
  2126. node) {
  2127. isert_info("cleaning isert_conn %p state (%d)\n",
  2128. isert_conn, isert_conn->state);
  2129. isert_connect_release(isert_conn);
  2130. }
  2131. }
  2132. if (!list_empty(&isert_np->accepted)) {
  2133. isert_info("Still have isert accepted connections\n");
  2134. list_for_each_entry_safe(isert_conn, n,
  2135. &isert_np->accepted,
  2136. node) {
  2137. isert_info("cleaning isert_conn %p state (%d)\n",
  2138. isert_conn, isert_conn->state);
  2139. isert_connect_release(isert_conn);
  2140. }
  2141. }
  2142. mutex_unlock(&isert_np->mutex);
  2143. np->np_context = NULL;
  2144. kfree(isert_np);
  2145. }
  2146. static void isert_release_work(struct work_struct *work)
  2147. {
  2148. struct isert_conn *isert_conn = container_of(work,
  2149. struct isert_conn,
  2150. release_work);
  2151. isert_info("Starting release conn %p\n", isert_conn);
  2152. mutex_lock(&isert_conn->mutex);
  2153. isert_conn->state = ISER_CONN_DOWN;
  2154. mutex_unlock(&isert_conn->mutex);
  2155. isert_info("Destroying conn %p\n", isert_conn);
  2156. isert_put_conn(isert_conn);
  2157. }
  2158. static void
  2159. isert_wait4logout(struct isert_conn *isert_conn)
  2160. {
  2161. struct iscsi_conn *conn = isert_conn->conn;
  2162. isert_info("conn %p\n", isert_conn);
  2163. if (isert_conn->logout_posted) {
  2164. isert_info("conn %p wait for conn_logout_comp\n", isert_conn);
  2165. wait_for_completion_timeout(&conn->conn_logout_comp,
  2166. SECONDS_FOR_LOGOUT_COMP * HZ);
  2167. }
  2168. }
  2169. static void
  2170. isert_wait4cmds(struct iscsi_conn *conn)
  2171. {
  2172. isert_info("iscsi_conn %p\n", conn);
  2173. if (conn->sess) {
  2174. target_sess_cmd_list_set_waiting(conn->sess->se_sess);
  2175. target_wait_for_sess_cmds(conn->sess->se_sess);
  2176. }
  2177. }
  2178. /**
  2179. * isert_put_unsol_pending_cmds() - Drop commands waiting for
  2180. * unsolicitate dataout
  2181. * @conn: iscsi connection
  2182. *
  2183. * We might still have commands that are waiting for unsolicited
  2184. * dataouts messages. We must put the extra reference on those
  2185. * before blocking on the target_wait_for_session_cmds
  2186. */
  2187. static void
  2188. isert_put_unsol_pending_cmds(struct iscsi_conn *conn)
  2189. {
  2190. struct iscsi_cmd *cmd, *tmp;
  2191. static LIST_HEAD(drop_cmd_list);
  2192. spin_lock_bh(&conn->cmd_lock);
  2193. list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {
  2194. if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) &&
  2195. (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) &&
  2196. (cmd->write_data_done < cmd->se_cmd.data_length))
  2197. list_move_tail(&cmd->i_conn_node, &drop_cmd_list);
  2198. }
  2199. spin_unlock_bh(&conn->cmd_lock);
  2200. list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) {
  2201. list_del_init(&cmd->i_conn_node);
  2202. if (cmd->i_state != ISTATE_REMOVE) {
  2203. struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
  2204. isert_info("conn %p dropping cmd %p\n", conn, cmd);
  2205. isert_put_cmd(isert_cmd, true);
  2206. }
  2207. }
  2208. }
  2209. static void isert_wait_conn(struct iscsi_conn *conn)
  2210. {
  2211. struct isert_conn *isert_conn = conn->context;
  2212. isert_info("Starting conn %p\n", isert_conn);
  2213. mutex_lock(&isert_conn->mutex);
  2214. isert_conn_terminate(isert_conn);
  2215. mutex_unlock(&isert_conn->mutex);
  2216. ib_drain_qp(isert_conn->qp);
  2217. isert_put_unsol_pending_cmds(conn);
  2218. isert_wait4cmds(conn);
  2219. isert_wait4logout(isert_conn);
  2220. queue_work(isert_release_wq, &isert_conn->release_work);
  2221. }
  2222. static void isert_free_conn(struct iscsi_conn *conn)
  2223. {
  2224. struct isert_conn *isert_conn = conn->context;
  2225. ib_drain_qp(isert_conn->qp);
  2226. isert_put_conn(isert_conn);
  2227. }
  2228. static void isert_get_rx_pdu(struct iscsi_conn *conn)
  2229. {
  2230. struct completion comp;
  2231. init_completion(&comp);
  2232. wait_for_completion_interruptible(&comp);
  2233. }
  2234. static struct iscsit_transport iser_target_transport = {
  2235. .name = "IB/iSER",
  2236. .transport_type = ISCSI_INFINIBAND,
  2237. .rdma_shutdown = true,
  2238. .priv_size = sizeof(struct isert_cmd),
  2239. .owner = THIS_MODULE,
  2240. .iscsit_setup_np = isert_setup_np,
  2241. .iscsit_accept_np = isert_accept_np,
  2242. .iscsit_free_np = isert_free_np,
  2243. .iscsit_wait_conn = isert_wait_conn,
  2244. .iscsit_free_conn = isert_free_conn,
  2245. .iscsit_get_login_rx = isert_get_login_rx,
  2246. .iscsit_put_login_tx = isert_put_login_tx,
  2247. .iscsit_immediate_queue = isert_immediate_queue,
  2248. .iscsit_response_queue = isert_response_queue,
  2249. .iscsit_get_dataout = isert_get_dataout,
  2250. .iscsit_queue_data_in = isert_put_datain,
  2251. .iscsit_queue_status = isert_put_response,
  2252. .iscsit_aborted_task = isert_aborted_task,
  2253. .iscsit_get_rx_pdu = isert_get_rx_pdu,
  2254. .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops,
  2255. };
  2256. static int __init isert_init(void)
  2257. {
  2258. int ret;
  2259. isert_comp_wq = alloc_workqueue("isert_comp_wq",
  2260. WQ_UNBOUND | WQ_HIGHPRI, 0);
  2261. if (!isert_comp_wq) {
  2262. isert_err("Unable to allocate isert_comp_wq\n");
  2263. return -ENOMEM;
  2264. }
  2265. isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND,
  2266. WQ_UNBOUND_MAX_ACTIVE);
  2267. if (!isert_release_wq) {
  2268. isert_err("Unable to allocate isert_release_wq\n");
  2269. ret = -ENOMEM;
  2270. goto destroy_comp_wq;
  2271. }
  2272. iscsit_register_transport(&iser_target_transport);
  2273. isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n");
  2274. return 0;
  2275. destroy_comp_wq:
  2276. destroy_workqueue(isert_comp_wq);
  2277. return ret;
  2278. }
  2279. static void __exit isert_exit(void)
  2280. {
  2281. flush_scheduled_work();
  2282. destroy_workqueue(isert_release_wq);
  2283. destroy_workqueue(isert_comp_wq);
  2284. iscsit_unregister_transport(&iser_target_transport);
  2285. isert_info("iSER_TARGET[0] - Released iser_target_transport\n");
  2286. }
  2287. MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
  2288. MODULE_AUTHOR("nab@Linux-iSCSI.org");
  2289. MODULE_LICENSE("GPL");
  2290. module_init(isert_init);
  2291. module_exit(isert_exit);