fc.c 91 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410
  1. /*
  2. * Copyright (c) 2016 Avago Technologies. All rights reserved.
  3. *
  4. * This program is free software; you can redistribute it and/or modify
  5. * it under the terms of version 2 of the GNU General Public License as
  6. * published by the Free Software Foundation.
  7. *
  8. * This program is distributed in the hope that it will be useful.
  9. * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
  10. * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
  11. * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
  12. * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
  13. * See the GNU General Public License for more details, a copy of which
  14. * can be found in the file COPYING included with this package
  15. *
  16. */
  17. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18. #include <linux/module.h>
  19. #include <linux/parser.h>
  20. #include <uapi/scsi/fc/fc_fs.h>
  21. #include <uapi/scsi/fc/fc_els.h>
  22. #include <linux/delay.h>
  23. #include "nvme.h"
  24. #include "fabrics.h"
  25. #include <linux/nvme-fc-driver.h>
  26. #include <linux/nvme-fc.h>
  27. /* *************************** Data Structures/Defines ****************** */
  28. enum nvme_fc_queue_flags {
  29. NVME_FC_Q_CONNECTED = 0,
  30. NVME_FC_Q_LIVE,
  31. };
  32. #define NVME_FC_DEFAULT_DEV_LOSS_TMO 60 /* seconds */
  33. struct nvme_fc_queue {
  34. struct nvme_fc_ctrl *ctrl;
  35. struct device *dev;
  36. struct blk_mq_hw_ctx *hctx;
  37. void *lldd_handle;
  38. size_t cmnd_capsule_len;
  39. u32 qnum;
  40. u32 rqcnt;
  41. u32 seqno;
  42. u64 connection_id;
  43. atomic_t csn;
  44. unsigned long flags;
  45. } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
  46. enum nvme_fcop_flags {
  47. FCOP_FLAGS_TERMIO = (1 << 0),
  48. FCOP_FLAGS_AEN = (1 << 1),
  49. };
  50. struct nvmefc_ls_req_op {
  51. struct nvmefc_ls_req ls_req;
  52. struct nvme_fc_rport *rport;
  53. struct nvme_fc_queue *queue;
  54. struct request *rq;
  55. u32 flags;
  56. int ls_error;
  57. struct completion ls_done;
  58. struct list_head lsreq_list; /* rport->ls_req_list */
  59. bool req_queued;
  60. };
  61. enum nvme_fcpop_state {
  62. FCPOP_STATE_UNINIT = 0,
  63. FCPOP_STATE_IDLE = 1,
  64. FCPOP_STATE_ACTIVE = 2,
  65. FCPOP_STATE_ABORTED = 3,
  66. FCPOP_STATE_COMPLETE = 4,
  67. };
  68. struct nvme_fc_fcp_op {
  69. struct nvme_request nreq; /*
  70. * nvme/host/core.c
  71. * requires this to be
  72. * the 1st element in the
  73. * private structure
  74. * associated with the
  75. * request.
  76. */
  77. struct nvmefc_fcp_req fcp_req;
  78. struct nvme_fc_ctrl *ctrl;
  79. struct nvme_fc_queue *queue;
  80. struct request *rq;
  81. atomic_t state;
  82. u32 flags;
  83. u32 rqno;
  84. u32 nents;
  85. struct nvme_fc_cmd_iu cmd_iu;
  86. struct nvme_fc_ersp_iu rsp_iu;
  87. };
  88. struct nvme_fc_lport {
  89. struct nvme_fc_local_port localport;
  90. struct ida endp_cnt;
  91. struct list_head port_list; /* nvme_fc_port_list */
  92. struct list_head endp_list;
  93. struct device *dev; /* physical device for dma */
  94. struct nvme_fc_port_template *ops;
  95. struct kref ref;
  96. atomic_t act_rport_cnt;
  97. } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
  98. struct nvme_fc_rport {
  99. struct nvme_fc_remote_port remoteport;
  100. struct list_head endp_list; /* for lport->endp_list */
  101. struct list_head ctrl_list;
  102. struct list_head ls_req_list;
  103. struct device *dev; /* physical device for dma */
  104. struct nvme_fc_lport *lport;
  105. spinlock_t lock;
  106. struct kref ref;
  107. atomic_t act_ctrl_cnt;
  108. unsigned long dev_loss_end;
  109. } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */
  110. enum nvme_fcctrl_flags {
  111. FCCTRL_TERMIO = (1 << 0),
  112. };
  113. struct nvme_fc_ctrl {
  114. spinlock_t lock;
  115. struct nvme_fc_queue *queues;
  116. struct device *dev;
  117. struct nvme_fc_lport *lport;
  118. struct nvme_fc_rport *rport;
  119. u32 cnum;
  120. bool ioq_live;
  121. bool assoc_active;
  122. atomic_t err_work_active;
  123. u64 association_id;
  124. struct list_head ctrl_list; /* rport->ctrl_list */
  125. struct blk_mq_tag_set admin_tag_set;
  126. struct blk_mq_tag_set tag_set;
  127. struct delayed_work connect_work;
  128. struct work_struct err_work;
  129. struct kref ref;
  130. u32 flags;
  131. u32 iocnt;
  132. wait_queue_head_t ioabort_wait;
  133. struct nvme_fc_fcp_op aen_ops[NVME_NR_AEN_COMMANDS];
  134. struct nvme_ctrl ctrl;
  135. };
  136. static inline struct nvme_fc_ctrl *
  137. to_fc_ctrl(struct nvme_ctrl *ctrl)
  138. {
  139. return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
  140. }
  141. static inline struct nvme_fc_lport *
  142. localport_to_lport(struct nvme_fc_local_port *portptr)
  143. {
  144. return container_of(portptr, struct nvme_fc_lport, localport);
  145. }
  146. static inline struct nvme_fc_rport *
  147. remoteport_to_rport(struct nvme_fc_remote_port *portptr)
  148. {
  149. return container_of(portptr, struct nvme_fc_rport, remoteport);
  150. }
  151. static inline struct nvmefc_ls_req_op *
  152. ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
  153. {
  154. return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
  155. }
  156. static inline struct nvme_fc_fcp_op *
  157. fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
  158. {
  159. return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
  160. }
  161. /* *************************** Globals **************************** */
  162. static DEFINE_SPINLOCK(nvme_fc_lock);
  163. static LIST_HEAD(nvme_fc_lport_list);
  164. static DEFINE_IDA(nvme_fc_local_port_cnt);
  165. static DEFINE_IDA(nvme_fc_ctrl_cnt);
  166. static struct workqueue_struct *nvme_fc_wq;
  167. /*
  168. * These items are short-term. They will eventually be moved into
  169. * a generic FC class. See comments in module init.
  170. */
  171. static struct class *fc_class;
  172. static struct device *fc_udev_device;
  173. /* *********************** FC-NVME Port Management ************************ */
  174. static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
  175. struct nvme_fc_queue *, unsigned int);
  176. static void
  177. nvme_fc_free_lport(struct kref *ref)
  178. {
  179. struct nvme_fc_lport *lport =
  180. container_of(ref, struct nvme_fc_lport, ref);
  181. unsigned long flags;
  182. WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
  183. WARN_ON(!list_empty(&lport->endp_list));
  184. /* remove from transport list */
  185. spin_lock_irqsave(&nvme_fc_lock, flags);
  186. list_del(&lport->port_list);
  187. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  188. ida_simple_remove(&nvme_fc_local_port_cnt, lport->localport.port_num);
  189. ida_destroy(&lport->endp_cnt);
  190. put_device(lport->dev);
  191. kfree(lport);
  192. }
  193. static void
  194. nvme_fc_lport_put(struct nvme_fc_lport *lport)
  195. {
  196. kref_put(&lport->ref, nvme_fc_free_lport);
  197. }
  198. static int
  199. nvme_fc_lport_get(struct nvme_fc_lport *lport)
  200. {
  201. return kref_get_unless_zero(&lport->ref);
  202. }
  203. static struct nvme_fc_lport *
  204. nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
  205. struct nvme_fc_port_template *ops,
  206. struct device *dev)
  207. {
  208. struct nvme_fc_lport *lport;
  209. unsigned long flags;
  210. spin_lock_irqsave(&nvme_fc_lock, flags);
  211. list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
  212. if (lport->localport.node_name != pinfo->node_name ||
  213. lport->localport.port_name != pinfo->port_name)
  214. continue;
  215. if (lport->dev != dev) {
  216. lport = ERR_PTR(-EXDEV);
  217. goto out_done;
  218. }
  219. if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
  220. lport = ERR_PTR(-EEXIST);
  221. goto out_done;
  222. }
  223. if (!nvme_fc_lport_get(lport)) {
  224. /*
  225. * fails if ref cnt already 0. If so,
  226. * act as if lport already deleted
  227. */
  228. lport = NULL;
  229. goto out_done;
  230. }
  231. /* resume the lport */
  232. lport->ops = ops;
  233. lport->localport.port_role = pinfo->port_role;
  234. lport->localport.port_id = pinfo->port_id;
  235. lport->localport.port_state = FC_OBJSTATE_ONLINE;
  236. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  237. return lport;
  238. }
  239. lport = NULL;
  240. out_done:
  241. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  242. return lport;
  243. }
  244. /**
  245. * nvme_fc_register_localport - transport entry point called by an
  246. * LLDD to register the existence of a NVME
  247. * host FC port.
  248. * @pinfo: pointer to information about the port to be registered
  249. * @template: LLDD entrypoints and operational parameters for the port
  250. * @dev: physical hardware device node port corresponds to. Will be
  251. * used for DMA mappings
  252. * @lport_p: pointer to a local port pointer. Upon success, the routine
  253. * will allocate a nvme_fc_local_port structure and place its
  254. * address in the local port pointer. Upon failure, local port
  255. * pointer will be set to 0.
  256. *
  257. * Returns:
  258. * a completion status. Must be 0 upon success; a negative errno
  259. * (ex: -ENXIO) upon failure.
  260. */
  261. int
  262. nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
  263. struct nvme_fc_port_template *template,
  264. struct device *dev,
  265. struct nvme_fc_local_port **portptr)
  266. {
  267. struct nvme_fc_lport *newrec;
  268. unsigned long flags;
  269. int ret, idx;
  270. if (!template->localport_delete || !template->remoteport_delete ||
  271. !template->ls_req || !template->fcp_io ||
  272. !template->ls_abort || !template->fcp_abort ||
  273. !template->max_hw_queues || !template->max_sgl_segments ||
  274. !template->max_dif_sgl_segments || !template->dma_boundary) {
  275. ret = -EINVAL;
  276. goto out_reghost_failed;
  277. }
  278. /*
  279. * look to see if there is already a localport that had been
  280. * deregistered and in the process of waiting for all the
  281. * references to fully be removed. If the references haven't
  282. * expired, we can simply re-enable the localport. Remoteports
  283. * and controller reconnections should resume naturally.
  284. */
  285. newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev);
  286. /* found an lport, but something about its state is bad */
  287. if (IS_ERR(newrec)) {
  288. ret = PTR_ERR(newrec);
  289. goto out_reghost_failed;
  290. /* found existing lport, which was resumed */
  291. } else if (newrec) {
  292. *portptr = &newrec->localport;
  293. return 0;
  294. }
  295. /* nothing found - allocate a new localport struct */
  296. newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
  297. GFP_KERNEL);
  298. if (!newrec) {
  299. ret = -ENOMEM;
  300. goto out_reghost_failed;
  301. }
  302. idx = ida_simple_get(&nvme_fc_local_port_cnt, 0, 0, GFP_KERNEL);
  303. if (idx < 0) {
  304. ret = -ENOSPC;
  305. goto out_fail_kfree;
  306. }
  307. if (!get_device(dev) && dev) {
  308. ret = -ENODEV;
  309. goto out_ida_put;
  310. }
  311. INIT_LIST_HEAD(&newrec->port_list);
  312. INIT_LIST_HEAD(&newrec->endp_list);
  313. kref_init(&newrec->ref);
  314. atomic_set(&newrec->act_rport_cnt, 0);
  315. newrec->ops = template;
  316. newrec->dev = dev;
  317. ida_init(&newrec->endp_cnt);
  318. newrec->localport.private = &newrec[1];
  319. newrec->localport.node_name = pinfo->node_name;
  320. newrec->localport.port_name = pinfo->port_name;
  321. newrec->localport.port_role = pinfo->port_role;
  322. newrec->localport.port_id = pinfo->port_id;
  323. newrec->localport.port_state = FC_OBJSTATE_ONLINE;
  324. newrec->localport.port_num = idx;
  325. spin_lock_irqsave(&nvme_fc_lock, flags);
  326. list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
  327. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  328. if (dev)
  329. dma_set_seg_boundary(dev, template->dma_boundary);
  330. *portptr = &newrec->localport;
  331. return 0;
  332. out_ida_put:
  333. ida_simple_remove(&nvme_fc_local_port_cnt, idx);
  334. out_fail_kfree:
  335. kfree(newrec);
  336. out_reghost_failed:
  337. *portptr = NULL;
  338. return ret;
  339. }
  340. EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
  341. /**
  342. * nvme_fc_unregister_localport - transport entry point called by an
  343. * LLDD to deregister/remove a previously
  344. * registered a NVME host FC port.
  345. * @localport: pointer to the (registered) local port that is to be
  346. * deregistered.
  347. *
  348. * Returns:
  349. * a completion status. Must be 0 upon success; a negative errno
  350. * (ex: -ENXIO) upon failure.
  351. */
  352. int
  353. nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
  354. {
  355. struct nvme_fc_lport *lport = localport_to_lport(portptr);
  356. unsigned long flags;
  357. if (!portptr)
  358. return -EINVAL;
  359. spin_lock_irqsave(&nvme_fc_lock, flags);
  360. if (portptr->port_state != FC_OBJSTATE_ONLINE) {
  361. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  362. return -EINVAL;
  363. }
  364. portptr->port_state = FC_OBJSTATE_DELETED;
  365. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  366. if (atomic_read(&lport->act_rport_cnt) == 0)
  367. lport->ops->localport_delete(&lport->localport);
  368. nvme_fc_lport_put(lport);
  369. return 0;
  370. }
  371. EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
  372. /*
  373. * TRADDR strings, per FC-NVME are fixed format:
  374. * "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
  375. * udev event will only differ by prefix of what field is
  376. * being specified:
  377. * "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
  378. * 19 + 43 + null_fudge = 64 characters
  379. */
  380. #define FCNVME_TRADDR_LENGTH 64
  381. static void
  382. nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
  383. struct nvme_fc_rport *rport)
  384. {
  385. char hostaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_HOST_TRADDR=...*/
  386. char tgtaddr[FCNVME_TRADDR_LENGTH]; /* NVMEFC_TRADDR=...*/
  387. char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
  388. if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
  389. return;
  390. snprintf(hostaddr, sizeof(hostaddr),
  391. "NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
  392. lport->localport.node_name, lport->localport.port_name);
  393. snprintf(tgtaddr, sizeof(tgtaddr),
  394. "NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
  395. rport->remoteport.node_name, rport->remoteport.port_name);
  396. kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp);
  397. }
  398. static void
  399. nvme_fc_free_rport(struct kref *ref)
  400. {
  401. struct nvme_fc_rport *rport =
  402. container_of(ref, struct nvme_fc_rport, ref);
  403. struct nvme_fc_lport *lport =
  404. localport_to_lport(rport->remoteport.localport);
  405. unsigned long flags;
  406. WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
  407. WARN_ON(!list_empty(&rport->ctrl_list));
  408. /* remove from lport list */
  409. spin_lock_irqsave(&nvme_fc_lock, flags);
  410. list_del(&rport->endp_list);
  411. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  412. ida_simple_remove(&lport->endp_cnt, rport->remoteport.port_num);
  413. kfree(rport);
  414. nvme_fc_lport_put(lport);
  415. }
  416. static void
  417. nvme_fc_rport_put(struct nvme_fc_rport *rport)
  418. {
  419. kref_put(&rport->ref, nvme_fc_free_rport);
  420. }
  421. static int
  422. nvme_fc_rport_get(struct nvme_fc_rport *rport)
  423. {
  424. return kref_get_unless_zero(&rport->ref);
  425. }
  426. static void
  427. nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
  428. {
  429. switch (ctrl->ctrl.state) {
  430. case NVME_CTRL_NEW:
  431. case NVME_CTRL_CONNECTING:
  432. /*
  433. * As all reconnects were suppressed, schedule a
  434. * connect.
  435. */
  436. dev_info(ctrl->ctrl.device,
  437. "NVME-FC{%d}: connectivity re-established. "
  438. "Attempting reconnect\n", ctrl->cnum);
  439. queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
  440. break;
  441. case NVME_CTRL_RESETTING:
  442. /*
  443. * Controller is already in the process of terminating the
  444. * association. No need to do anything further. The reconnect
  445. * step will naturally occur after the reset completes.
  446. */
  447. break;
  448. default:
  449. /* no action to take - let it delete */
  450. break;
  451. }
  452. }
  453. static struct nvme_fc_rport *
  454. nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
  455. struct nvme_fc_port_info *pinfo)
  456. {
  457. struct nvme_fc_rport *rport;
  458. struct nvme_fc_ctrl *ctrl;
  459. unsigned long flags;
  460. spin_lock_irqsave(&nvme_fc_lock, flags);
  461. list_for_each_entry(rport, &lport->endp_list, endp_list) {
  462. if (rport->remoteport.node_name != pinfo->node_name ||
  463. rport->remoteport.port_name != pinfo->port_name)
  464. continue;
  465. if (!nvme_fc_rport_get(rport)) {
  466. rport = ERR_PTR(-ENOLCK);
  467. goto out_done;
  468. }
  469. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  470. spin_lock_irqsave(&rport->lock, flags);
  471. /* has it been unregistered */
  472. if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
  473. /* means lldd called us twice */
  474. spin_unlock_irqrestore(&rport->lock, flags);
  475. nvme_fc_rport_put(rport);
  476. return ERR_PTR(-ESTALE);
  477. }
  478. rport->remoteport.port_role = pinfo->port_role;
  479. rport->remoteport.port_id = pinfo->port_id;
  480. rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
  481. rport->dev_loss_end = 0;
  482. /*
  483. * kick off a reconnect attempt on all associations to the
  484. * remote port. A successful reconnects will resume i/o.
  485. */
  486. list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
  487. nvme_fc_resume_controller(ctrl);
  488. spin_unlock_irqrestore(&rport->lock, flags);
  489. return rport;
  490. }
  491. rport = NULL;
  492. out_done:
  493. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  494. return rport;
  495. }
  496. static inline void
  497. __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
  498. struct nvme_fc_port_info *pinfo)
  499. {
  500. if (pinfo->dev_loss_tmo)
  501. rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
  502. else
  503. rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
  504. }
  505. /**
  506. * nvme_fc_register_remoteport - transport entry point called by an
  507. * LLDD to register the existence of a NVME
  508. * subsystem FC port on its fabric.
  509. * @localport: pointer to the (registered) local port that the remote
  510. * subsystem port is connected to.
  511. * @pinfo: pointer to information about the port to be registered
  512. * @rport_p: pointer to a remote port pointer. Upon success, the routine
  513. * will allocate a nvme_fc_remote_port structure and place its
  514. * address in the remote port pointer. Upon failure, remote port
  515. * pointer will be set to 0.
  516. *
  517. * Returns:
  518. * a completion status. Must be 0 upon success; a negative errno
  519. * (ex: -ENXIO) upon failure.
  520. */
  521. int
  522. nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
  523. struct nvme_fc_port_info *pinfo,
  524. struct nvme_fc_remote_port **portptr)
  525. {
  526. struct nvme_fc_lport *lport = localport_to_lport(localport);
  527. struct nvme_fc_rport *newrec;
  528. unsigned long flags;
  529. int ret, idx;
  530. if (!nvme_fc_lport_get(lport)) {
  531. ret = -ESHUTDOWN;
  532. goto out_reghost_failed;
  533. }
  534. /*
  535. * look to see if there is already a remoteport that is waiting
  536. * for a reconnect (within dev_loss_tmo) with the same WWN's.
  537. * If so, transition to it and reconnect.
  538. */
  539. newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
  540. /* found an rport, but something about its state is bad */
  541. if (IS_ERR(newrec)) {
  542. ret = PTR_ERR(newrec);
  543. goto out_lport_put;
  544. /* found existing rport, which was resumed */
  545. } else if (newrec) {
  546. nvme_fc_lport_put(lport);
  547. __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
  548. nvme_fc_signal_discovery_scan(lport, newrec);
  549. *portptr = &newrec->remoteport;
  550. return 0;
  551. }
  552. /* nothing found - allocate a new remoteport struct */
  553. newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
  554. GFP_KERNEL);
  555. if (!newrec) {
  556. ret = -ENOMEM;
  557. goto out_lport_put;
  558. }
  559. idx = ida_simple_get(&lport->endp_cnt, 0, 0, GFP_KERNEL);
  560. if (idx < 0) {
  561. ret = -ENOSPC;
  562. goto out_kfree_rport;
  563. }
  564. INIT_LIST_HEAD(&newrec->endp_list);
  565. INIT_LIST_HEAD(&newrec->ctrl_list);
  566. INIT_LIST_HEAD(&newrec->ls_req_list);
  567. kref_init(&newrec->ref);
  568. atomic_set(&newrec->act_ctrl_cnt, 0);
  569. spin_lock_init(&newrec->lock);
  570. newrec->remoteport.localport = &lport->localport;
  571. newrec->dev = lport->dev;
  572. newrec->lport = lport;
  573. newrec->remoteport.private = &newrec[1];
  574. newrec->remoteport.port_role = pinfo->port_role;
  575. newrec->remoteport.node_name = pinfo->node_name;
  576. newrec->remoteport.port_name = pinfo->port_name;
  577. newrec->remoteport.port_id = pinfo->port_id;
  578. newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
  579. newrec->remoteport.port_num = idx;
  580. __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
  581. spin_lock_irqsave(&nvme_fc_lock, flags);
  582. list_add_tail(&newrec->endp_list, &lport->endp_list);
  583. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  584. nvme_fc_signal_discovery_scan(lport, newrec);
  585. *portptr = &newrec->remoteport;
  586. return 0;
  587. out_kfree_rport:
  588. kfree(newrec);
  589. out_lport_put:
  590. nvme_fc_lport_put(lport);
  591. out_reghost_failed:
  592. *portptr = NULL;
  593. return ret;
  594. }
  595. EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
  596. static int
  597. nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
  598. {
  599. struct nvmefc_ls_req_op *lsop;
  600. unsigned long flags;
  601. restart:
  602. spin_lock_irqsave(&rport->lock, flags);
  603. list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
  604. if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
  605. lsop->flags |= FCOP_FLAGS_TERMIO;
  606. spin_unlock_irqrestore(&rport->lock, flags);
  607. rport->lport->ops->ls_abort(&rport->lport->localport,
  608. &rport->remoteport,
  609. &lsop->ls_req);
  610. goto restart;
  611. }
  612. }
  613. spin_unlock_irqrestore(&rport->lock, flags);
  614. return 0;
  615. }
  616. static void
  617. nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
  618. {
  619. dev_info(ctrl->ctrl.device,
  620. "NVME-FC{%d}: controller connectivity lost. Awaiting "
  621. "Reconnect", ctrl->cnum);
  622. switch (ctrl->ctrl.state) {
  623. case NVME_CTRL_NEW:
  624. case NVME_CTRL_LIVE:
  625. /*
  626. * Schedule a controller reset. The reset will terminate the
  627. * association and schedule the reconnect timer. Reconnects
  628. * will be attempted until either the ctlr_loss_tmo
  629. * (max_retries * connect_delay) expires or the remoteport's
  630. * dev_loss_tmo expires.
  631. */
  632. if (nvme_reset_ctrl(&ctrl->ctrl)) {
  633. dev_warn(ctrl->ctrl.device,
  634. "NVME-FC{%d}: Couldn't schedule reset.\n",
  635. ctrl->cnum);
  636. nvme_delete_ctrl(&ctrl->ctrl);
  637. }
  638. break;
  639. case NVME_CTRL_CONNECTING:
  640. /*
  641. * The association has already been terminated and the
  642. * controller is attempting reconnects. No need to do anything
  643. * futher. Reconnects will be attempted until either the
  644. * ctlr_loss_tmo (max_retries * connect_delay) expires or the
  645. * remoteport's dev_loss_tmo expires.
  646. */
  647. break;
  648. case NVME_CTRL_RESETTING:
  649. /*
  650. * Controller is already in the process of terminating the
  651. * association. No need to do anything further. The reconnect
  652. * step will kick in naturally after the association is
  653. * terminated.
  654. */
  655. break;
  656. case NVME_CTRL_DELETING:
  657. default:
  658. /* no action to take - let it delete */
  659. break;
  660. }
  661. }
  662. /**
  663. * nvme_fc_unregister_remoteport - transport entry point called by an
  664. * LLDD to deregister/remove a previously
  665. * registered a NVME subsystem FC port.
  666. * @remoteport: pointer to the (registered) remote port that is to be
  667. * deregistered.
  668. *
  669. * Returns:
  670. * a completion status. Must be 0 upon success; a negative errno
  671. * (ex: -ENXIO) upon failure.
  672. */
  673. int
  674. nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
  675. {
  676. struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
  677. struct nvme_fc_ctrl *ctrl;
  678. unsigned long flags;
  679. if (!portptr)
  680. return -EINVAL;
  681. spin_lock_irqsave(&rport->lock, flags);
  682. if (portptr->port_state != FC_OBJSTATE_ONLINE) {
  683. spin_unlock_irqrestore(&rport->lock, flags);
  684. return -EINVAL;
  685. }
  686. portptr->port_state = FC_OBJSTATE_DELETED;
  687. rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
  688. list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
  689. /* if dev_loss_tmo==0, dev loss is immediate */
  690. if (!portptr->dev_loss_tmo) {
  691. dev_warn(ctrl->ctrl.device,
  692. "NVME-FC{%d}: controller connectivity lost.\n",
  693. ctrl->cnum);
  694. nvme_delete_ctrl(&ctrl->ctrl);
  695. } else
  696. nvme_fc_ctrl_connectivity_loss(ctrl);
  697. }
  698. spin_unlock_irqrestore(&rport->lock, flags);
  699. nvme_fc_abort_lsops(rport);
  700. if (atomic_read(&rport->act_ctrl_cnt) == 0)
  701. rport->lport->ops->remoteport_delete(portptr);
  702. /*
  703. * release the reference, which will allow, if all controllers
  704. * go away, which should only occur after dev_loss_tmo occurs,
  705. * for the rport to be torn down.
  706. */
  707. nvme_fc_rport_put(rport);
  708. return 0;
  709. }
  710. EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
  711. /**
  712. * nvme_fc_rescan_remoteport - transport entry point called by an
  713. * LLDD to request a nvme device rescan.
  714. * @remoteport: pointer to the (registered) remote port that is to be
  715. * rescanned.
  716. *
  717. * Returns: N/A
  718. */
  719. void
  720. nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
  721. {
  722. struct nvme_fc_rport *rport = remoteport_to_rport(remoteport);
  723. nvme_fc_signal_discovery_scan(rport->lport, rport);
  724. }
  725. EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
  726. int
  727. nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
  728. u32 dev_loss_tmo)
  729. {
  730. struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
  731. unsigned long flags;
  732. spin_lock_irqsave(&rport->lock, flags);
  733. if (portptr->port_state != FC_OBJSTATE_ONLINE) {
  734. spin_unlock_irqrestore(&rport->lock, flags);
  735. return -EINVAL;
  736. }
  737. /* a dev_loss_tmo of 0 (immediate) is allowed to be set */
  738. rport->remoteport.dev_loss_tmo = dev_loss_tmo;
  739. spin_unlock_irqrestore(&rport->lock, flags);
  740. return 0;
  741. }
  742. EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
  743. /* *********************** FC-NVME DMA Handling **************************** */
  744. /*
  745. * The fcloop device passes in a NULL device pointer. Real LLD's will
  746. * pass in a valid device pointer. If NULL is passed to the dma mapping
  747. * routines, depending on the platform, it may or may not succeed, and
  748. * may crash.
  749. *
  750. * As such:
  751. * Wrapper all the dma routines and check the dev pointer.
  752. *
  753. * If simple mappings (return just a dma address, we'll noop them,
  754. * returning a dma address of 0.
  755. *
  756. * On more complex mappings (dma_map_sg), a pseudo routine fills
  757. * in the scatter list, setting all dma addresses to 0.
  758. */
  759. static inline dma_addr_t
  760. fc_dma_map_single(struct device *dev, void *ptr, size_t size,
  761. enum dma_data_direction dir)
  762. {
  763. return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
  764. }
  765. static inline int
  766. fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
  767. {
  768. return dev ? dma_mapping_error(dev, dma_addr) : 0;
  769. }
  770. static inline void
  771. fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
  772. enum dma_data_direction dir)
  773. {
  774. if (dev)
  775. dma_unmap_single(dev, addr, size, dir);
  776. }
  777. static inline void
  778. fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
  779. enum dma_data_direction dir)
  780. {
  781. if (dev)
  782. dma_sync_single_for_cpu(dev, addr, size, dir);
  783. }
  784. static inline void
  785. fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
  786. enum dma_data_direction dir)
  787. {
  788. if (dev)
  789. dma_sync_single_for_device(dev, addr, size, dir);
  790. }
  791. /* pseudo dma_map_sg call */
  792. static int
  793. fc_map_sg(struct scatterlist *sg, int nents)
  794. {
  795. struct scatterlist *s;
  796. int i;
  797. WARN_ON(nents == 0 || sg[0].length == 0);
  798. for_each_sg(sg, s, nents, i) {
  799. s->dma_address = 0L;
  800. #ifdef CONFIG_NEED_SG_DMA_LENGTH
  801. s->dma_length = s->length;
  802. #endif
  803. }
  804. return nents;
  805. }
  806. static inline int
  807. fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
  808. enum dma_data_direction dir)
  809. {
  810. return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
  811. }
  812. static inline void
  813. fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
  814. enum dma_data_direction dir)
  815. {
  816. if (dev)
  817. dma_unmap_sg(dev, sg, nents, dir);
  818. }
  819. /* *********************** FC-NVME LS Handling **************************** */
  820. static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
  821. static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
  822. static void
  823. __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
  824. {
  825. struct nvme_fc_rport *rport = lsop->rport;
  826. struct nvmefc_ls_req *lsreq = &lsop->ls_req;
  827. unsigned long flags;
  828. spin_lock_irqsave(&rport->lock, flags);
  829. if (!lsop->req_queued) {
  830. spin_unlock_irqrestore(&rport->lock, flags);
  831. return;
  832. }
  833. list_del(&lsop->lsreq_list);
  834. lsop->req_queued = false;
  835. spin_unlock_irqrestore(&rport->lock, flags);
  836. fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
  837. (lsreq->rqstlen + lsreq->rsplen),
  838. DMA_BIDIRECTIONAL);
  839. nvme_fc_rport_put(rport);
  840. }
  841. static int
  842. __nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
  843. struct nvmefc_ls_req_op *lsop,
  844. void (*done)(struct nvmefc_ls_req *req, int status))
  845. {
  846. struct nvmefc_ls_req *lsreq = &lsop->ls_req;
  847. unsigned long flags;
  848. int ret = 0;
  849. if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
  850. return -ECONNREFUSED;
  851. if (!nvme_fc_rport_get(rport))
  852. return -ESHUTDOWN;
  853. lsreq->done = done;
  854. lsop->rport = rport;
  855. lsop->req_queued = false;
  856. INIT_LIST_HEAD(&lsop->lsreq_list);
  857. init_completion(&lsop->ls_done);
  858. lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
  859. lsreq->rqstlen + lsreq->rsplen,
  860. DMA_BIDIRECTIONAL);
  861. if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
  862. ret = -EFAULT;
  863. goto out_putrport;
  864. }
  865. lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
  866. spin_lock_irqsave(&rport->lock, flags);
  867. list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
  868. lsop->req_queued = true;
  869. spin_unlock_irqrestore(&rport->lock, flags);
  870. ret = rport->lport->ops->ls_req(&rport->lport->localport,
  871. &rport->remoteport, lsreq);
  872. if (ret)
  873. goto out_unlink;
  874. return 0;
  875. out_unlink:
  876. lsop->ls_error = ret;
  877. spin_lock_irqsave(&rport->lock, flags);
  878. lsop->req_queued = false;
  879. list_del(&lsop->lsreq_list);
  880. spin_unlock_irqrestore(&rport->lock, flags);
  881. fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
  882. (lsreq->rqstlen + lsreq->rsplen),
  883. DMA_BIDIRECTIONAL);
  884. out_putrport:
  885. nvme_fc_rport_put(rport);
  886. return ret;
  887. }
  888. static void
  889. nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
  890. {
  891. struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
  892. lsop->ls_error = status;
  893. complete(&lsop->ls_done);
  894. }
  895. static int
  896. nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
  897. {
  898. struct nvmefc_ls_req *lsreq = &lsop->ls_req;
  899. struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
  900. int ret;
  901. ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
  902. if (!ret) {
  903. /*
  904. * No timeout/not interruptible as we need the struct
  905. * to exist until the lldd calls us back. Thus mandate
  906. * wait until driver calls back. lldd responsible for
  907. * the timeout action
  908. */
  909. wait_for_completion(&lsop->ls_done);
  910. __nvme_fc_finish_ls_req(lsop);
  911. ret = lsop->ls_error;
  912. }
  913. if (ret)
  914. return ret;
  915. /* ACC or RJT payload ? */
  916. if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
  917. return -ENXIO;
  918. return 0;
  919. }
  920. static int
  921. nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
  922. struct nvmefc_ls_req_op *lsop,
  923. void (*done)(struct nvmefc_ls_req *req, int status))
  924. {
  925. /* don't wait for completion */
  926. return __nvme_fc_send_ls_req(rport, lsop, done);
  927. }
  928. /* Validation Error indexes into the string table below */
  929. enum {
  930. VERR_NO_ERROR = 0,
  931. VERR_LSACC = 1,
  932. VERR_LSDESC_RQST = 2,
  933. VERR_LSDESC_RQST_LEN = 3,
  934. VERR_ASSOC_ID = 4,
  935. VERR_ASSOC_ID_LEN = 5,
  936. VERR_CONN_ID = 6,
  937. VERR_CONN_ID_LEN = 7,
  938. VERR_CR_ASSOC = 8,
  939. VERR_CR_ASSOC_ACC_LEN = 9,
  940. VERR_CR_CONN = 10,
  941. VERR_CR_CONN_ACC_LEN = 11,
  942. VERR_DISCONN = 12,
  943. VERR_DISCONN_ACC_LEN = 13,
  944. };
  945. static char *validation_errors[] = {
  946. "OK",
  947. "Not LS_ACC",
  948. "Not LSDESC_RQST",
  949. "Bad LSDESC_RQST Length",
  950. "Not Association ID",
  951. "Bad Association ID Length",
  952. "Not Connection ID",
  953. "Bad Connection ID Length",
  954. "Not CR_ASSOC Rqst",
  955. "Bad CR_ASSOC ACC Length",
  956. "Not CR_CONN Rqst",
  957. "Bad CR_CONN ACC Length",
  958. "Not Disconnect Rqst",
  959. "Bad Disconnect ACC Length",
  960. };
  961. static int
  962. nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
  963. struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
  964. {
  965. struct nvmefc_ls_req_op *lsop;
  966. struct nvmefc_ls_req *lsreq;
  967. struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
  968. struct fcnvme_ls_cr_assoc_acc *assoc_acc;
  969. int ret, fcret = 0;
  970. lsop = kzalloc((sizeof(*lsop) +
  971. ctrl->lport->ops->lsrqst_priv_sz +
  972. sizeof(*assoc_rqst) + sizeof(*assoc_acc)), GFP_KERNEL);
  973. if (!lsop) {
  974. ret = -ENOMEM;
  975. goto out_no_memory;
  976. }
  977. lsreq = &lsop->ls_req;
  978. lsreq->private = (void *)&lsop[1];
  979. assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)
  980. (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz);
  981. assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
  982. assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
  983. assoc_rqst->desc_list_len =
  984. cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
  985. assoc_rqst->assoc_cmd.desc_tag =
  986. cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
  987. assoc_rqst->assoc_cmd.desc_len =
  988. fcnvme_lsdesc_len(
  989. sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
  990. assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
  991. assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
  992. /* Linux supports only Dynamic controllers */
  993. assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
  994. uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
  995. strncpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
  996. min(FCNVME_ASSOC_HOSTNQN_LEN, NVMF_NQN_SIZE));
  997. strncpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
  998. min(FCNVME_ASSOC_SUBNQN_LEN, NVMF_NQN_SIZE));
  999. lsop->queue = queue;
  1000. lsreq->rqstaddr = assoc_rqst;
  1001. lsreq->rqstlen = sizeof(*assoc_rqst);
  1002. lsreq->rspaddr = assoc_acc;
  1003. lsreq->rsplen = sizeof(*assoc_acc);
  1004. lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;
  1005. ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
  1006. if (ret)
  1007. goto out_free_buffer;
  1008. /* process connect LS completion */
  1009. /* validate the ACC response */
  1010. if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
  1011. fcret = VERR_LSACC;
  1012. else if (assoc_acc->hdr.desc_list_len !=
  1013. fcnvme_lsdesc_len(
  1014. sizeof(struct fcnvme_ls_cr_assoc_acc)))
  1015. fcret = VERR_CR_ASSOC_ACC_LEN;
  1016. else if (assoc_acc->hdr.rqst.desc_tag !=
  1017. cpu_to_be32(FCNVME_LSDESC_RQST))
  1018. fcret = VERR_LSDESC_RQST;
  1019. else if (assoc_acc->hdr.rqst.desc_len !=
  1020. fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
  1021. fcret = VERR_LSDESC_RQST_LEN;
  1022. else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
  1023. fcret = VERR_CR_ASSOC;
  1024. else if (assoc_acc->associd.desc_tag !=
  1025. cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
  1026. fcret = VERR_ASSOC_ID;
  1027. else if (assoc_acc->associd.desc_len !=
  1028. fcnvme_lsdesc_len(
  1029. sizeof(struct fcnvme_lsdesc_assoc_id)))
  1030. fcret = VERR_ASSOC_ID_LEN;
  1031. else if (assoc_acc->connectid.desc_tag !=
  1032. cpu_to_be32(FCNVME_LSDESC_CONN_ID))
  1033. fcret = VERR_CONN_ID;
  1034. else if (assoc_acc->connectid.desc_len !=
  1035. fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
  1036. fcret = VERR_CONN_ID_LEN;
  1037. if (fcret) {
  1038. ret = -EBADF;
  1039. dev_err(ctrl->dev,
  1040. "q %d connect failed: %s\n",
  1041. queue->qnum, validation_errors[fcret]);
  1042. } else {
  1043. ctrl->association_id =
  1044. be64_to_cpu(assoc_acc->associd.association_id);
  1045. queue->connection_id =
  1046. be64_to_cpu(assoc_acc->connectid.connection_id);
  1047. set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
  1048. }
  1049. out_free_buffer:
  1050. kfree(lsop);
  1051. out_no_memory:
  1052. if (ret)
  1053. dev_err(ctrl->dev,
  1054. "queue %d connect admin queue failed (%d).\n",
  1055. queue->qnum, ret);
  1056. return ret;
  1057. }
  1058. static int
  1059. nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
  1060. u16 qsize, u16 ersp_ratio)
  1061. {
  1062. struct nvmefc_ls_req_op *lsop;
  1063. struct nvmefc_ls_req *lsreq;
  1064. struct fcnvme_ls_cr_conn_rqst *conn_rqst;
  1065. struct fcnvme_ls_cr_conn_acc *conn_acc;
  1066. int ret, fcret = 0;
  1067. lsop = kzalloc((sizeof(*lsop) +
  1068. ctrl->lport->ops->lsrqst_priv_sz +
  1069. sizeof(*conn_rqst) + sizeof(*conn_acc)), GFP_KERNEL);
  1070. if (!lsop) {
  1071. ret = -ENOMEM;
  1072. goto out_no_memory;
  1073. }
  1074. lsreq = &lsop->ls_req;
  1075. lsreq->private = (void *)&lsop[1];
  1076. conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)
  1077. (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz);
  1078. conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
  1079. conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
  1080. conn_rqst->desc_list_len = cpu_to_be32(
  1081. sizeof(struct fcnvme_lsdesc_assoc_id) +
  1082. sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
  1083. conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
  1084. conn_rqst->associd.desc_len =
  1085. fcnvme_lsdesc_len(
  1086. sizeof(struct fcnvme_lsdesc_assoc_id));
  1087. conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
  1088. conn_rqst->connect_cmd.desc_tag =
  1089. cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
  1090. conn_rqst->connect_cmd.desc_len =
  1091. fcnvme_lsdesc_len(
  1092. sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
  1093. conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
  1094. conn_rqst->connect_cmd.qid = cpu_to_be16(queue->qnum);
  1095. conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
  1096. lsop->queue = queue;
  1097. lsreq->rqstaddr = conn_rqst;
  1098. lsreq->rqstlen = sizeof(*conn_rqst);
  1099. lsreq->rspaddr = conn_acc;
  1100. lsreq->rsplen = sizeof(*conn_acc);
  1101. lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;
  1102. ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
  1103. if (ret)
  1104. goto out_free_buffer;
  1105. /* process connect LS completion */
  1106. /* validate the ACC response */
  1107. if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
  1108. fcret = VERR_LSACC;
  1109. else if (conn_acc->hdr.desc_list_len !=
  1110. fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
  1111. fcret = VERR_CR_CONN_ACC_LEN;
  1112. else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
  1113. fcret = VERR_LSDESC_RQST;
  1114. else if (conn_acc->hdr.rqst.desc_len !=
  1115. fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
  1116. fcret = VERR_LSDESC_RQST_LEN;
  1117. else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
  1118. fcret = VERR_CR_CONN;
  1119. else if (conn_acc->connectid.desc_tag !=
  1120. cpu_to_be32(FCNVME_LSDESC_CONN_ID))
  1121. fcret = VERR_CONN_ID;
  1122. else if (conn_acc->connectid.desc_len !=
  1123. fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
  1124. fcret = VERR_CONN_ID_LEN;
  1125. if (fcret) {
  1126. ret = -EBADF;
  1127. dev_err(ctrl->dev,
  1128. "q %d connect failed: %s\n",
  1129. queue->qnum, validation_errors[fcret]);
  1130. } else {
  1131. queue->connection_id =
  1132. be64_to_cpu(conn_acc->connectid.connection_id);
  1133. set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
  1134. }
  1135. out_free_buffer:
  1136. kfree(lsop);
  1137. out_no_memory:
  1138. if (ret)
  1139. dev_err(ctrl->dev,
  1140. "queue %d connect command failed (%d).\n",
  1141. queue->qnum, ret);
  1142. return ret;
  1143. }
  1144. static void
  1145. nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
  1146. {
  1147. struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
  1148. __nvme_fc_finish_ls_req(lsop);
  1149. /* fc-nvme iniator doesn't care about success or failure of cmd */
  1150. kfree(lsop);
  1151. }
  1152. /*
  1153. * This routine sends a FC-NVME LS to disconnect (aka terminate)
  1154. * the FC-NVME Association. Terminating the association also
  1155. * terminates the FC-NVME connections (per queue, both admin and io
  1156. * queues) that are part of the association. E.g. things are torn
  1157. * down, and the related FC-NVME Association ID and Connection IDs
  1158. * become invalid.
  1159. *
  1160. * The behavior of the fc-nvme initiator is such that it's
  1161. * understanding of the association and connections will implicitly
  1162. * be torn down. The action is implicit as it may be due to a loss of
  1163. * connectivity with the fc-nvme target, so you may never get a
  1164. * response even if you tried. As such, the action of this routine
  1165. * is to asynchronously send the LS, ignore any results of the LS, and
  1166. * continue on with terminating the association. If the fc-nvme target
  1167. * is present and receives the LS, it too can tear down.
  1168. */
  1169. static void
  1170. nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
  1171. {
  1172. struct fcnvme_ls_disconnect_rqst *discon_rqst;
  1173. struct fcnvme_ls_disconnect_acc *discon_acc;
  1174. struct nvmefc_ls_req_op *lsop;
  1175. struct nvmefc_ls_req *lsreq;
  1176. int ret;
  1177. lsop = kzalloc((sizeof(*lsop) +
  1178. ctrl->lport->ops->lsrqst_priv_sz +
  1179. sizeof(*discon_rqst) + sizeof(*discon_acc)),
  1180. GFP_KERNEL);
  1181. if (!lsop)
  1182. /* couldn't sent it... too bad */
  1183. return;
  1184. lsreq = &lsop->ls_req;
  1185. lsreq->private = (void *)&lsop[1];
  1186. discon_rqst = (struct fcnvme_ls_disconnect_rqst *)
  1187. (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz);
  1188. discon_acc = (struct fcnvme_ls_disconnect_acc *)&discon_rqst[1];
  1189. discon_rqst->w0.ls_cmd = FCNVME_LS_DISCONNECT;
  1190. discon_rqst->desc_list_len = cpu_to_be32(
  1191. sizeof(struct fcnvme_lsdesc_assoc_id) +
  1192. sizeof(struct fcnvme_lsdesc_disconn_cmd));
  1193. discon_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
  1194. discon_rqst->associd.desc_len =
  1195. fcnvme_lsdesc_len(
  1196. sizeof(struct fcnvme_lsdesc_assoc_id));
  1197. discon_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
  1198. discon_rqst->discon_cmd.desc_tag = cpu_to_be32(
  1199. FCNVME_LSDESC_DISCONN_CMD);
  1200. discon_rqst->discon_cmd.desc_len =
  1201. fcnvme_lsdesc_len(
  1202. sizeof(struct fcnvme_lsdesc_disconn_cmd));
  1203. discon_rqst->discon_cmd.scope = FCNVME_DISCONN_ASSOCIATION;
  1204. discon_rqst->discon_cmd.id = cpu_to_be64(ctrl->association_id);
  1205. lsreq->rqstaddr = discon_rqst;
  1206. lsreq->rqstlen = sizeof(*discon_rqst);
  1207. lsreq->rspaddr = discon_acc;
  1208. lsreq->rsplen = sizeof(*discon_acc);
  1209. lsreq->timeout = NVME_FC_CONNECT_TIMEOUT_SEC;
  1210. ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
  1211. nvme_fc_disconnect_assoc_done);
  1212. if (ret)
  1213. kfree(lsop);
  1214. /* only meaningful part to terminating the association */
  1215. ctrl->association_id = 0;
  1216. }
  1217. /* *********************** NVME Ctrl Routines **************************** */
  1218. static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
  1219. static void
  1220. __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
  1221. struct nvme_fc_fcp_op *op)
  1222. {
  1223. fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
  1224. sizeof(op->rsp_iu), DMA_FROM_DEVICE);
  1225. fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
  1226. sizeof(op->cmd_iu), DMA_TO_DEVICE);
  1227. atomic_set(&op->state, FCPOP_STATE_UNINIT);
  1228. }
  1229. static void
  1230. nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
  1231. unsigned int hctx_idx)
  1232. {
  1233. struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
  1234. return __nvme_fc_exit_request(set->driver_data, op);
  1235. }
  1236. static int
  1237. __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
  1238. {
  1239. unsigned long flags;
  1240. int opstate;
  1241. spin_lock_irqsave(&ctrl->lock, flags);
  1242. opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
  1243. if (opstate != FCPOP_STATE_ACTIVE)
  1244. atomic_set(&op->state, opstate);
  1245. else if (ctrl->flags & FCCTRL_TERMIO)
  1246. ctrl->iocnt++;
  1247. spin_unlock_irqrestore(&ctrl->lock, flags);
  1248. if (opstate != FCPOP_STATE_ACTIVE)
  1249. return -ECANCELED;
  1250. ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
  1251. &ctrl->rport->remoteport,
  1252. op->queue->lldd_handle,
  1253. &op->fcp_req);
  1254. return 0;
  1255. }
  1256. static void
  1257. nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
  1258. {
  1259. struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
  1260. int i;
  1261. /* ensure we've initialized the ops once */
  1262. if (!(aen_op->flags & FCOP_FLAGS_AEN))
  1263. return;
  1264. for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
  1265. __nvme_fc_abort_op(ctrl, aen_op);
  1266. }
  1267. static inline void
  1268. __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
  1269. struct nvme_fc_fcp_op *op, int opstate)
  1270. {
  1271. unsigned long flags;
  1272. if (opstate == FCPOP_STATE_ABORTED) {
  1273. spin_lock_irqsave(&ctrl->lock, flags);
  1274. if (ctrl->flags & FCCTRL_TERMIO) {
  1275. if (!--ctrl->iocnt)
  1276. wake_up(&ctrl->ioabort_wait);
  1277. }
  1278. spin_unlock_irqrestore(&ctrl->lock, flags);
  1279. }
  1280. }
  1281. static void
  1282. nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
  1283. {
  1284. struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
  1285. struct request *rq = op->rq;
  1286. struct nvmefc_fcp_req *freq = &op->fcp_req;
  1287. struct nvme_fc_ctrl *ctrl = op->ctrl;
  1288. struct nvme_fc_queue *queue = op->queue;
  1289. struct nvme_completion *cqe = &op->rsp_iu.cqe;
  1290. struct nvme_command *sqe = &op->cmd_iu.sqe;
  1291. __le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
  1292. union nvme_result result;
  1293. bool terminate_assoc = true;
  1294. int opstate;
  1295. /*
  1296. * WARNING:
  1297. * The current linux implementation of a nvme controller
  1298. * allocates a single tag set for all io queues and sizes
  1299. * the io queues to fully hold all possible tags. Thus, the
  1300. * implementation does not reference or care about the sqhd
  1301. * value as it never needs to use the sqhd/sqtail pointers
  1302. * for submission pacing.
  1303. *
  1304. * This affects the FC-NVME implementation in two ways:
  1305. * 1) As the value doesn't matter, we don't need to waste
  1306. * cycles extracting it from ERSPs and stamping it in the
  1307. * cases where the transport fabricates CQEs on successful
  1308. * completions.
  1309. * 2) The FC-NVME implementation requires that delivery of
  1310. * ERSP completions are to go back to the nvme layer in order
  1311. * relative to the rsn, such that the sqhd value will always
  1312. * be "in order" for the nvme layer. As the nvme layer in
  1313. * linux doesn't care about sqhd, there's no need to return
  1314. * them in order.
  1315. *
  1316. * Additionally:
  1317. * As the core nvme layer in linux currently does not look at
  1318. * every field in the cqe - in cases where the FC transport must
  1319. * fabricate a CQE, the following fields will not be set as they
  1320. * are not referenced:
  1321. * cqe.sqid, cqe.sqhd, cqe.command_id
  1322. *
  1323. * Failure or error of an individual i/o, in a transport
  1324. * detected fashion unrelated to the nvme completion status,
  1325. * potentially cause the initiator and target sides to get out
  1326. * of sync on SQ head/tail (aka outstanding io count allowed).
  1327. * Per FC-NVME spec, failure of an individual command requires
  1328. * the connection to be terminated, which in turn requires the
  1329. * association to be terminated.
  1330. */
  1331. opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
  1332. fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
  1333. sizeof(op->rsp_iu), DMA_FROM_DEVICE);
  1334. if (opstate == FCPOP_STATE_ABORTED)
  1335. status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);
  1336. else if (freq->status)
  1337. status = cpu_to_le16(NVME_SC_INTERNAL << 1);
  1338. /*
  1339. * For the linux implementation, if we have an unsuccesful
  1340. * status, they blk-mq layer can typically be called with the
  1341. * non-zero status and the content of the cqe isn't important.
  1342. */
  1343. if (status)
  1344. goto done;
  1345. /*
  1346. * command completed successfully relative to the wire
  1347. * protocol. However, validate anything received and
  1348. * extract the status and result from the cqe (create it
  1349. * where necessary).
  1350. */
  1351. switch (freq->rcv_rsplen) {
  1352. case 0:
  1353. case NVME_FC_SIZEOF_ZEROS_RSP:
  1354. /*
  1355. * No response payload or 12 bytes of payload (which
  1356. * should all be zeros) are considered successful and
  1357. * no payload in the CQE by the transport.
  1358. */
  1359. if (freq->transferred_length !=
  1360. be32_to_cpu(op->cmd_iu.data_len)) {
  1361. status = cpu_to_le16(NVME_SC_INTERNAL << 1);
  1362. goto done;
  1363. }
  1364. result.u64 = 0;
  1365. break;
  1366. case sizeof(struct nvme_fc_ersp_iu):
  1367. /*
  1368. * The ERSP IU contains a full completion with CQE.
  1369. * Validate ERSP IU and look at cqe.
  1370. */
  1371. if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
  1372. (freq->rcv_rsplen / 4) ||
  1373. be32_to_cpu(op->rsp_iu.xfrd_len) !=
  1374. freq->transferred_length ||
  1375. op->rsp_iu.status_code ||
  1376. sqe->common.command_id != cqe->command_id)) {
  1377. status = cpu_to_le16(NVME_SC_INTERNAL << 1);
  1378. goto done;
  1379. }
  1380. result = cqe->result;
  1381. status = cqe->status;
  1382. break;
  1383. default:
  1384. status = cpu_to_le16(NVME_SC_INTERNAL << 1);
  1385. goto done;
  1386. }
  1387. terminate_assoc = false;
  1388. done:
  1389. if (op->flags & FCOP_FLAGS_AEN) {
  1390. nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
  1391. __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
  1392. atomic_set(&op->state, FCPOP_STATE_IDLE);
  1393. op->flags = FCOP_FLAGS_AEN; /* clear other flags */
  1394. nvme_fc_ctrl_put(ctrl);
  1395. goto check_error;
  1396. }
  1397. __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
  1398. nvme_end_request(rq, status, result);
  1399. check_error:
  1400. if (terminate_assoc)
  1401. nvme_fc_error_recovery(ctrl, "transport detected io error");
  1402. }
  1403. static int
  1404. __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
  1405. struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
  1406. struct request *rq, u32 rqno)
  1407. {
  1408. struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
  1409. int ret = 0;
  1410. memset(op, 0, sizeof(*op));
  1411. op->fcp_req.cmdaddr = &op->cmd_iu;
  1412. op->fcp_req.cmdlen = sizeof(op->cmd_iu);
  1413. op->fcp_req.rspaddr = &op->rsp_iu;
  1414. op->fcp_req.rsplen = sizeof(op->rsp_iu);
  1415. op->fcp_req.done = nvme_fc_fcpio_done;
  1416. op->fcp_req.first_sgl = (struct scatterlist *)&op[1];
  1417. op->fcp_req.private = &op->fcp_req.first_sgl[SG_CHUNK_SIZE];
  1418. op->ctrl = ctrl;
  1419. op->queue = queue;
  1420. op->rq = rq;
  1421. op->rqno = rqno;
  1422. cmdiu->scsi_id = NVME_CMD_SCSI_ID;
  1423. cmdiu->fc_id = NVME_CMD_FC_ID;
  1424. cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
  1425. op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
  1426. &op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
  1427. if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
  1428. dev_err(ctrl->dev,
  1429. "FCP Op failed - cmdiu dma mapping failed.\n");
  1430. ret = -EFAULT;
  1431. goto out_on_error;
  1432. }
  1433. op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
  1434. &op->rsp_iu, sizeof(op->rsp_iu),
  1435. DMA_FROM_DEVICE);
  1436. if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
  1437. dev_err(ctrl->dev,
  1438. "FCP Op failed - rspiu dma mapping failed.\n");
  1439. ret = -EFAULT;
  1440. }
  1441. atomic_set(&op->state, FCPOP_STATE_IDLE);
  1442. out_on_error:
  1443. return ret;
  1444. }
  1445. static int
  1446. nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
  1447. unsigned int hctx_idx, unsigned int numa_node)
  1448. {
  1449. struct nvme_fc_ctrl *ctrl = set->driver_data;
  1450. struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
  1451. int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
  1452. struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
  1453. nvme_req(rq)->ctrl = &ctrl->ctrl;
  1454. return __nvme_fc_init_request(ctrl, queue, op, rq, queue->rqcnt++);
  1455. }
  1456. static int
  1457. nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
  1458. {
  1459. struct nvme_fc_fcp_op *aen_op;
  1460. struct nvme_fc_cmd_iu *cmdiu;
  1461. struct nvme_command *sqe;
  1462. void *private;
  1463. int i, ret;
  1464. aen_op = ctrl->aen_ops;
  1465. for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
  1466. private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
  1467. GFP_KERNEL);
  1468. if (!private)
  1469. return -ENOMEM;
  1470. cmdiu = &aen_op->cmd_iu;
  1471. sqe = &cmdiu->sqe;
  1472. ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
  1473. aen_op, (struct request *)NULL,
  1474. (NVME_AQ_BLK_MQ_DEPTH + i));
  1475. if (ret) {
  1476. kfree(private);
  1477. return ret;
  1478. }
  1479. aen_op->flags = FCOP_FLAGS_AEN;
  1480. aen_op->fcp_req.first_sgl = NULL; /* no sg list */
  1481. aen_op->fcp_req.private = private;
  1482. memset(sqe, 0, sizeof(*sqe));
  1483. sqe->common.opcode = nvme_admin_async_event;
  1484. /* Note: core layer may overwrite the sqe.command_id value */
  1485. sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
  1486. }
  1487. return 0;
  1488. }
  1489. static void
  1490. nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
  1491. {
  1492. struct nvme_fc_fcp_op *aen_op;
  1493. int i;
  1494. cancel_work_sync(&ctrl->ctrl.async_event_work);
  1495. aen_op = ctrl->aen_ops;
  1496. for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
  1497. if (!aen_op->fcp_req.private)
  1498. continue;
  1499. __nvme_fc_exit_request(ctrl, aen_op);
  1500. kfree(aen_op->fcp_req.private);
  1501. aen_op->fcp_req.private = NULL;
  1502. }
  1503. }
  1504. static inline void
  1505. __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, struct nvme_fc_ctrl *ctrl,
  1506. unsigned int qidx)
  1507. {
  1508. struct nvme_fc_queue *queue = &ctrl->queues[qidx];
  1509. hctx->driver_data = queue;
  1510. queue->hctx = hctx;
  1511. }
  1512. static int
  1513. nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
  1514. unsigned int hctx_idx)
  1515. {
  1516. struct nvme_fc_ctrl *ctrl = data;
  1517. __nvme_fc_init_hctx(hctx, ctrl, hctx_idx + 1);
  1518. return 0;
  1519. }
  1520. static int
  1521. nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
  1522. unsigned int hctx_idx)
  1523. {
  1524. struct nvme_fc_ctrl *ctrl = data;
  1525. __nvme_fc_init_hctx(hctx, ctrl, hctx_idx);
  1526. return 0;
  1527. }
  1528. static void
  1529. nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
  1530. {
  1531. struct nvme_fc_queue *queue;
  1532. queue = &ctrl->queues[idx];
  1533. memset(queue, 0, sizeof(*queue));
  1534. queue->ctrl = ctrl;
  1535. queue->qnum = idx;
  1536. atomic_set(&queue->csn, 0);
  1537. queue->dev = ctrl->dev;
  1538. if (idx > 0)
  1539. queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
  1540. else
  1541. queue->cmnd_capsule_len = sizeof(struct nvme_command);
  1542. /*
  1543. * Considered whether we should allocate buffers for all SQEs
  1544. * and CQEs and dma map them - mapping their respective entries
  1545. * into the request structures (kernel vm addr and dma address)
  1546. * thus the driver could use the buffers/mappings directly.
  1547. * It only makes sense if the LLDD would use them for its
  1548. * messaging api. It's very unlikely most adapter api's would use
  1549. * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
  1550. * structures were used instead.
  1551. */
  1552. }
  1553. /*
  1554. * This routine terminates a queue at the transport level.
  1555. * The transport has already ensured that all outstanding ios on
  1556. * the queue have been terminated.
  1557. * The transport will send a Disconnect LS request to terminate
  1558. * the queue's connection. Termination of the admin queue will also
  1559. * terminate the association at the target.
  1560. */
  1561. static void
  1562. nvme_fc_free_queue(struct nvme_fc_queue *queue)
  1563. {
  1564. if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
  1565. return;
  1566. clear_bit(NVME_FC_Q_LIVE, &queue->flags);
  1567. /*
  1568. * Current implementation never disconnects a single queue.
  1569. * It always terminates a whole association. So there is never
  1570. * a disconnect(queue) LS sent to the target.
  1571. */
  1572. queue->connection_id = 0;
  1573. atomic_set(&queue->csn, 0);
  1574. }
  1575. static void
  1576. __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
  1577. struct nvme_fc_queue *queue, unsigned int qidx)
  1578. {
  1579. if (ctrl->lport->ops->delete_queue)
  1580. ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
  1581. queue->lldd_handle);
  1582. queue->lldd_handle = NULL;
  1583. }
  1584. static void
  1585. nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
  1586. {
  1587. int i;
  1588. for (i = 1; i < ctrl->ctrl.queue_count; i++)
  1589. nvme_fc_free_queue(&ctrl->queues[i]);
  1590. }
  1591. static int
  1592. __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
  1593. struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
  1594. {
  1595. int ret = 0;
  1596. queue->lldd_handle = NULL;
  1597. if (ctrl->lport->ops->create_queue)
  1598. ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
  1599. qidx, qsize, &queue->lldd_handle);
  1600. return ret;
  1601. }
  1602. static void
  1603. nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
  1604. {
  1605. struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
  1606. int i;
  1607. for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
  1608. __nvme_fc_delete_hw_queue(ctrl, queue, i);
  1609. }
  1610. static int
  1611. nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
  1612. {
  1613. struct nvme_fc_queue *queue = &ctrl->queues[1];
  1614. int i, ret;
  1615. for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
  1616. ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
  1617. if (ret)
  1618. goto delete_queues;
  1619. }
  1620. return 0;
  1621. delete_queues:
  1622. for (; i >= 0; i--)
  1623. __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i);
  1624. return ret;
  1625. }
  1626. static int
  1627. nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
  1628. {
  1629. int i, ret = 0;
  1630. for (i = 1; i < ctrl->ctrl.queue_count; i++) {
  1631. ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
  1632. (qsize / 5));
  1633. if (ret)
  1634. break;
  1635. ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
  1636. if (ret)
  1637. break;
  1638. set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags);
  1639. }
  1640. return ret;
  1641. }
  1642. static void
  1643. nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
  1644. {
  1645. int i;
  1646. for (i = 1; i < ctrl->ctrl.queue_count; i++)
  1647. nvme_fc_init_queue(ctrl, i);
  1648. }
  1649. static void
  1650. nvme_fc_ctrl_free(struct kref *ref)
  1651. {
  1652. struct nvme_fc_ctrl *ctrl =
  1653. container_of(ref, struct nvme_fc_ctrl, ref);
  1654. unsigned long flags;
  1655. if (ctrl->ctrl.tagset) {
  1656. blk_cleanup_queue(ctrl->ctrl.connect_q);
  1657. blk_mq_free_tag_set(&ctrl->tag_set);
  1658. }
  1659. /* remove from rport list */
  1660. spin_lock_irqsave(&ctrl->rport->lock, flags);
  1661. list_del(&ctrl->ctrl_list);
  1662. spin_unlock_irqrestore(&ctrl->rport->lock, flags);
  1663. blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
  1664. blk_cleanup_queue(ctrl->ctrl.admin_q);
  1665. blk_mq_free_tag_set(&ctrl->admin_tag_set);
  1666. kfree(ctrl->queues);
  1667. put_device(ctrl->dev);
  1668. nvme_fc_rport_put(ctrl->rport);
  1669. ida_simple_remove(&nvme_fc_ctrl_cnt, ctrl->cnum);
  1670. if (ctrl->ctrl.opts)
  1671. nvmf_free_options(ctrl->ctrl.opts);
  1672. kfree(ctrl);
  1673. }
  1674. static void
  1675. nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
  1676. {
  1677. kref_put(&ctrl->ref, nvme_fc_ctrl_free);
  1678. }
  1679. static int
  1680. nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
  1681. {
  1682. return kref_get_unless_zero(&ctrl->ref);
  1683. }
  1684. /*
  1685. * All accesses from nvme core layer done - can now free the
  1686. * controller. Called after last nvme_put_ctrl() call
  1687. */
  1688. static void
  1689. nvme_fc_nvme_ctrl_freed(struct nvme_ctrl *nctrl)
  1690. {
  1691. struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
  1692. WARN_ON(nctrl != &ctrl->ctrl);
  1693. nvme_fc_ctrl_put(ctrl);
  1694. }
  1695. static void
  1696. nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
  1697. {
  1698. int active;
  1699. /*
  1700. * if an error (io timeout, etc) while (re)connecting,
  1701. * it's an error on creating the new association.
  1702. * Start the error recovery thread if it hasn't already
  1703. * been started. It is expected there could be multiple
  1704. * ios hitting this path before things are cleaned up.
  1705. */
  1706. if (ctrl->ctrl.state == NVME_CTRL_CONNECTING) {
  1707. active = atomic_xchg(&ctrl->err_work_active, 1);
  1708. if (!active && !queue_work(nvme_fc_wq, &ctrl->err_work)) {
  1709. atomic_set(&ctrl->err_work_active, 0);
  1710. WARN_ON(1);
  1711. }
  1712. return;
  1713. }
  1714. /* Otherwise, only proceed if in LIVE state - e.g. on first error */
  1715. if (ctrl->ctrl.state != NVME_CTRL_LIVE)
  1716. return;
  1717. dev_warn(ctrl->ctrl.device,
  1718. "NVME-FC{%d}: transport association error detected: %s\n",
  1719. ctrl->cnum, errmsg);
  1720. dev_warn(ctrl->ctrl.device,
  1721. "NVME-FC{%d}: resetting controller\n", ctrl->cnum);
  1722. nvme_reset_ctrl(&ctrl->ctrl);
  1723. }
  1724. static enum blk_eh_timer_return
  1725. nvme_fc_timeout(struct request *rq, bool reserved)
  1726. {
  1727. struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
  1728. struct nvme_fc_ctrl *ctrl = op->ctrl;
  1729. /*
  1730. * we can't individually ABTS an io without affecting the queue,
  1731. * thus killing the queue, and thus the association.
  1732. * So resolve by performing a controller reset, which will stop
  1733. * the host/io stack, terminate the association on the link,
  1734. * and recreate an association on the link.
  1735. */
  1736. nvme_fc_error_recovery(ctrl, "io timeout error");
  1737. /*
  1738. * the io abort has been initiated. Have the reset timer
  1739. * restarted and the abort completion will complete the io
  1740. * shortly. Avoids a synchronous wait while the abort finishes.
  1741. */
  1742. return BLK_EH_RESET_TIMER;
  1743. }
  1744. static int
  1745. nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
  1746. struct nvme_fc_fcp_op *op)
  1747. {
  1748. struct nvmefc_fcp_req *freq = &op->fcp_req;
  1749. enum dma_data_direction dir;
  1750. int ret;
  1751. freq->sg_cnt = 0;
  1752. if (!blk_rq_payload_bytes(rq))
  1753. return 0;
  1754. freq->sg_table.sgl = freq->first_sgl;
  1755. ret = sg_alloc_table_chained(&freq->sg_table,
  1756. blk_rq_nr_phys_segments(rq), freq->sg_table.sgl);
  1757. if (ret)
  1758. return -ENOMEM;
  1759. op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl);
  1760. WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
  1761. dir = (rq_data_dir(rq) == WRITE) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
  1762. freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl,
  1763. op->nents, dir);
  1764. if (unlikely(freq->sg_cnt <= 0)) {
  1765. sg_free_table_chained(&freq->sg_table, true);
  1766. freq->sg_cnt = 0;
  1767. return -EFAULT;
  1768. }
  1769. /*
  1770. * TODO: blk_integrity_rq(rq) for DIF
  1771. */
  1772. return 0;
  1773. }
  1774. static void
  1775. nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
  1776. struct nvme_fc_fcp_op *op)
  1777. {
  1778. struct nvmefc_fcp_req *freq = &op->fcp_req;
  1779. if (!freq->sg_cnt)
  1780. return;
  1781. fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents,
  1782. ((rq_data_dir(rq) == WRITE) ?
  1783. DMA_TO_DEVICE : DMA_FROM_DEVICE));
  1784. nvme_cleanup_cmd(rq);
  1785. sg_free_table_chained(&freq->sg_table, true);
  1786. freq->sg_cnt = 0;
  1787. }
  1788. /*
  1789. * In FC, the queue is a logical thing. At transport connect, the target
  1790. * creates its "queue" and returns a handle that is to be given to the
  1791. * target whenever it posts something to the corresponding SQ. When an
  1792. * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
  1793. * command contained within the SQE, an io, and assigns a FC exchange
  1794. * to it. The SQE and the associated SQ handle are sent in the initial
  1795. * CMD IU sents on the exchange. All transfers relative to the io occur
  1796. * as part of the exchange. The CQE is the last thing for the io,
  1797. * which is transferred (explicitly or implicitly) with the RSP IU
  1798. * sent on the exchange. After the CQE is received, the FC exchange is
  1799. * terminaed and the Exchange may be used on a different io.
  1800. *
  1801. * The transport to LLDD api has the transport making a request for a
  1802. * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
  1803. * resource and transfers the command. The LLDD will then process all
  1804. * steps to complete the io. Upon completion, the transport done routine
  1805. * is called.
  1806. *
  1807. * So - while the operation is outstanding to the LLDD, there is a link
  1808. * level FC exchange resource that is also outstanding. This must be
  1809. * considered in all cleanup operations.
  1810. */
  1811. static blk_status_t
  1812. nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
  1813. struct nvme_fc_fcp_op *op, u32 data_len,
  1814. enum nvmefc_fcp_datadir io_dir)
  1815. {
  1816. struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
  1817. struct nvme_command *sqe = &cmdiu->sqe;
  1818. int ret, opstate;
  1819. /*
  1820. * before attempting to send the io, check to see if we believe
  1821. * the target device is present
  1822. */
  1823. if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
  1824. return BLK_STS_RESOURCE;
  1825. if (!nvme_fc_ctrl_get(ctrl))
  1826. return BLK_STS_IOERR;
  1827. /* format the FC-NVME CMD IU and fcp_req */
  1828. cmdiu->connection_id = cpu_to_be64(queue->connection_id);
  1829. cmdiu->data_len = cpu_to_be32(data_len);
  1830. switch (io_dir) {
  1831. case NVMEFC_FCP_WRITE:
  1832. cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
  1833. break;
  1834. case NVMEFC_FCP_READ:
  1835. cmdiu->flags = FCNVME_CMD_FLAGS_READ;
  1836. break;
  1837. case NVMEFC_FCP_NODATA:
  1838. cmdiu->flags = 0;
  1839. break;
  1840. }
  1841. op->fcp_req.payload_length = data_len;
  1842. op->fcp_req.io_dir = io_dir;
  1843. op->fcp_req.transferred_length = 0;
  1844. op->fcp_req.rcv_rsplen = 0;
  1845. op->fcp_req.status = NVME_SC_SUCCESS;
  1846. op->fcp_req.sqid = cpu_to_le16(queue->qnum);
  1847. /*
  1848. * validate per fabric rules, set fields mandated by fabric spec
  1849. * as well as those by FC-NVME spec.
  1850. */
  1851. WARN_ON_ONCE(sqe->common.metadata);
  1852. sqe->common.flags |= NVME_CMD_SGL_METABUF;
  1853. /*
  1854. * format SQE DPTR field per FC-NVME rules:
  1855. * type=0x5 Transport SGL Data Block Descriptor
  1856. * subtype=0xA Transport-specific value
  1857. * address=0
  1858. * length=length of the data series
  1859. */
  1860. sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
  1861. NVME_SGL_FMT_TRANSPORT_A;
  1862. sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
  1863. sqe->rw.dptr.sgl.addr = 0;
  1864. if (!(op->flags & FCOP_FLAGS_AEN)) {
  1865. ret = nvme_fc_map_data(ctrl, op->rq, op);
  1866. if (ret < 0) {
  1867. nvme_cleanup_cmd(op->rq);
  1868. nvme_fc_ctrl_put(ctrl);
  1869. if (ret == -ENOMEM || ret == -EAGAIN)
  1870. return BLK_STS_RESOURCE;
  1871. return BLK_STS_IOERR;
  1872. }
  1873. }
  1874. fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma,
  1875. sizeof(op->cmd_iu), DMA_TO_DEVICE);
  1876. atomic_set(&op->state, FCPOP_STATE_ACTIVE);
  1877. if (!(op->flags & FCOP_FLAGS_AEN))
  1878. blk_mq_start_request(op->rq);
  1879. cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
  1880. ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
  1881. &ctrl->rport->remoteport,
  1882. queue->lldd_handle, &op->fcp_req);
  1883. if (ret) {
  1884. /*
  1885. * If the lld fails to send the command is there an issue with
  1886. * the csn value? If the command that fails is the Connect,
  1887. * no - as the connection won't be live. If it is a command
  1888. * post-connect, it's possible a gap in csn may be created.
  1889. * Does this matter? As Linux initiators don't send fused
  1890. * commands, no. The gap would exist, but as there's nothing
  1891. * that depends on csn order to be delivered on the target
  1892. * side, it shouldn't hurt. It would be difficult for a
  1893. * target to even detect the csn gap as it has no idea when the
  1894. * cmd with the csn was supposed to arrive.
  1895. */
  1896. opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
  1897. __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
  1898. if (!(op->flags & FCOP_FLAGS_AEN))
  1899. nvme_fc_unmap_data(ctrl, op->rq, op);
  1900. nvme_fc_ctrl_put(ctrl);
  1901. if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
  1902. ret != -EBUSY)
  1903. return BLK_STS_IOERR;
  1904. return BLK_STS_RESOURCE;
  1905. }
  1906. return BLK_STS_OK;
  1907. }
  1908. static blk_status_t
  1909. nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
  1910. const struct blk_mq_queue_data *bd)
  1911. {
  1912. struct nvme_ns *ns = hctx->queue->queuedata;
  1913. struct nvme_fc_queue *queue = hctx->driver_data;
  1914. struct nvme_fc_ctrl *ctrl = queue->ctrl;
  1915. struct request *rq = bd->rq;
  1916. struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
  1917. struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
  1918. struct nvme_command *sqe = &cmdiu->sqe;
  1919. enum nvmefc_fcp_datadir io_dir;
  1920. bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
  1921. u32 data_len;
  1922. blk_status_t ret;
  1923. if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
  1924. !nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
  1925. return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq);
  1926. ret = nvme_setup_cmd(ns, rq, sqe);
  1927. if (ret)
  1928. return ret;
  1929. data_len = blk_rq_payload_bytes(rq);
  1930. if (data_len)
  1931. io_dir = ((rq_data_dir(rq) == WRITE) ?
  1932. NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
  1933. else
  1934. io_dir = NVMEFC_FCP_NODATA;
  1935. return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
  1936. }
  1937. static struct blk_mq_tags *
  1938. nvme_fc_tagset(struct nvme_fc_queue *queue)
  1939. {
  1940. if (queue->qnum == 0)
  1941. return queue->ctrl->admin_tag_set.tags[queue->qnum];
  1942. return queue->ctrl->tag_set.tags[queue->qnum - 1];
  1943. }
  1944. static int
  1945. nvme_fc_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
  1946. {
  1947. struct nvme_fc_queue *queue = hctx->driver_data;
  1948. struct nvme_fc_ctrl *ctrl = queue->ctrl;
  1949. struct request *req;
  1950. struct nvme_fc_fcp_op *op;
  1951. req = blk_mq_tag_to_rq(nvme_fc_tagset(queue), tag);
  1952. if (!req)
  1953. return 0;
  1954. op = blk_mq_rq_to_pdu(req);
  1955. if ((atomic_read(&op->state) == FCPOP_STATE_ACTIVE) &&
  1956. (ctrl->lport->ops->poll_queue))
  1957. ctrl->lport->ops->poll_queue(&ctrl->lport->localport,
  1958. queue->lldd_handle);
  1959. return ((atomic_read(&op->state) != FCPOP_STATE_ACTIVE));
  1960. }
  1961. static void
  1962. nvme_fc_submit_async_event(struct nvme_ctrl *arg)
  1963. {
  1964. struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg);
  1965. struct nvme_fc_fcp_op *aen_op;
  1966. unsigned long flags;
  1967. bool terminating = false;
  1968. blk_status_t ret;
  1969. spin_lock_irqsave(&ctrl->lock, flags);
  1970. if (ctrl->flags & FCCTRL_TERMIO)
  1971. terminating = true;
  1972. spin_unlock_irqrestore(&ctrl->lock, flags);
  1973. if (terminating)
  1974. return;
  1975. aen_op = &ctrl->aen_ops[0];
  1976. ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0,
  1977. NVMEFC_FCP_NODATA);
  1978. if (ret)
  1979. dev_err(ctrl->ctrl.device,
  1980. "failed async event work\n");
  1981. }
  1982. static void
  1983. nvme_fc_complete_rq(struct request *rq)
  1984. {
  1985. struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
  1986. struct nvme_fc_ctrl *ctrl = op->ctrl;
  1987. atomic_set(&op->state, FCPOP_STATE_IDLE);
  1988. nvme_fc_unmap_data(ctrl, rq, op);
  1989. nvme_complete_rq(rq);
  1990. nvme_fc_ctrl_put(ctrl);
  1991. }
  1992. /*
  1993. * This routine is used by the transport when it needs to find active
  1994. * io on a queue that is to be terminated. The transport uses
  1995. * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
  1996. * this routine to kill them on a 1 by 1 basis.
  1997. *
  1998. * As FC allocates FC exchange for each io, the transport must contact
  1999. * the LLDD to terminate the exchange, thus releasing the FC exchange.
  2000. * After terminating the exchange the LLDD will call the transport's
  2001. * normal io done path for the request, but it will have an aborted
  2002. * status. The done path will return the io request back to the block
  2003. * layer with an error status.
  2004. */
  2005. static void
  2006. nvme_fc_terminate_exchange(struct request *req, void *data, bool reserved)
  2007. {
  2008. struct nvme_ctrl *nctrl = data;
  2009. struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
  2010. struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
  2011. __nvme_fc_abort_op(ctrl, op);
  2012. }
  2013. static const struct blk_mq_ops nvme_fc_mq_ops = {
  2014. .queue_rq = nvme_fc_queue_rq,
  2015. .complete = nvme_fc_complete_rq,
  2016. .init_request = nvme_fc_init_request,
  2017. .exit_request = nvme_fc_exit_request,
  2018. .init_hctx = nvme_fc_init_hctx,
  2019. .poll = nvme_fc_poll,
  2020. .timeout = nvme_fc_timeout,
  2021. };
  2022. static int
  2023. nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
  2024. {
  2025. struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
  2026. unsigned int nr_io_queues;
  2027. int ret;
  2028. nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
  2029. ctrl->lport->ops->max_hw_queues);
  2030. ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
  2031. if (ret) {
  2032. dev_info(ctrl->ctrl.device,
  2033. "set_queue_count failed: %d\n", ret);
  2034. return ret;
  2035. }
  2036. ctrl->ctrl.queue_count = nr_io_queues + 1;
  2037. if (!nr_io_queues)
  2038. return 0;
  2039. nvme_fc_init_io_queues(ctrl);
  2040. memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
  2041. ctrl->tag_set.ops = &nvme_fc_mq_ops;
  2042. ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
  2043. ctrl->tag_set.reserved_tags = 1; /* fabric connect */
  2044. ctrl->tag_set.numa_node = NUMA_NO_NODE;
  2045. ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
  2046. ctrl->tag_set.cmd_size = sizeof(struct nvme_fc_fcp_op) +
  2047. (SG_CHUNK_SIZE *
  2048. sizeof(struct scatterlist)) +
  2049. ctrl->lport->ops->fcprqst_priv_sz;
  2050. ctrl->tag_set.driver_data = ctrl;
  2051. ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;
  2052. ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
  2053. ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
  2054. if (ret)
  2055. return ret;
  2056. ctrl->ctrl.tagset = &ctrl->tag_set;
  2057. ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
  2058. if (IS_ERR(ctrl->ctrl.connect_q)) {
  2059. ret = PTR_ERR(ctrl->ctrl.connect_q);
  2060. goto out_free_tag_set;
  2061. }
  2062. ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
  2063. if (ret)
  2064. goto out_cleanup_blk_queue;
  2065. ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
  2066. if (ret)
  2067. goto out_delete_hw_queues;
  2068. ctrl->ioq_live = true;
  2069. return 0;
  2070. out_delete_hw_queues:
  2071. nvme_fc_delete_hw_io_queues(ctrl);
  2072. out_cleanup_blk_queue:
  2073. blk_cleanup_queue(ctrl->ctrl.connect_q);
  2074. out_free_tag_set:
  2075. blk_mq_free_tag_set(&ctrl->tag_set);
  2076. nvme_fc_free_io_queues(ctrl);
  2077. /* force put free routine to ignore io queues */
  2078. ctrl->ctrl.tagset = NULL;
  2079. return ret;
  2080. }
  2081. static int
  2082. nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
  2083. {
  2084. struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
  2085. unsigned int nr_io_queues;
  2086. int ret;
  2087. nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
  2088. ctrl->lport->ops->max_hw_queues);
  2089. ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
  2090. if (ret) {
  2091. dev_info(ctrl->ctrl.device,
  2092. "set_queue_count failed: %d\n", ret);
  2093. return ret;
  2094. }
  2095. ctrl->ctrl.queue_count = nr_io_queues + 1;
  2096. /* check for io queues existing */
  2097. if (ctrl->ctrl.queue_count == 1)
  2098. return 0;
  2099. ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
  2100. if (ret)
  2101. goto out_free_io_queues;
  2102. ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
  2103. if (ret)
  2104. goto out_delete_hw_queues;
  2105. blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues);
  2106. return 0;
  2107. out_delete_hw_queues:
  2108. nvme_fc_delete_hw_io_queues(ctrl);
  2109. out_free_io_queues:
  2110. nvme_fc_free_io_queues(ctrl);
  2111. return ret;
  2112. }
  2113. static void
  2114. nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
  2115. {
  2116. struct nvme_fc_lport *lport = rport->lport;
  2117. atomic_inc(&lport->act_rport_cnt);
  2118. }
  2119. static void
  2120. nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
  2121. {
  2122. struct nvme_fc_lport *lport = rport->lport;
  2123. u32 cnt;
  2124. cnt = atomic_dec_return(&lport->act_rport_cnt);
  2125. if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
  2126. lport->ops->localport_delete(&lport->localport);
  2127. }
  2128. static int
  2129. nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
  2130. {
  2131. struct nvme_fc_rport *rport = ctrl->rport;
  2132. u32 cnt;
  2133. if (ctrl->assoc_active)
  2134. return 1;
  2135. ctrl->assoc_active = true;
  2136. cnt = atomic_inc_return(&rport->act_ctrl_cnt);
  2137. if (cnt == 1)
  2138. nvme_fc_rport_active_on_lport(rport);
  2139. return 0;
  2140. }
  2141. static int
  2142. nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
  2143. {
  2144. struct nvme_fc_rport *rport = ctrl->rport;
  2145. struct nvme_fc_lport *lport = rport->lport;
  2146. u32 cnt;
  2147. /* ctrl->assoc_active=false will be set independently */
  2148. cnt = atomic_dec_return(&rport->act_ctrl_cnt);
  2149. if (cnt == 0) {
  2150. if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
  2151. lport->ops->remoteport_delete(&rport->remoteport);
  2152. nvme_fc_rport_inactive_on_lport(rport);
  2153. }
  2154. return 0;
  2155. }
  2156. /*
  2157. * This routine restarts the controller on the host side, and
  2158. * on the link side, recreates the controller association.
  2159. */
  2160. static int
  2161. nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
  2162. {
  2163. struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
  2164. int ret;
  2165. bool changed;
  2166. ++ctrl->ctrl.nr_reconnects;
  2167. if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
  2168. return -ENODEV;
  2169. if (nvme_fc_ctlr_active_on_rport(ctrl))
  2170. return -ENOTUNIQ;
  2171. /*
  2172. * Create the admin queue
  2173. */
  2174. ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
  2175. NVME_AQ_DEPTH);
  2176. if (ret)
  2177. goto out_free_queue;
  2178. ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
  2179. NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4));
  2180. if (ret)
  2181. goto out_delete_hw_queue;
  2182. blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
  2183. ret = nvmf_connect_admin_queue(&ctrl->ctrl);
  2184. if (ret)
  2185. goto out_disconnect_admin_queue;
  2186. set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
  2187. /*
  2188. * Check controller capabilities
  2189. *
  2190. * todo:- add code to check if ctrl attributes changed from
  2191. * prior connection values
  2192. */
  2193. ret = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->ctrl.cap);
  2194. if (ret) {
  2195. dev_err(ctrl->ctrl.device,
  2196. "prop_get NVME_REG_CAP failed\n");
  2197. goto out_disconnect_admin_queue;
  2198. }
  2199. ctrl->ctrl.sqsize =
  2200. min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);
  2201. ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
  2202. if (ret)
  2203. goto out_disconnect_admin_queue;
  2204. ctrl->ctrl.max_hw_sectors =
  2205. (ctrl->lport->ops->max_sgl_segments - 1) << (PAGE_SHIFT - 9);
  2206. ret = nvme_init_identify(&ctrl->ctrl);
  2207. if (ret)
  2208. goto out_disconnect_admin_queue;
  2209. /* sanity checks */
  2210. /* FC-NVME does not have other data in the capsule */
  2211. if (ctrl->ctrl.icdoff) {
  2212. dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
  2213. ctrl->ctrl.icdoff);
  2214. goto out_disconnect_admin_queue;
  2215. }
  2216. /* FC-NVME supports normal SGL Data Block Descriptors */
  2217. if (opts->queue_size > ctrl->ctrl.maxcmd) {
  2218. /* warn if maxcmd is lower than queue_size */
  2219. dev_warn(ctrl->ctrl.device,
  2220. "queue_size %zu > ctrl maxcmd %u, reducing "
  2221. "to queue_size\n",
  2222. opts->queue_size, ctrl->ctrl.maxcmd);
  2223. opts->queue_size = ctrl->ctrl.maxcmd;
  2224. }
  2225. if (opts->queue_size > ctrl->ctrl.sqsize + 1) {
  2226. /* warn if sqsize is lower than queue_size */
  2227. dev_warn(ctrl->ctrl.device,
  2228. "queue_size %zu > ctrl sqsize %u, clamping down\n",
  2229. opts->queue_size, ctrl->ctrl.sqsize + 1);
  2230. opts->queue_size = ctrl->ctrl.sqsize + 1;
  2231. }
  2232. ret = nvme_fc_init_aen_ops(ctrl);
  2233. if (ret)
  2234. goto out_term_aen_ops;
  2235. /*
  2236. * Create the io queues
  2237. */
  2238. if (ctrl->ctrl.queue_count > 1) {
  2239. if (!ctrl->ioq_live)
  2240. ret = nvme_fc_create_io_queues(ctrl);
  2241. else
  2242. ret = nvme_fc_recreate_io_queues(ctrl);
  2243. if (ret)
  2244. goto out_term_aen_ops;
  2245. }
  2246. changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
  2247. ctrl->ctrl.nr_reconnects = 0;
  2248. if (changed)
  2249. nvme_start_ctrl(&ctrl->ctrl);
  2250. return 0; /* Success */
  2251. out_term_aen_ops:
  2252. nvme_fc_term_aen_ops(ctrl);
  2253. out_disconnect_admin_queue:
  2254. /* send a Disconnect(association) LS to fc-nvme target */
  2255. nvme_fc_xmt_disconnect_assoc(ctrl);
  2256. out_delete_hw_queue:
  2257. __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
  2258. out_free_queue:
  2259. nvme_fc_free_queue(&ctrl->queues[0]);
  2260. ctrl->assoc_active = false;
  2261. nvme_fc_ctlr_inactive_on_rport(ctrl);
  2262. return ret;
  2263. }
  2264. /*
  2265. * This routine stops operation of the controller on the host side.
  2266. * On the host os stack side: Admin and IO queues are stopped,
  2267. * outstanding ios on them terminated via FC ABTS.
  2268. * On the link side: the association is terminated.
  2269. */
  2270. static void
  2271. nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
  2272. {
  2273. unsigned long flags;
  2274. if (!ctrl->assoc_active)
  2275. return;
  2276. ctrl->assoc_active = false;
  2277. spin_lock_irqsave(&ctrl->lock, flags);
  2278. ctrl->flags |= FCCTRL_TERMIO;
  2279. ctrl->iocnt = 0;
  2280. spin_unlock_irqrestore(&ctrl->lock, flags);
  2281. /*
  2282. * If io queues are present, stop them and terminate all outstanding
  2283. * ios on them. As FC allocates FC exchange for each io, the
  2284. * transport must contact the LLDD to terminate the exchange,
  2285. * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
  2286. * to tell us what io's are busy and invoke a transport routine
  2287. * to kill them with the LLDD. After terminating the exchange
  2288. * the LLDD will call the transport's normal io done path, but it
  2289. * will have an aborted status. The done path will return the
  2290. * io requests back to the block layer as part of normal completions
  2291. * (but with error status).
  2292. */
  2293. if (ctrl->ctrl.queue_count > 1) {
  2294. nvme_stop_queues(&ctrl->ctrl);
  2295. blk_mq_tagset_busy_iter(&ctrl->tag_set,
  2296. nvme_fc_terminate_exchange, &ctrl->ctrl);
  2297. }
  2298. /*
  2299. * Other transports, which don't have link-level contexts bound
  2300. * to sqe's, would try to gracefully shutdown the controller by
  2301. * writing the registers for shutdown and polling (call
  2302. * nvme_shutdown_ctrl()). Given a bunch of i/o was potentially
  2303. * just aborted and we will wait on those contexts, and given
  2304. * there was no indication of how live the controlelr is on the
  2305. * link, don't send more io to create more contexts for the
  2306. * shutdown. Let the controller fail via keepalive failure if
  2307. * its still present.
  2308. */
  2309. /*
  2310. * clean up the admin queue. Same thing as above.
  2311. * use blk_mq_tagset_busy_itr() and the transport routine to
  2312. * terminate the exchanges.
  2313. */
  2314. blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
  2315. blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
  2316. nvme_fc_terminate_exchange, &ctrl->ctrl);
  2317. /* kill the aens as they are a separate path */
  2318. nvme_fc_abort_aen_ops(ctrl);
  2319. /* wait for all io that had to be aborted */
  2320. spin_lock_irq(&ctrl->lock);
  2321. wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
  2322. ctrl->flags &= ~FCCTRL_TERMIO;
  2323. spin_unlock_irq(&ctrl->lock);
  2324. nvme_fc_term_aen_ops(ctrl);
  2325. /*
  2326. * send a Disconnect(association) LS to fc-nvme target
  2327. * Note: could have been sent at top of process, but
  2328. * cleaner on link traffic if after the aborts complete.
  2329. * Note: if association doesn't exist, association_id will be 0
  2330. */
  2331. if (ctrl->association_id)
  2332. nvme_fc_xmt_disconnect_assoc(ctrl);
  2333. if (ctrl->ctrl.tagset) {
  2334. nvme_fc_delete_hw_io_queues(ctrl);
  2335. nvme_fc_free_io_queues(ctrl);
  2336. }
  2337. __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
  2338. nvme_fc_free_queue(&ctrl->queues[0]);
  2339. /* re-enable the admin_q so anything new can fast fail */
  2340. blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
  2341. /* resume the io queues so that things will fast fail */
  2342. nvme_start_queues(&ctrl->ctrl);
  2343. nvme_fc_ctlr_inactive_on_rport(ctrl);
  2344. }
  2345. static void
  2346. nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
  2347. {
  2348. struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
  2349. cancel_work_sync(&ctrl->err_work);
  2350. cancel_delayed_work_sync(&ctrl->connect_work);
  2351. /*
  2352. * kill the association on the link side. this will block
  2353. * waiting for io to terminate
  2354. */
  2355. nvme_fc_delete_association(ctrl);
  2356. }
  2357. static void
  2358. nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
  2359. {
  2360. struct nvme_fc_rport *rport = ctrl->rport;
  2361. struct nvme_fc_remote_port *portptr = &rport->remoteport;
  2362. unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
  2363. bool recon = true;
  2364. if (ctrl->ctrl.state != NVME_CTRL_CONNECTING)
  2365. return;
  2366. if (portptr->port_state == FC_OBJSTATE_ONLINE)
  2367. dev_info(ctrl->ctrl.device,
  2368. "NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
  2369. ctrl->cnum, status);
  2370. else if (time_after_eq(jiffies, rport->dev_loss_end))
  2371. recon = false;
  2372. if (recon && nvmf_should_reconnect(&ctrl->ctrl)) {
  2373. if (portptr->port_state == FC_OBJSTATE_ONLINE)
  2374. dev_info(ctrl->ctrl.device,
  2375. "NVME-FC{%d}: Reconnect attempt in %ld "
  2376. "seconds\n",
  2377. ctrl->cnum, recon_delay / HZ);
  2378. else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
  2379. recon_delay = rport->dev_loss_end - jiffies;
  2380. queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay);
  2381. } else {
  2382. if (portptr->port_state == FC_OBJSTATE_ONLINE)
  2383. dev_warn(ctrl->ctrl.device,
  2384. "NVME-FC{%d}: Max reconnect attempts (%d) "
  2385. "reached.\n",
  2386. ctrl->cnum, ctrl->ctrl.nr_reconnects);
  2387. else
  2388. dev_warn(ctrl->ctrl.device,
  2389. "NVME-FC{%d}: dev_loss_tmo (%d) expired "
  2390. "while waiting for remoteport connectivity.\n",
  2391. ctrl->cnum, portptr->dev_loss_tmo);
  2392. WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
  2393. }
  2394. }
  2395. static void
  2396. __nvme_fc_terminate_io(struct nvme_fc_ctrl *ctrl)
  2397. {
  2398. /*
  2399. * if state is connecting - the error occurred as part of a
  2400. * reconnect attempt. The create_association error paths will
  2401. * clean up any outstanding io.
  2402. *
  2403. * if it's a different state - ensure all pending io is
  2404. * terminated. Given this can delay while waiting for the
  2405. * aborted io to return, we recheck adapter state below
  2406. * before changing state.
  2407. */
  2408. if (ctrl->ctrl.state != NVME_CTRL_CONNECTING) {
  2409. nvme_stop_keep_alive(&ctrl->ctrl);
  2410. /* will block will waiting for io to terminate */
  2411. nvme_fc_delete_association(ctrl);
  2412. }
  2413. if (ctrl->ctrl.state != NVME_CTRL_CONNECTING &&
  2414. !nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
  2415. dev_err(ctrl->ctrl.device,
  2416. "NVME-FC{%d}: error_recovery: Couldn't change state "
  2417. "to CONNECTING\n", ctrl->cnum);
  2418. }
  2419. static void
  2420. nvme_fc_reset_ctrl_work(struct work_struct *work)
  2421. {
  2422. struct nvme_fc_ctrl *ctrl =
  2423. container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
  2424. int ret;
  2425. __nvme_fc_terminate_io(ctrl);
  2426. nvme_stop_ctrl(&ctrl->ctrl);
  2427. if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE)
  2428. ret = nvme_fc_create_association(ctrl);
  2429. else
  2430. ret = -ENOTCONN;
  2431. if (ret)
  2432. nvme_fc_reconnect_or_delete(ctrl, ret);
  2433. else
  2434. dev_info(ctrl->ctrl.device,
  2435. "NVME-FC{%d}: controller reset complete\n",
  2436. ctrl->cnum);
  2437. }
  2438. static void
  2439. nvme_fc_connect_err_work(struct work_struct *work)
  2440. {
  2441. struct nvme_fc_ctrl *ctrl =
  2442. container_of(work, struct nvme_fc_ctrl, err_work);
  2443. __nvme_fc_terminate_io(ctrl);
  2444. atomic_set(&ctrl->err_work_active, 0);
  2445. /*
  2446. * Rescheduling the connection after recovering
  2447. * from the io error is left to the reconnect work
  2448. * item, which is what should have stalled waiting on
  2449. * the io that had the error that scheduled this work.
  2450. */
  2451. }
  2452. static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
  2453. .name = "fc",
  2454. .module = THIS_MODULE,
  2455. .flags = NVME_F_FABRICS,
  2456. .reg_read32 = nvmf_reg_read32,
  2457. .reg_read64 = nvmf_reg_read64,
  2458. .reg_write32 = nvmf_reg_write32,
  2459. .free_ctrl = nvme_fc_nvme_ctrl_freed,
  2460. .submit_async_event = nvme_fc_submit_async_event,
  2461. .delete_ctrl = nvme_fc_delete_ctrl,
  2462. .get_address = nvmf_get_address,
  2463. };
  2464. static void
  2465. nvme_fc_connect_ctrl_work(struct work_struct *work)
  2466. {
  2467. int ret;
  2468. struct nvme_fc_ctrl *ctrl =
  2469. container_of(to_delayed_work(work),
  2470. struct nvme_fc_ctrl, connect_work);
  2471. ret = nvme_fc_create_association(ctrl);
  2472. if (ret)
  2473. nvme_fc_reconnect_or_delete(ctrl, ret);
  2474. else
  2475. dev_info(ctrl->ctrl.device,
  2476. "NVME-FC{%d}: controller connect complete\n",
  2477. ctrl->cnum);
  2478. }
  2479. static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
  2480. .queue_rq = nvme_fc_queue_rq,
  2481. .complete = nvme_fc_complete_rq,
  2482. .init_request = nvme_fc_init_request,
  2483. .exit_request = nvme_fc_exit_request,
  2484. .init_hctx = nvme_fc_init_admin_hctx,
  2485. .timeout = nvme_fc_timeout,
  2486. };
  2487. /*
  2488. * Fails a controller request if it matches an existing controller
  2489. * (association) with the same tuple:
  2490. * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
  2491. *
  2492. * The ports don't need to be compared as they are intrinsically
  2493. * already matched by the port pointers supplied.
  2494. */
  2495. static bool
  2496. nvme_fc_existing_controller(struct nvme_fc_rport *rport,
  2497. struct nvmf_ctrl_options *opts)
  2498. {
  2499. struct nvme_fc_ctrl *ctrl;
  2500. unsigned long flags;
  2501. bool found = false;
  2502. spin_lock_irqsave(&rport->lock, flags);
  2503. list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
  2504. found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts);
  2505. if (found)
  2506. break;
  2507. }
  2508. spin_unlock_irqrestore(&rport->lock, flags);
  2509. return found;
  2510. }
  2511. static struct nvme_ctrl *
  2512. nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
  2513. struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
  2514. {
  2515. struct nvme_fc_ctrl *ctrl;
  2516. unsigned long flags;
  2517. int ret, idx;
  2518. if (!(rport->remoteport.port_role &
  2519. (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
  2520. ret = -EBADR;
  2521. goto out_fail;
  2522. }
  2523. if (!opts->duplicate_connect &&
  2524. nvme_fc_existing_controller(rport, opts)) {
  2525. ret = -EALREADY;
  2526. goto out_fail;
  2527. }
  2528. ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
  2529. if (!ctrl) {
  2530. ret = -ENOMEM;
  2531. goto out_fail;
  2532. }
  2533. idx = ida_simple_get(&nvme_fc_ctrl_cnt, 0, 0, GFP_KERNEL);
  2534. if (idx < 0) {
  2535. ret = -ENOSPC;
  2536. goto out_free_ctrl;
  2537. }
  2538. ctrl->ctrl.opts = opts;
  2539. ctrl->ctrl.nr_reconnects = 0;
  2540. INIT_LIST_HEAD(&ctrl->ctrl_list);
  2541. ctrl->lport = lport;
  2542. ctrl->rport = rport;
  2543. ctrl->dev = lport->dev;
  2544. ctrl->cnum = idx;
  2545. ctrl->ioq_live = false;
  2546. ctrl->assoc_active = false;
  2547. atomic_set(&ctrl->err_work_active, 0);
  2548. init_waitqueue_head(&ctrl->ioabort_wait);
  2549. get_device(ctrl->dev);
  2550. kref_init(&ctrl->ref);
  2551. INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
  2552. INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
  2553. INIT_WORK(&ctrl->err_work, nvme_fc_connect_err_work);
  2554. spin_lock_init(&ctrl->lock);
  2555. /* io queue count */
  2556. ctrl->ctrl.queue_count = min_t(unsigned int,
  2557. opts->nr_io_queues,
  2558. lport->ops->max_hw_queues);
  2559. ctrl->ctrl.queue_count++; /* +1 for admin queue */
  2560. ctrl->ctrl.sqsize = opts->queue_size - 1;
  2561. ctrl->ctrl.kato = opts->kato;
  2562. ctrl->ctrl.cntlid = 0xffff;
  2563. ret = -ENOMEM;
  2564. ctrl->queues = kcalloc(ctrl->ctrl.queue_count,
  2565. sizeof(struct nvme_fc_queue), GFP_KERNEL);
  2566. if (!ctrl->queues)
  2567. goto out_free_ida;
  2568. nvme_fc_init_queue(ctrl, 0);
  2569. memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
  2570. ctrl->admin_tag_set.ops = &nvme_fc_admin_mq_ops;
  2571. ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
  2572. ctrl->admin_tag_set.reserved_tags = 2; /* fabric connect + Keep-Alive */
  2573. ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;
  2574. ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_fc_fcp_op) +
  2575. (SG_CHUNK_SIZE *
  2576. sizeof(struct scatterlist)) +
  2577. ctrl->lport->ops->fcprqst_priv_sz;
  2578. ctrl->admin_tag_set.driver_data = ctrl;
  2579. ctrl->admin_tag_set.nr_hw_queues = 1;
  2580. ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;
  2581. ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED;
  2582. ret = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
  2583. if (ret)
  2584. goto out_free_queues;
  2585. ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;
  2586. ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
  2587. if (IS_ERR(ctrl->ctrl.admin_q)) {
  2588. ret = PTR_ERR(ctrl->ctrl.admin_q);
  2589. goto out_free_admin_tag_set;
  2590. }
  2591. /*
  2592. * Would have been nice to init io queues tag set as well.
  2593. * However, we require interaction from the controller
  2594. * for max io queue count before we can do so.
  2595. * Defer this to the connect path.
  2596. */
  2597. ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0);
  2598. if (ret)
  2599. goto out_cleanup_admin_q;
  2600. /* at this point, teardown path changes to ref counting on nvme ctrl */
  2601. spin_lock_irqsave(&rport->lock, flags);
  2602. list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list);
  2603. spin_unlock_irqrestore(&rport->lock, flags);
  2604. if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING) ||
  2605. !nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
  2606. dev_err(ctrl->ctrl.device,
  2607. "NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
  2608. goto fail_ctrl;
  2609. }
  2610. nvme_get_ctrl(&ctrl->ctrl);
  2611. if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
  2612. nvme_put_ctrl(&ctrl->ctrl);
  2613. dev_err(ctrl->ctrl.device,
  2614. "NVME-FC{%d}: failed to schedule initial connect\n",
  2615. ctrl->cnum);
  2616. goto fail_ctrl;
  2617. }
  2618. flush_delayed_work(&ctrl->connect_work);
  2619. dev_info(ctrl->ctrl.device,
  2620. "NVME-FC{%d}: new ctrl: NQN \"%s\"\n",
  2621. ctrl->cnum, ctrl->ctrl.opts->subsysnqn);
  2622. return &ctrl->ctrl;
  2623. fail_ctrl:
  2624. nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING);
  2625. cancel_work_sync(&ctrl->ctrl.reset_work);
  2626. cancel_work_sync(&ctrl->err_work);
  2627. cancel_delayed_work_sync(&ctrl->connect_work);
  2628. ctrl->ctrl.opts = NULL;
  2629. /* initiate nvme ctrl ref counting teardown */
  2630. nvme_uninit_ctrl(&ctrl->ctrl);
  2631. /* Remove core ctrl ref. */
  2632. nvme_put_ctrl(&ctrl->ctrl);
  2633. /* as we're past the point where we transition to the ref
  2634. * counting teardown path, if we return a bad pointer here,
  2635. * the calling routine, thinking it's prior to the
  2636. * transition, will do an rport put. Since the teardown
  2637. * path also does a rport put, we do an extra get here to
  2638. * so proper order/teardown happens.
  2639. */
  2640. nvme_fc_rport_get(rport);
  2641. return ERR_PTR(-EIO);
  2642. out_cleanup_admin_q:
  2643. blk_cleanup_queue(ctrl->ctrl.admin_q);
  2644. out_free_admin_tag_set:
  2645. blk_mq_free_tag_set(&ctrl->admin_tag_set);
  2646. out_free_queues:
  2647. kfree(ctrl->queues);
  2648. out_free_ida:
  2649. put_device(ctrl->dev);
  2650. ida_simple_remove(&nvme_fc_ctrl_cnt, ctrl->cnum);
  2651. out_free_ctrl:
  2652. kfree(ctrl);
  2653. out_fail:
  2654. /* exit via here doesn't follow ctlr ref points */
  2655. return ERR_PTR(ret);
  2656. }
  2657. struct nvmet_fc_traddr {
  2658. u64 nn;
  2659. u64 pn;
  2660. };
  2661. static int
  2662. __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
  2663. {
  2664. u64 token64;
  2665. if (match_u64(sstr, &token64))
  2666. return -EINVAL;
  2667. *val = token64;
  2668. return 0;
  2669. }
  2670. /*
  2671. * This routine validates and extracts the WWN's from the TRADDR string.
  2672. * As kernel parsers need the 0x to determine number base, universally
  2673. * build string to parse with 0x prefix before parsing name strings.
  2674. */
  2675. static int
  2676. nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
  2677. {
  2678. char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
  2679. substring_t wwn = { name, &name[sizeof(name)-1] };
  2680. int nnoffset, pnoffset;
  2681. /* validate it string one of the 2 allowed formats */
  2682. if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
  2683. !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
  2684. !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
  2685. "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
  2686. nnoffset = NVME_FC_TRADDR_OXNNLEN;
  2687. pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
  2688. NVME_FC_TRADDR_OXNNLEN;
  2689. } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
  2690. !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
  2691. !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
  2692. "pn-", NVME_FC_TRADDR_NNLEN))) {
  2693. nnoffset = NVME_FC_TRADDR_NNLEN;
  2694. pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
  2695. } else
  2696. goto out_einval;
  2697. name[0] = '0';
  2698. name[1] = 'x';
  2699. name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
  2700. memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
  2701. if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
  2702. goto out_einval;
  2703. memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
  2704. if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
  2705. goto out_einval;
  2706. return 0;
  2707. out_einval:
  2708. pr_warn("%s: bad traddr string\n", __func__);
  2709. return -EINVAL;
  2710. }
  2711. static struct nvme_ctrl *
  2712. nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
  2713. {
  2714. struct nvme_fc_lport *lport;
  2715. struct nvme_fc_rport *rport;
  2716. struct nvme_ctrl *ctrl;
  2717. struct nvmet_fc_traddr laddr = { 0L, 0L };
  2718. struct nvmet_fc_traddr raddr = { 0L, 0L };
  2719. unsigned long flags;
  2720. int ret;
  2721. ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE);
  2722. if (ret || !raddr.nn || !raddr.pn)
  2723. return ERR_PTR(-EINVAL);
  2724. ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE);
  2725. if (ret || !laddr.nn || !laddr.pn)
  2726. return ERR_PTR(-EINVAL);
  2727. /* find the host and remote ports to connect together */
  2728. spin_lock_irqsave(&nvme_fc_lock, flags);
  2729. list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
  2730. if (lport->localport.node_name != laddr.nn ||
  2731. lport->localport.port_name != laddr.pn ||
  2732. lport->localport.port_state != FC_OBJSTATE_ONLINE)
  2733. continue;
  2734. list_for_each_entry(rport, &lport->endp_list, endp_list) {
  2735. if (rport->remoteport.node_name != raddr.nn ||
  2736. rport->remoteport.port_name != raddr.pn ||
  2737. rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
  2738. continue;
  2739. /* if fail to get reference fall through. Will error */
  2740. if (!nvme_fc_rport_get(rport))
  2741. break;
  2742. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  2743. ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
  2744. if (IS_ERR(ctrl))
  2745. nvme_fc_rport_put(rport);
  2746. return ctrl;
  2747. }
  2748. }
  2749. spin_unlock_irqrestore(&nvme_fc_lock, flags);
  2750. pr_warn("%s: %s - %s combination not found\n",
  2751. __func__, opts->traddr, opts->host_traddr);
  2752. return ERR_PTR(-ENOENT);
  2753. }
  2754. static struct nvmf_transport_ops nvme_fc_transport = {
  2755. .name = "fc",
  2756. .module = THIS_MODULE,
  2757. .required_opts = NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
  2758. .allowed_opts = NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
  2759. .create_ctrl = nvme_fc_create_ctrl,
  2760. };
  2761. static int __init nvme_fc_init_module(void)
  2762. {
  2763. int ret;
  2764. nvme_fc_wq = alloc_workqueue("nvme_fc_wq", WQ_MEM_RECLAIM, 0);
  2765. if (!nvme_fc_wq)
  2766. return -ENOMEM;
  2767. /*
  2768. * NOTE:
  2769. * It is expected that in the future the kernel will combine
  2770. * the FC-isms that are currently under scsi and now being
  2771. * added to by NVME into a new standalone FC class. The SCSI
  2772. * and NVME protocols and their devices would be under this
  2773. * new FC class.
  2774. *
  2775. * As we need something to post FC-specific udev events to,
  2776. * specifically for nvme probe events, start by creating the
  2777. * new device class. When the new standalone FC class is
  2778. * put in place, this code will move to a more generic
  2779. * location for the class.
  2780. */
  2781. fc_class = class_create(THIS_MODULE, "fc");
  2782. if (IS_ERR(fc_class)) {
  2783. pr_err("couldn't register class fc\n");
  2784. ret = PTR_ERR(fc_class);
  2785. goto out_destroy_wq;
  2786. }
  2787. /*
  2788. * Create a device for the FC-centric udev events
  2789. */
  2790. fc_udev_device = device_create(fc_class, NULL, MKDEV(0, 0), NULL,
  2791. "fc_udev_device");
  2792. if (IS_ERR(fc_udev_device)) {
  2793. pr_err("couldn't create fc_udev device!\n");
  2794. ret = PTR_ERR(fc_udev_device);
  2795. goto out_destroy_class;
  2796. }
  2797. ret = nvmf_register_transport(&nvme_fc_transport);
  2798. if (ret)
  2799. goto out_destroy_device;
  2800. return 0;
  2801. out_destroy_device:
  2802. device_destroy(fc_class, MKDEV(0, 0));
  2803. out_destroy_class:
  2804. class_destroy(fc_class);
  2805. out_destroy_wq:
  2806. destroy_workqueue(nvme_fc_wq);
  2807. return ret;
  2808. }
  2809. static void __exit nvme_fc_exit_module(void)
  2810. {
  2811. /* sanity check - all lports should be removed */
  2812. if (!list_empty(&nvme_fc_lport_list))
  2813. pr_warn("%s: localport list not empty\n", __func__);
  2814. nvmf_unregister_transport(&nvme_fc_transport);
  2815. ida_destroy(&nvme_fc_local_port_cnt);
  2816. ida_destroy(&nvme_fc_ctrl_cnt);
  2817. device_destroy(fc_class, MKDEV(0, 0));
  2818. class_destroy(fc_class);
  2819. destroy_workqueue(nvme_fc_wq);
  2820. }
  2821. module_init(nvme_fc_init_module);
  2822. module_exit(nvme_fc_exit_module);
  2823. MODULE_LICENSE("GPL v2");