core.c 97 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868
  1. /*
  2. * NVM Express device driver
  3. * Copyright (c) 2011-2014, Intel Corporation.
  4. *
  5. * This program is free software; you can redistribute it and/or modify it
  6. * under the terms and conditions of the GNU General Public License,
  7. * version 2, as published by the Free Software Foundation.
  8. *
  9. * This program is distributed in the hope it will be useful, but WITHOUT
  10. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  12. * more details.
  13. */
  14. #include <linux/blkdev.h>
  15. #include <linux/blk-mq.h>
  16. #include <linux/delay.h>
  17. #include <linux/errno.h>
  18. #include <linux/hdreg.h>
  19. #include <linux/kernel.h>
  20. #include <linux/module.h>
  21. #include <linux/list_sort.h>
  22. #include <linux/slab.h>
  23. #include <linux/types.h>
  24. #include <linux/pr.h>
  25. #include <linux/ptrace.h>
  26. #include <linux/nvme_ioctl.h>
  27. #include <linux/t10-pi.h>
  28. #include <linux/pm_qos.h>
  29. #include <asm/unaligned.h>
  30. #define CREATE_TRACE_POINTS
  31. #include "trace.h"
  32. #include "nvme.h"
  33. #include "fabrics.h"
  34. #define NVME_MINORS (1U << MINORBITS)
  35. unsigned int admin_timeout = 60;
  36. module_param(admin_timeout, uint, 0644);
  37. MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  38. EXPORT_SYMBOL_GPL(admin_timeout);
  39. unsigned int nvme_io_timeout = 30;
  40. module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  41. MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  42. EXPORT_SYMBOL_GPL(nvme_io_timeout);
  43. static unsigned char shutdown_timeout = 5;
  44. module_param(shutdown_timeout, byte, 0644);
  45. MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  46. static u8 nvme_max_retries = 5;
  47. module_param_named(max_retries, nvme_max_retries, byte, 0644);
  48. MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  49. static unsigned long default_ps_max_latency_us = 100000;
  50. module_param(default_ps_max_latency_us, ulong, 0644);
  51. MODULE_PARM_DESC(default_ps_max_latency_us,
  52. "max power saving latency for new devices; use PM QOS to change per device");
  53. static bool force_apst;
  54. module_param(force_apst, bool, 0644);
  55. MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  56. static bool streams;
  57. module_param(streams, bool, 0644);
  58. MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
  59. /*
  60. * nvme_wq - hosts nvme related works that are not reset or delete
  61. * nvme_reset_wq - hosts nvme reset works
  62. * nvme_delete_wq - hosts nvme delete works
  63. *
  64. * nvme_wq will host works such are scan, aen handling, fw activation,
  65. * keep-alive error recovery, periodic reconnects etc. nvme_reset_wq
  66. * runs reset works which also flush works hosted on nvme_wq for
  67. * serialization purposes. nvme_delete_wq host controller deletion
  68. * works which flush reset works for serialization.
  69. */
  70. struct workqueue_struct *nvme_wq;
  71. EXPORT_SYMBOL_GPL(nvme_wq);
  72. struct workqueue_struct *nvme_reset_wq;
  73. EXPORT_SYMBOL_GPL(nvme_reset_wq);
  74. struct workqueue_struct *nvme_delete_wq;
  75. EXPORT_SYMBOL_GPL(nvme_delete_wq);
  76. static DEFINE_IDA(nvme_subsystems_ida);
  77. static LIST_HEAD(nvme_subsystems);
  78. static DEFINE_MUTEX(nvme_subsystems_lock);
  79. static DEFINE_IDA(nvme_instance_ida);
  80. static dev_t nvme_chr_devt;
  81. static struct class *nvme_class;
  82. static struct class *nvme_subsys_class;
  83. static void nvme_ns_remove(struct nvme_ns *ns);
  84. static int nvme_revalidate_disk(struct gendisk *disk);
  85. static void nvme_put_subsystem(struct nvme_subsystem *subsys);
  86. static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
  87. unsigned nsid);
  88. static void nvme_set_queue_dying(struct nvme_ns *ns)
  89. {
  90. /*
  91. * Revalidating a dead namespace sets capacity to 0. This will end
  92. * buffered writers dirtying pages that can't be synced.
  93. */
  94. if (!ns->disk || test_and_set_bit(NVME_NS_DEAD, &ns->flags))
  95. return;
  96. blk_set_queue_dying(ns->queue);
  97. /* Forcibly unquiesce queues to avoid blocking dispatch */
  98. blk_mq_unquiesce_queue(ns->queue);
  99. /*
  100. * Revalidate after unblocking dispatchers that may be holding bd_butex
  101. */
  102. revalidate_disk(ns->disk);
  103. }
  104. static void nvme_queue_scan(struct nvme_ctrl *ctrl)
  105. {
  106. /*
  107. * Only new queue scan work when admin and IO queues are both alive
  108. */
  109. if (ctrl->state == NVME_CTRL_LIVE)
  110. queue_work(nvme_wq, &ctrl->scan_work);
  111. }
  112. int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
  113. {
  114. if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
  115. return -EBUSY;
  116. if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
  117. return -EBUSY;
  118. return 0;
  119. }
  120. EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
  121. int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
  122. {
  123. int ret;
  124. ret = nvme_reset_ctrl(ctrl);
  125. if (!ret) {
  126. flush_work(&ctrl->reset_work);
  127. if (ctrl->state != NVME_CTRL_LIVE &&
  128. ctrl->state != NVME_CTRL_ADMIN_ONLY)
  129. ret = -ENETRESET;
  130. }
  131. return ret;
  132. }
  133. EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
  134. static void nvme_delete_ctrl_work(struct work_struct *work)
  135. {
  136. struct nvme_ctrl *ctrl =
  137. container_of(work, struct nvme_ctrl, delete_work);
  138. dev_info(ctrl->device,
  139. "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
  140. flush_work(&ctrl->reset_work);
  141. nvme_stop_ctrl(ctrl);
  142. nvme_remove_namespaces(ctrl);
  143. ctrl->ops->delete_ctrl(ctrl);
  144. nvme_uninit_ctrl(ctrl);
  145. nvme_put_ctrl(ctrl);
  146. }
  147. int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
  148. {
  149. if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
  150. return -EBUSY;
  151. if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
  152. return -EBUSY;
  153. return 0;
  154. }
  155. EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
  156. int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
  157. {
  158. int ret = 0;
  159. /*
  160. * Keep a reference until the work is flushed since ->delete_ctrl
  161. * can free the controller.
  162. */
  163. nvme_get_ctrl(ctrl);
  164. ret = nvme_delete_ctrl(ctrl);
  165. if (!ret)
  166. flush_work(&ctrl->delete_work);
  167. nvme_put_ctrl(ctrl);
  168. return ret;
  169. }
  170. EXPORT_SYMBOL_GPL(nvme_delete_ctrl_sync);
  171. static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
  172. {
  173. return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
  174. }
  175. static blk_status_t nvme_error_status(struct request *req)
  176. {
  177. switch (nvme_req(req)->status & 0x7ff) {
  178. case NVME_SC_SUCCESS:
  179. return BLK_STS_OK;
  180. case NVME_SC_CAP_EXCEEDED:
  181. return BLK_STS_NOSPC;
  182. case NVME_SC_LBA_RANGE:
  183. return BLK_STS_TARGET;
  184. case NVME_SC_BAD_ATTRIBUTES:
  185. case NVME_SC_ONCS_NOT_SUPPORTED:
  186. case NVME_SC_INVALID_OPCODE:
  187. case NVME_SC_INVALID_FIELD:
  188. case NVME_SC_INVALID_NS:
  189. return BLK_STS_NOTSUPP;
  190. case NVME_SC_WRITE_FAULT:
  191. case NVME_SC_READ_ERROR:
  192. case NVME_SC_UNWRITTEN_BLOCK:
  193. case NVME_SC_ACCESS_DENIED:
  194. case NVME_SC_READ_ONLY:
  195. case NVME_SC_COMPARE_FAILED:
  196. return BLK_STS_MEDIUM;
  197. case NVME_SC_GUARD_CHECK:
  198. case NVME_SC_APPTAG_CHECK:
  199. case NVME_SC_REFTAG_CHECK:
  200. case NVME_SC_INVALID_PI:
  201. return BLK_STS_PROTECTION;
  202. case NVME_SC_RESERVATION_CONFLICT:
  203. return BLK_STS_NEXUS;
  204. default:
  205. return BLK_STS_IOERR;
  206. }
  207. }
  208. static inline bool nvme_req_needs_retry(struct request *req)
  209. {
  210. if (blk_noretry_request(req))
  211. return false;
  212. if (nvme_req(req)->status & NVME_SC_DNR)
  213. return false;
  214. if (nvme_req(req)->retries >= nvme_max_retries)
  215. return false;
  216. return true;
  217. }
  218. void nvme_complete_rq(struct request *req)
  219. {
  220. blk_status_t status = nvme_error_status(req);
  221. trace_nvme_complete_rq(req);
  222. if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) {
  223. if ((req->cmd_flags & REQ_NVME_MPATH) && nvme_failover_req(req))
  224. return;
  225. if (!blk_queue_dying(req->q)) {
  226. nvme_req(req)->retries++;
  227. blk_mq_requeue_request(req, true);
  228. return;
  229. }
  230. }
  231. blk_mq_end_request(req, status);
  232. }
  233. EXPORT_SYMBOL_GPL(nvme_complete_rq);
  234. void nvme_cancel_request(struct request *req, void *data, bool reserved)
  235. {
  236. dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
  237. "Cancelling I/O %d", req->tag);
  238. nvme_req(req)->status = NVME_SC_ABORT_REQ;
  239. blk_mq_complete_request(req);
  240. }
  241. EXPORT_SYMBOL_GPL(nvme_cancel_request);
  242. bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
  243. enum nvme_ctrl_state new_state)
  244. {
  245. enum nvme_ctrl_state old_state;
  246. unsigned long flags;
  247. bool changed = false;
  248. spin_lock_irqsave(&ctrl->lock, flags);
  249. old_state = ctrl->state;
  250. switch (new_state) {
  251. case NVME_CTRL_ADMIN_ONLY:
  252. switch (old_state) {
  253. case NVME_CTRL_CONNECTING:
  254. changed = true;
  255. /* FALLTHRU */
  256. default:
  257. break;
  258. }
  259. break;
  260. case NVME_CTRL_LIVE:
  261. switch (old_state) {
  262. case NVME_CTRL_NEW:
  263. case NVME_CTRL_RESETTING:
  264. case NVME_CTRL_CONNECTING:
  265. changed = true;
  266. /* FALLTHRU */
  267. default:
  268. break;
  269. }
  270. break;
  271. case NVME_CTRL_RESETTING:
  272. switch (old_state) {
  273. case NVME_CTRL_NEW:
  274. case NVME_CTRL_LIVE:
  275. case NVME_CTRL_ADMIN_ONLY:
  276. changed = true;
  277. /* FALLTHRU */
  278. default:
  279. break;
  280. }
  281. break;
  282. case NVME_CTRL_CONNECTING:
  283. switch (old_state) {
  284. case NVME_CTRL_NEW:
  285. case NVME_CTRL_RESETTING:
  286. changed = true;
  287. /* FALLTHRU */
  288. default:
  289. break;
  290. }
  291. break;
  292. case NVME_CTRL_DELETING:
  293. switch (old_state) {
  294. case NVME_CTRL_LIVE:
  295. case NVME_CTRL_ADMIN_ONLY:
  296. case NVME_CTRL_RESETTING:
  297. case NVME_CTRL_CONNECTING:
  298. changed = true;
  299. /* FALLTHRU */
  300. default:
  301. break;
  302. }
  303. break;
  304. case NVME_CTRL_DEAD:
  305. switch (old_state) {
  306. case NVME_CTRL_DELETING:
  307. changed = true;
  308. /* FALLTHRU */
  309. default:
  310. break;
  311. }
  312. break;
  313. default:
  314. break;
  315. }
  316. if (changed)
  317. ctrl->state = new_state;
  318. spin_unlock_irqrestore(&ctrl->lock, flags);
  319. if (changed && ctrl->state == NVME_CTRL_LIVE)
  320. nvme_kick_requeue_lists(ctrl);
  321. return changed;
  322. }
  323. EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
  324. static void nvme_free_ns_head(struct kref *ref)
  325. {
  326. struct nvme_ns_head *head =
  327. container_of(ref, struct nvme_ns_head, ref);
  328. nvme_mpath_remove_disk(head);
  329. ida_simple_remove(&head->subsys->ns_ida, head->instance);
  330. list_del_init(&head->entry);
  331. cleanup_srcu_struct_quiesced(&head->srcu);
  332. nvme_put_subsystem(head->subsys);
  333. kfree(head);
  334. }
  335. static void nvme_put_ns_head(struct nvme_ns_head *head)
  336. {
  337. kref_put(&head->ref, nvme_free_ns_head);
  338. }
  339. static void nvme_free_ns(struct kref *kref)
  340. {
  341. struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
  342. if (ns->ndev)
  343. nvme_nvm_unregister(ns);
  344. put_disk(ns->disk);
  345. nvme_put_ns_head(ns->head);
  346. nvme_put_ctrl(ns->ctrl);
  347. kfree(ns);
  348. }
  349. static void nvme_put_ns(struct nvme_ns *ns)
  350. {
  351. kref_put(&ns->kref, nvme_free_ns);
  352. }
  353. static inline void nvme_clear_nvme_request(struct request *req)
  354. {
  355. if (!(req->rq_flags & RQF_DONTPREP)) {
  356. nvme_req(req)->retries = 0;
  357. nvme_req(req)->flags = 0;
  358. req->rq_flags |= RQF_DONTPREP;
  359. }
  360. }
  361. struct request *nvme_alloc_request(struct request_queue *q,
  362. struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
  363. {
  364. unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
  365. struct request *req;
  366. if (qid == NVME_QID_ANY) {
  367. req = blk_mq_alloc_request(q, op, flags);
  368. } else {
  369. req = blk_mq_alloc_request_hctx(q, op, flags,
  370. qid ? qid - 1 : 0);
  371. }
  372. if (IS_ERR(req))
  373. return req;
  374. req->cmd_flags |= REQ_FAILFAST_DRIVER;
  375. nvme_clear_nvme_request(req);
  376. nvme_req(req)->cmd = cmd;
  377. return req;
  378. }
  379. EXPORT_SYMBOL_GPL(nvme_alloc_request);
  380. static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
  381. {
  382. struct nvme_command c;
  383. memset(&c, 0, sizeof(c));
  384. c.directive.opcode = nvme_admin_directive_send;
  385. c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
  386. c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
  387. c.directive.dtype = NVME_DIR_IDENTIFY;
  388. c.directive.tdtype = NVME_DIR_STREAMS;
  389. c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
  390. return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
  391. }
  392. static int nvme_disable_streams(struct nvme_ctrl *ctrl)
  393. {
  394. return nvme_toggle_streams(ctrl, false);
  395. }
  396. static int nvme_enable_streams(struct nvme_ctrl *ctrl)
  397. {
  398. return nvme_toggle_streams(ctrl, true);
  399. }
  400. static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
  401. struct streams_directive_params *s, u32 nsid)
  402. {
  403. struct nvme_command c;
  404. memset(&c, 0, sizeof(c));
  405. memset(s, 0, sizeof(*s));
  406. c.directive.opcode = nvme_admin_directive_recv;
  407. c.directive.nsid = cpu_to_le32(nsid);
  408. c.directive.numd = cpu_to_le32((sizeof(*s) >> 2) - 1);
  409. c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
  410. c.directive.dtype = NVME_DIR_STREAMS;
  411. return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
  412. }
  413. static int nvme_configure_directives(struct nvme_ctrl *ctrl)
  414. {
  415. struct streams_directive_params s;
  416. int ret;
  417. if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
  418. return 0;
  419. if (!streams)
  420. return 0;
  421. ret = nvme_enable_streams(ctrl);
  422. if (ret)
  423. return ret;
  424. ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
  425. if (ret)
  426. return ret;
  427. ctrl->nssa = le16_to_cpu(s.nssa);
  428. if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
  429. dev_info(ctrl->device, "too few streams (%u) available\n",
  430. ctrl->nssa);
  431. nvme_disable_streams(ctrl);
  432. return 0;
  433. }
  434. ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
  435. dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
  436. return 0;
  437. }
  438. /*
  439. * Check if 'req' has a write hint associated with it. If it does, assign
  440. * a valid namespace stream to the write.
  441. */
  442. static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
  443. struct request *req, u16 *control,
  444. u32 *dsmgmt)
  445. {
  446. enum rw_hint streamid = req->write_hint;
  447. if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
  448. streamid = 0;
  449. else {
  450. streamid--;
  451. if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
  452. return;
  453. *control |= NVME_RW_DTYPE_STREAMS;
  454. *dsmgmt |= streamid << 16;
  455. }
  456. if (streamid < ARRAY_SIZE(req->q->write_hints))
  457. req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
  458. }
  459. static inline void nvme_setup_flush(struct nvme_ns *ns,
  460. struct nvme_command *cmnd)
  461. {
  462. memset(cmnd, 0, sizeof(*cmnd));
  463. cmnd->common.opcode = nvme_cmd_flush;
  464. cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
  465. }
  466. static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
  467. struct nvme_command *cmnd)
  468. {
  469. unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
  470. struct nvme_dsm_range *range;
  471. struct bio *bio;
  472. /*
  473. * Some devices do not consider the DSM 'Number of Ranges' field when
  474. * determining how much data to DMA. Always allocate memory for maximum
  475. * number of segments to prevent device reading beyond end of buffer.
  476. */
  477. static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
  478. range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
  479. if (!range) {
  480. /*
  481. * If we fail allocation our range, fallback to the controller
  482. * discard page. If that's also busy, it's safe to return
  483. * busy, as we know we can make progress once that's freed.
  484. */
  485. if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
  486. return BLK_STS_RESOURCE;
  487. range = page_address(ns->ctrl->discard_page);
  488. }
  489. __rq_for_each_bio(bio, req) {
  490. u64 slba = nvme_block_nr(ns, bio->bi_iter.bi_sector);
  491. u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
  492. if (n < segments) {
  493. range[n].cattr = cpu_to_le32(0);
  494. range[n].nlb = cpu_to_le32(nlb);
  495. range[n].slba = cpu_to_le64(slba);
  496. }
  497. n++;
  498. }
  499. if (WARN_ON_ONCE(n != segments)) {
  500. if (virt_to_page(range) == ns->ctrl->discard_page)
  501. clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
  502. else
  503. kfree(range);
  504. return BLK_STS_IOERR;
  505. }
  506. memset(cmnd, 0, sizeof(*cmnd));
  507. cmnd->dsm.opcode = nvme_cmd_dsm;
  508. cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
  509. cmnd->dsm.nr = cpu_to_le32(segments - 1);
  510. cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
  511. req->special_vec.bv_page = virt_to_page(range);
  512. req->special_vec.bv_offset = offset_in_page(range);
  513. req->special_vec.bv_len = alloc_size;
  514. req->rq_flags |= RQF_SPECIAL_PAYLOAD;
  515. return BLK_STS_OK;
  516. }
  517. static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
  518. struct request *req, struct nvme_command *cmnd)
  519. {
  520. struct nvme_ctrl *ctrl = ns->ctrl;
  521. u16 control = 0;
  522. u32 dsmgmt = 0;
  523. if (req->cmd_flags & REQ_FUA)
  524. control |= NVME_RW_FUA;
  525. if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
  526. control |= NVME_RW_LR;
  527. if (req->cmd_flags & REQ_RAHEAD)
  528. dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
  529. memset(cmnd, 0, sizeof(*cmnd));
  530. cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
  531. cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
  532. cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
  533. cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
  534. if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
  535. nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
  536. if (ns->ms) {
  537. /*
  538. * If formated with metadata, the block layer always provides a
  539. * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled. Else
  540. * we enable the PRACT bit for protection information or set the
  541. * namespace capacity to zero to prevent any I/O.
  542. */
  543. if (!blk_integrity_rq(req)) {
  544. if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
  545. return BLK_STS_NOTSUPP;
  546. control |= NVME_RW_PRINFO_PRACT;
  547. } else if (req_op(req) == REQ_OP_WRITE) {
  548. t10_pi_prepare(req, ns->pi_type);
  549. }
  550. switch (ns->pi_type) {
  551. case NVME_NS_DPS_PI_TYPE3:
  552. control |= NVME_RW_PRINFO_PRCHK_GUARD;
  553. break;
  554. case NVME_NS_DPS_PI_TYPE1:
  555. case NVME_NS_DPS_PI_TYPE2:
  556. control |= NVME_RW_PRINFO_PRCHK_GUARD |
  557. NVME_RW_PRINFO_PRCHK_REF;
  558. cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
  559. break;
  560. }
  561. }
  562. cmnd->rw.control = cpu_to_le16(control);
  563. cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
  564. return 0;
  565. }
  566. void nvme_cleanup_cmd(struct request *req)
  567. {
  568. if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
  569. nvme_req(req)->status == 0) {
  570. struct nvme_ns *ns = req->rq_disk->private_data;
  571. t10_pi_complete(req, ns->pi_type,
  572. blk_rq_bytes(req) >> ns->lba_shift);
  573. }
  574. if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
  575. struct nvme_ns *ns = req->rq_disk->private_data;
  576. struct page *page = req->special_vec.bv_page;
  577. if (page == ns->ctrl->discard_page)
  578. clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
  579. else
  580. kfree(page_address(page) + req->special_vec.bv_offset);
  581. }
  582. }
  583. EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
  584. blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
  585. struct nvme_command *cmd)
  586. {
  587. blk_status_t ret = BLK_STS_OK;
  588. nvme_clear_nvme_request(req);
  589. switch (req_op(req)) {
  590. case REQ_OP_DRV_IN:
  591. case REQ_OP_DRV_OUT:
  592. memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
  593. break;
  594. case REQ_OP_FLUSH:
  595. nvme_setup_flush(ns, cmd);
  596. break;
  597. case REQ_OP_WRITE_ZEROES:
  598. /* currently only aliased to deallocate for a few ctrls: */
  599. case REQ_OP_DISCARD:
  600. ret = nvme_setup_discard(ns, req, cmd);
  601. break;
  602. case REQ_OP_READ:
  603. case REQ_OP_WRITE:
  604. ret = nvme_setup_rw(ns, req, cmd);
  605. break;
  606. default:
  607. WARN_ON_ONCE(1);
  608. return BLK_STS_IOERR;
  609. }
  610. cmd->common.command_id = req->tag;
  611. trace_nvme_setup_cmd(req, cmd);
  612. return ret;
  613. }
  614. EXPORT_SYMBOL_GPL(nvme_setup_cmd);
  615. /*
  616. * Returns 0 on success. If the result is negative, it's a Linux error code;
  617. * if the result is positive, it's an NVM Express status code
  618. */
  619. int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
  620. union nvme_result *result, void *buffer, unsigned bufflen,
  621. unsigned timeout, int qid, int at_head,
  622. blk_mq_req_flags_t flags)
  623. {
  624. struct request *req;
  625. int ret;
  626. req = nvme_alloc_request(q, cmd, flags, qid);
  627. if (IS_ERR(req))
  628. return PTR_ERR(req);
  629. req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
  630. if (buffer && bufflen) {
  631. ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
  632. if (ret)
  633. goto out;
  634. }
  635. blk_execute_rq(req->q, NULL, req, at_head);
  636. if (result)
  637. *result = nvme_req(req)->result;
  638. if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
  639. ret = -EINTR;
  640. else
  641. ret = nvme_req(req)->status;
  642. out:
  643. blk_mq_free_request(req);
  644. return ret;
  645. }
  646. EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
  647. int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
  648. void *buffer, unsigned bufflen)
  649. {
  650. return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
  651. NVME_QID_ANY, 0, 0);
  652. }
  653. EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
  654. static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
  655. unsigned len, u32 seed, bool write)
  656. {
  657. struct bio_integrity_payload *bip;
  658. int ret = -ENOMEM;
  659. void *buf;
  660. buf = kmalloc(len, GFP_KERNEL);
  661. if (!buf)
  662. goto out;
  663. ret = -EFAULT;
  664. if (write && copy_from_user(buf, ubuf, len))
  665. goto out_free_meta;
  666. bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
  667. if (IS_ERR(bip)) {
  668. ret = PTR_ERR(bip);
  669. goto out_free_meta;
  670. }
  671. bip->bip_iter.bi_size = len;
  672. bip->bip_iter.bi_sector = seed;
  673. ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
  674. offset_in_page(buf));
  675. if (ret == len)
  676. return buf;
  677. ret = -ENOMEM;
  678. out_free_meta:
  679. kfree(buf);
  680. out:
  681. return ERR_PTR(ret);
  682. }
  683. static int nvme_submit_user_cmd(struct request_queue *q,
  684. struct nvme_command *cmd, void __user *ubuffer,
  685. unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
  686. u32 meta_seed, u32 *result, unsigned timeout)
  687. {
  688. bool write = nvme_is_write(cmd);
  689. struct nvme_ns *ns = q->queuedata;
  690. struct gendisk *disk = ns ? ns->disk : NULL;
  691. struct request *req;
  692. struct bio *bio = NULL;
  693. void *meta = NULL;
  694. int ret;
  695. req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
  696. if (IS_ERR(req))
  697. return PTR_ERR(req);
  698. req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
  699. nvme_req(req)->flags |= NVME_REQ_USERCMD;
  700. if (ubuffer && bufflen) {
  701. ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
  702. GFP_KERNEL);
  703. if (ret)
  704. goto out;
  705. bio = req->bio;
  706. bio->bi_disk = disk;
  707. if (disk && meta_buffer && meta_len) {
  708. meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
  709. meta_seed, write);
  710. if (IS_ERR(meta)) {
  711. ret = PTR_ERR(meta);
  712. goto out_unmap;
  713. }
  714. req->cmd_flags |= REQ_INTEGRITY;
  715. }
  716. }
  717. blk_execute_rq(req->q, disk, req, 0);
  718. if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
  719. ret = -EINTR;
  720. else
  721. ret = nvme_req(req)->status;
  722. if (result)
  723. *result = le32_to_cpu(nvme_req(req)->result.u32);
  724. if (meta && !ret && !write) {
  725. if (copy_to_user(meta_buffer, meta, meta_len))
  726. ret = -EFAULT;
  727. }
  728. kfree(meta);
  729. out_unmap:
  730. if (bio)
  731. blk_rq_unmap_user(bio);
  732. out:
  733. blk_mq_free_request(req);
  734. return ret;
  735. }
  736. static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
  737. {
  738. struct nvme_ctrl *ctrl = rq->end_io_data;
  739. unsigned long flags;
  740. bool startka = false;
  741. blk_mq_free_request(rq);
  742. if (status) {
  743. dev_err(ctrl->device,
  744. "failed nvme_keep_alive_end_io error=%d\n",
  745. status);
  746. return;
  747. }
  748. spin_lock_irqsave(&ctrl->lock, flags);
  749. if (ctrl->state == NVME_CTRL_LIVE ||
  750. ctrl->state == NVME_CTRL_CONNECTING)
  751. startka = true;
  752. spin_unlock_irqrestore(&ctrl->lock, flags);
  753. if (startka)
  754. schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
  755. }
  756. static int nvme_keep_alive(struct nvme_ctrl *ctrl)
  757. {
  758. struct request *rq;
  759. rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
  760. NVME_QID_ANY);
  761. if (IS_ERR(rq))
  762. return PTR_ERR(rq);
  763. rq->timeout = ctrl->kato * HZ;
  764. rq->end_io_data = ctrl;
  765. blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
  766. return 0;
  767. }
  768. static void nvme_keep_alive_work(struct work_struct *work)
  769. {
  770. struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
  771. struct nvme_ctrl, ka_work);
  772. if (nvme_keep_alive(ctrl)) {
  773. /* allocation failure, reset the controller */
  774. dev_err(ctrl->device, "keep-alive failed\n");
  775. nvme_reset_ctrl(ctrl);
  776. return;
  777. }
  778. }
  779. static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
  780. {
  781. if (unlikely(ctrl->kato == 0))
  782. return;
  783. schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
  784. }
  785. void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
  786. {
  787. if (unlikely(ctrl->kato == 0))
  788. return;
  789. cancel_delayed_work_sync(&ctrl->ka_work);
  790. }
  791. EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
  792. /*
  793. * In NVMe 1.0 the CNS field was just a binary controller or namespace
  794. * flag, thus sending any new CNS opcodes has a big chance of not working.
  795. * Qemu unfortunately had that bug after reporting a 1.1 version compliance
  796. * (but not for any later version).
  797. */
  798. static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
  799. {
  800. if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
  801. return ctrl->vs < NVME_VS(1, 2, 0);
  802. return ctrl->vs < NVME_VS(1, 1, 0);
  803. }
  804. static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
  805. {
  806. struct nvme_command c = { };
  807. int error;
  808. /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
  809. c.identify.opcode = nvme_admin_identify;
  810. c.identify.cns = NVME_ID_CNS_CTRL;
  811. *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
  812. if (!*id)
  813. return -ENOMEM;
  814. error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
  815. sizeof(struct nvme_id_ctrl));
  816. if (error)
  817. kfree(*id);
  818. return error;
  819. }
  820. static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
  821. struct nvme_ns_ids *ids)
  822. {
  823. struct nvme_command c = { };
  824. int status;
  825. void *data;
  826. int pos;
  827. int len;
  828. c.identify.opcode = nvme_admin_identify;
  829. c.identify.nsid = cpu_to_le32(nsid);
  830. c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
  831. data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
  832. if (!data)
  833. return -ENOMEM;
  834. status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
  835. NVME_IDENTIFY_DATA_SIZE);
  836. if (status)
  837. goto free_data;
  838. for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
  839. struct nvme_ns_id_desc *cur = data + pos;
  840. if (cur->nidl == 0)
  841. break;
  842. switch (cur->nidt) {
  843. case NVME_NIDT_EUI64:
  844. if (cur->nidl != NVME_NIDT_EUI64_LEN) {
  845. dev_warn(ctrl->device,
  846. "ctrl returned bogus length: %d for NVME_NIDT_EUI64\n",
  847. cur->nidl);
  848. goto free_data;
  849. }
  850. len = NVME_NIDT_EUI64_LEN;
  851. memcpy(ids->eui64, data + pos + sizeof(*cur), len);
  852. break;
  853. case NVME_NIDT_NGUID:
  854. if (cur->nidl != NVME_NIDT_NGUID_LEN) {
  855. dev_warn(ctrl->device,
  856. "ctrl returned bogus length: %d for NVME_NIDT_NGUID\n",
  857. cur->nidl);
  858. goto free_data;
  859. }
  860. len = NVME_NIDT_NGUID_LEN;
  861. memcpy(ids->nguid, data + pos + sizeof(*cur), len);
  862. break;
  863. case NVME_NIDT_UUID:
  864. if (cur->nidl != NVME_NIDT_UUID_LEN) {
  865. dev_warn(ctrl->device,
  866. "ctrl returned bogus length: %d for NVME_NIDT_UUID\n",
  867. cur->nidl);
  868. goto free_data;
  869. }
  870. len = NVME_NIDT_UUID_LEN;
  871. uuid_copy(&ids->uuid, data + pos + sizeof(*cur));
  872. break;
  873. default:
  874. /* Skip unnkown types */
  875. len = cur->nidl;
  876. break;
  877. }
  878. len += sizeof(*cur);
  879. }
  880. free_data:
  881. kfree(data);
  882. return status;
  883. }
  884. static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
  885. {
  886. struct nvme_command c = { };
  887. c.identify.opcode = nvme_admin_identify;
  888. c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
  889. c.identify.nsid = cpu_to_le32(nsid);
  890. return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list,
  891. NVME_IDENTIFY_DATA_SIZE);
  892. }
  893. static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl,
  894. unsigned nsid)
  895. {
  896. struct nvme_id_ns *id;
  897. struct nvme_command c = { };
  898. int error;
  899. /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
  900. c.identify.opcode = nvme_admin_identify;
  901. c.identify.nsid = cpu_to_le32(nsid);
  902. c.identify.cns = NVME_ID_CNS_NS;
  903. id = kmalloc(sizeof(*id), GFP_KERNEL);
  904. if (!id)
  905. return NULL;
  906. error = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
  907. if (error) {
  908. dev_warn(ctrl->device, "Identify namespace failed\n");
  909. kfree(id);
  910. return NULL;
  911. }
  912. return id;
  913. }
  914. static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
  915. void *buffer, size_t buflen, u32 *result)
  916. {
  917. union nvme_result res = { 0 };
  918. struct nvme_command c;
  919. int ret;
  920. memset(&c, 0, sizeof(c));
  921. c.features.opcode = nvme_admin_set_features;
  922. c.features.fid = cpu_to_le32(fid);
  923. c.features.dword11 = cpu_to_le32(dword11);
  924. ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
  925. buffer, buflen, 0, NVME_QID_ANY, 0, 0);
  926. if (ret >= 0 && result)
  927. *result = le32_to_cpu(res.u32);
  928. return ret;
  929. }
  930. int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
  931. {
  932. u32 q_count = (*count - 1) | ((*count - 1) << 16);
  933. u32 result;
  934. int status, nr_io_queues;
  935. status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
  936. &result);
  937. if (status < 0)
  938. return status;
  939. /*
  940. * Degraded controllers might return an error when setting the queue
  941. * count. We still want to be able to bring them online and offer
  942. * access to the admin queue, as that might be only way to fix them up.
  943. */
  944. if (status > 0) {
  945. dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
  946. *count = 0;
  947. } else {
  948. nr_io_queues = min(result & 0xffff, result >> 16) + 1;
  949. *count = min(*count, nr_io_queues);
  950. }
  951. return 0;
  952. }
  953. EXPORT_SYMBOL_GPL(nvme_set_queue_count);
  954. #define NVME_AEN_SUPPORTED \
  955. (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | NVME_AEN_CFG_ANA_CHANGE)
  956. static void nvme_enable_aen(struct nvme_ctrl *ctrl)
  957. {
  958. u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
  959. int status;
  960. if (!supported_aens)
  961. return;
  962. status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
  963. NULL, 0, &result);
  964. if (status)
  965. dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
  966. supported_aens);
  967. }
  968. static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
  969. {
  970. struct nvme_user_io io;
  971. struct nvme_command c;
  972. unsigned length, meta_len;
  973. void __user *metadata;
  974. if (copy_from_user(&io, uio, sizeof(io)))
  975. return -EFAULT;
  976. if (io.flags)
  977. return -EINVAL;
  978. switch (io.opcode) {
  979. case nvme_cmd_write:
  980. case nvme_cmd_read:
  981. case nvme_cmd_compare:
  982. break;
  983. default:
  984. return -EINVAL;
  985. }
  986. length = (io.nblocks + 1) << ns->lba_shift;
  987. meta_len = (io.nblocks + 1) * ns->ms;
  988. metadata = (void __user *)(uintptr_t)io.metadata;
  989. if (ns->ext) {
  990. length += meta_len;
  991. meta_len = 0;
  992. } else if (meta_len) {
  993. if ((io.metadata & 3) || !io.metadata)
  994. return -EINVAL;
  995. }
  996. memset(&c, 0, sizeof(c));
  997. c.rw.opcode = io.opcode;
  998. c.rw.flags = io.flags;
  999. c.rw.nsid = cpu_to_le32(ns->head->ns_id);
  1000. c.rw.slba = cpu_to_le64(io.slba);
  1001. c.rw.length = cpu_to_le16(io.nblocks);
  1002. c.rw.control = cpu_to_le16(io.control);
  1003. c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
  1004. c.rw.reftag = cpu_to_le32(io.reftag);
  1005. c.rw.apptag = cpu_to_le16(io.apptag);
  1006. c.rw.appmask = cpu_to_le16(io.appmask);
  1007. return nvme_submit_user_cmd(ns->queue, &c,
  1008. (void __user *)(uintptr_t)io.addr, length,
  1009. metadata, meta_len, io.slba, NULL, 0);
  1010. }
  1011. static u32 nvme_known_admin_effects(u8 opcode)
  1012. {
  1013. switch (opcode) {
  1014. case nvme_admin_format_nvm:
  1015. return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
  1016. NVME_CMD_EFFECTS_CSE_MASK;
  1017. case nvme_admin_sanitize_nvm:
  1018. return NVME_CMD_EFFECTS_CSE_MASK;
  1019. default:
  1020. break;
  1021. }
  1022. return 0;
  1023. }
  1024. static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
  1025. u8 opcode)
  1026. {
  1027. u32 effects = 0;
  1028. if (ns) {
  1029. if (ctrl->effects)
  1030. effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
  1031. if (effects & ~NVME_CMD_EFFECTS_CSUPP)
  1032. dev_warn(ctrl->device,
  1033. "IO command:%02x has unhandled effects:%08x\n",
  1034. opcode, effects);
  1035. return 0;
  1036. }
  1037. if (ctrl->effects)
  1038. effects = le32_to_cpu(ctrl->effects->acs[opcode]);
  1039. else
  1040. effects = nvme_known_admin_effects(opcode);
  1041. /*
  1042. * For simplicity, IO to all namespaces is quiesced even if the command
  1043. * effects say only one namespace is affected.
  1044. */
  1045. if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
  1046. mutex_lock(&ctrl->scan_lock);
  1047. mutex_lock(&ctrl->subsys->lock);
  1048. nvme_mpath_start_freeze(ctrl->subsys);
  1049. nvme_mpath_wait_freeze(ctrl->subsys);
  1050. nvme_start_freeze(ctrl);
  1051. nvme_wait_freeze(ctrl);
  1052. }
  1053. return effects;
  1054. }
  1055. static void nvme_update_formats(struct nvme_ctrl *ctrl)
  1056. {
  1057. struct nvme_ns *ns;
  1058. down_read(&ctrl->namespaces_rwsem);
  1059. list_for_each_entry(ns, &ctrl->namespaces, list)
  1060. if (ns->disk && nvme_revalidate_disk(ns->disk))
  1061. nvme_set_queue_dying(ns);
  1062. up_read(&ctrl->namespaces_rwsem);
  1063. nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
  1064. }
  1065. static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
  1066. {
  1067. /*
  1068. * Revalidate LBA changes prior to unfreezing. This is necessary to
  1069. * prevent memory corruption if a logical block size was changed by
  1070. * this command.
  1071. */
  1072. if (effects & NVME_CMD_EFFECTS_LBCC)
  1073. nvme_update_formats(ctrl);
  1074. if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
  1075. nvme_unfreeze(ctrl);
  1076. nvme_mpath_unfreeze(ctrl->subsys);
  1077. mutex_unlock(&ctrl->subsys->lock);
  1078. mutex_unlock(&ctrl->scan_lock);
  1079. }
  1080. if (effects & NVME_CMD_EFFECTS_CCC)
  1081. nvme_init_identify(ctrl);
  1082. if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC))
  1083. nvme_queue_scan(ctrl);
  1084. }
  1085. static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
  1086. struct nvme_passthru_cmd __user *ucmd)
  1087. {
  1088. struct nvme_passthru_cmd cmd;
  1089. struct nvme_command c;
  1090. unsigned timeout = 0;
  1091. u32 effects;
  1092. int status;
  1093. if (!capable(CAP_SYS_ADMIN))
  1094. return -EACCES;
  1095. if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
  1096. return -EFAULT;
  1097. if (cmd.flags)
  1098. return -EINVAL;
  1099. memset(&c, 0, sizeof(c));
  1100. c.common.opcode = cmd.opcode;
  1101. c.common.flags = cmd.flags;
  1102. c.common.nsid = cpu_to_le32(cmd.nsid);
  1103. c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
  1104. c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
  1105. c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
  1106. c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
  1107. c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
  1108. c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
  1109. c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
  1110. c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
  1111. if (cmd.timeout_ms)
  1112. timeout = msecs_to_jiffies(cmd.timeout_ms);
  1113. effects = nvme_passthru_start(ctrl, ns, cmd.opcode);
  1114. status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
  1115. (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
  1116. (void __user *)(uintptr_t)cmd.metadata, cmd.metadata_len,
  1117. 0, &cmd.result, timeout);
  1118. nvme_passthru_end(ctrl, effects);
  1119. if (status >= 0) {
  1120. if (put_user(cmd.result, &ucmd->result))
  1121. return -EFAULT;
  1122. }
  1123. return status;
  1124. }
  1125. /*
  1126. * Issue ioctl requests on the first available path. Note that unlike normal
  1127. * block layer requests we will not retry failed request on another controller.
  1128. */
  1129. static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
  1130. struct nvme_ns_head **head, int *srcu_idx)
  1131. {
  1132. #ifdef CONFIG_NVME_MULTIPATH
  1133. if (disk->fops == &nvme_ns_head_ops) {
  1134. struct nvme_ns *ns;
  1135. *head = disk->private_data;
  1136. *srcu_idx = srcu_read_lock(&(*head)->srcu);
  1137. ns = nvme_find_path(*head);
  1138. if (!ns)
  1139. srcu_read_unlock(&(*head)->srcu, *srcu_idx);
  1140. return ns;
  1141. }
  1142. #endif
  1143. *head = NULL;
  1144. *srcu_idx = -1;
  1145. return disk->private_data;
  1146. }
  1147. static void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
  1148. {
  1149. if (head)
  1150. srcu_read_unlock(&head->srcu, idx);
  1151. }
  1152. static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
  1153. unsigned int cmd, unsigned long arg)
  1154. {
  1155. struct nvme_ns_head *head = NULL;
  1156. void __user *argp = (void __user *)arg;
  1157. struct nvme_ns *ns;
  1158. int srcu_idx, ret;
  1159. ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
  1160. if (unlikely(!ns))
  1161. return -EWOULDBLOCK;
  1162. /*
  1163. * Handle ioctls that apply to the controller instead of the namespace
  1164. * seperately and drop the ns SRCU reference early. This avoids a
  1165. * deadlock when deleting namespaces using the passthrough interface.
  1166. */
  1167. if (cmd == NVME_IOCTL_ADMIN_CMD || is_sed_ioctl(cmd)) {
  1168. struct nvme_ctrl *ctrl = ns->ctrl;
  1169. nvme_get_ctrl(ns->ctrl);
  1170. nvme_put_ns_from_disk(head, srcu_idx);
  1171. if (cmd == NVME_IOCTL_ADMIN_CMD)
  1172. ret = nvme_user_cmd(ctrl, NULL, argp);
  1173. else
  1174. ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
  1175. nvme_put_ctrl(ctrl);
  1176. return ret;
  1177. }
  1178. switch (cmd) {
  1179. case NVME_IOCTL_ID:
  1180. force_successful_syscall_return();
  1181. ret = ns->head->ns_id;
  1182. break;
  1183. case NVME_IOCTL_IO_CMD:
  1184. ret = nvme_user_cmd(ns->ctrl, ns, argp);
  1185. break;
  1186. case NVME_IOCTL_SUBMIT_IO:
  1187. ret = nvme_submit_io(ns, argp);
  1188. break;
  1189. default:
  1190. if (ns->ndev)
  1191. ret = nvme_nvm_ioctl(ns, cmd, arg);
  1192. else
  1193. ret = -ENOTTY;
  1194. }
  1195. nvme_put_ns_from_disk(head, srcu_idx);
  1196. return ret;
  1197. }
  1198. static int nvme_open(struct block_device *bdev, fmode_t mode)
  1199. {
  1200. struct nvme_ns *ns = bdev->bd_disk->private_data;
  1201. #ifdef CONFIG_NVME_MULTIPATH
  1202. /* should never be called due to GENHD_FL_HIDDEN */
  1203. if (WARN_ON_ONCE(ns->head->disk))
  1204. goto fail;
  1205. #endif
  1206. if (!kref_get_unless_zero(&ns->kref))
  1207. goto fail;
  1208. if (!try_module_get(ns->ctrl->ops->module))
  1209. goto fail_put_ns;
  1210. return 0;
  1211. fail_put_ns:
  1212. nvme_put_ns(ns);
  1213. fail:
  1214. return -ENXIO;
  1215. }
  1216. static void nvme_release(struct gendisk *disk, fmode_t mode)
  1217. {
  1218. struct nvme_ns *ns = disk->private_data;
  1219. module_put(ns->ctrl->ops->module);
  1220. nvme_put_ns(ns);
  1221. }
  1222. static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
  1223. {
  1224. /* some standard values */
  1225. geo->heads = 1 << 6;
  1226. geo->sectors = 1 << 5;
  1227. geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
  1228. return 0;
  1229. }
  1230. #ifdef CONFIG_BLK_DEV_INTEGRITY
  1231. static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
  1232. {
  1233. struct blk_integrity integrity;
  1234. memset(&integrity, 0, sizeof(integrity));
  1235. switch (pi_type) {
  1236. case NVME_NS_DPS_PI_TYPE3:
  1237. integrity.profile = &t10_pi_type3_crc;
  1238. integrity.tag_size = sizeof(u16) + sizeof(u32);
  1239. integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
  1240. break;
  1241. case NVME_NS_DPS_PI_TYPE1:
  1242. case NVME_NS_DPS_PI_TYPE2:
  1243. integrity.profile = &t10_pi_type1_crc;
  1244. integrity.tag_size = sizeof(u16);
  1245. integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
  1246. break;
  1247. default:
  1248. integrity.profile = NULL;
  1249. break;
  1250. }
  1251. integrity.tuple_size = ms;
  1252. blk_integrity_register(disk, &integrity);
  1253. blk_queue_max_integrity_segments(disk->queue, 1);
  1254. }
  1255. #else
  1256. static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
  1257. {
  1258. }
  1259. #endif /* CONFIG_BLK_DEV_INTEGRITY */
  1260. static void nvme_set_chunk_size(struct nvme_ns *ns)
  1261. {
  1262. u32 chunk_size = (((u32)ns->noiob) << (ns->lba_shift - 9));
  1263. blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size));
  1264. }
  1265. static void nvme_config_discard(struct nvme_ns *ns)
  1266. {
  1267. struct nvme_ctrl *ctrl = ns->ctrl;
  1268. struct request_queue *queue = ns->queue;
  1269. u32 size = queue_logical_block_size(queue);
  1270. if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
  1271. blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
  1272. return;
  1273. }
  1274. if (ctrl->nr_streams && ns->sws && ns->sgs)
  1275. size *= ns->sws * ns->sgs;
  1276. BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
  1277. NVME_DSM_MAX_RANGES);
  1278. queue->limits.discard_alignment = 0;
  1279. queue->limits.discard_granularity = size;
  1280. /* If discard is already enabled, don't reset queue limits */
  1281. if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
  1282. return;
  1283. blk_queue_max_discard_sectors(queue, UINT_MAX);
  1284. blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
  1285. if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
  1286. blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
  1287. }
  1288. static void nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
  1289. struct nvme_id_ns *id, struct nvme_ns_ids *ids)
  1290. {
  1291. memset(ids, 0, sizeof(*ids));
  1292. if (ctrl->vs >= NVME_VS(1, 1, 0))
  1293. memcpy(ids->eui64, id->eui64, sizeof(id->eui64));
  1294. if (ctrl->vs >= NVME_VS(1, 2, 0))
  1295. memcpy(ids->nguid, id->nguid, sizeof(id->nguid));
  1296. if (ctrl->vs >= NVME_VS(1, 3, 0)) {
  1297. /* Don't treat error as fatal we potentially
  1298. * already have a NGUID or EUI-64
  1299. */
  1300. if (nvme_identify_ns_descs(ctrl, nsid, ids))
  1301. dev_warn(ctrl->device,
  1302. "%s: Identify Descriptors failed\n", __func__);
  1303. }
  1304. }
  1305. static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
  1306. {
  1307. return !uuid_is_null(&ids->uuid) ||
  1308. memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
  1309. memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
  1310. }
  1311. static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
  1312. {
  1313. return uuid_equal(&a->uuid, &b->uuid) &&
  1314. memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
  1315. memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0;
  1316. }
  1317. static void nvme_update_disk_info(struct gendisk *disk,
  1318. struct nvme_ns *ns, struct nvme_id_ns *id)
  1319. {
  1320. sector_t capacity = le64_to_cpup(&id->nsze) << (ns->lba_shift - 9);
  1321. unsigned short bs = 1 << ns->lba_shift;
  1322. if (ns->lba_shift > PAGE_SHIFT) {
  1323. /* unsupported block size, set capacity to 0 later */
  1324. bs = (1 << 9);
  1325. }
  1326. blk_mq_freeze_queue(disk->queue);
  1327. blk_integrity_unregister(disk);
  1328. blk_queue_logical_block_size(disk->queue, bs);
  1329. blk_queue_physical_block_size(disk->queue, bs);
  1330. blk_queue_io_min(disk->queue, bs);
  1331. if (ns->ms && !ns->ext &&
  1332. (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
  1333. nvme_init_integrity(disk, ns->ms, ns->pi_type);
  1334. if ((ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk)) ||
  1335. ns->lba_shift > PAGE_SHIFT)
  1336. capacity = 0;
  1337. set_capacity(disk, capacity);
  1338. nvme_config_discard(ns);
  1339. if (id->nsattr & (1 << 0))
  1340. set_disk_ro(disk, true);
  1341. else
  1342. set_disk_ro(disk, false);
  1343. blk_mq_unfreeze_queue(disk->queue);
  1344. }
  1345. static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
  1346. {
  1347. struct nvme_ns *ns = disk->private_data;
  1348. /*
  1349. * If identify namespace failed, use default 512 byte block size so
  1350. * block layer can use before failing read/write for 0 capacity.
  1351. */
  1352. ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds;
  1353. if (ns->lba_shift == 0)
  1354. ns->lba_shift = 9;
  1355. ns->noiob = le16_to_cpu(id->noiob);
  1356. ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
  1357. ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
  1358. /* the PI implementation requires metadata equal t10 pi tuple size */
  1359. if (ns->ms == sizeof(struct t10_pi_tuple))
  1360. ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
  1361. else
  1362. ns->pi_type = 0;
  1363. if (ns->noiob)
  1364. nvme_set_chunk_size(ns);
  1365. nvme_update_disk_info(disk, ns, id);
  1366. if (ns->ndev)
  1367. nvme_nvm_update_nvm_info(ns);
  1368. #ifdef CONFIG_NVME_MULTIPATH
  1369. if (ns->head->disk) {
  1370. nvme_update_disk_info(ns->head->disk, ns, id);
  1371. blk_queue_stack_limits(ns->head->disk->queue, ns->queue);
  1372. nvme_mpath_update_disk_size(ns->head->disk);
  1373. }
  1374. #endif
  1375. }
  1376. static int nvme_revalidate_disk(struct gendisk *disk)
  1377. {
  1378. struct nvme_ns *ns = disk->private_data;
  1379. struct nvme_ctrl *ctrl = ns->ctrl;
  1380. struct nvme_id_ns *id;
  1381. struct nvme_ns_ids ids;
  1382. int ret = 0;
  1383. if (test_bit(NVME_NS_DEAD, &ns->flags)) {
  1384. set_capacity(disk, 0);
  1385. return -ENODEV;
  1386. }
  1387. id = nvme_identify_ns(ctrl, ns->head->ns_id);
  1388. if (!id)
  1389. return -ENODEV;
  1390. if (id->ncap == 0) {
  1391. ret = -ENODEV;
  1392. goto out;
  1393. }
  1394. __nvme_revalidate_disk(disk, id);
  1395. nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids);
  1396. if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) {
  1397. dev_err(ctrl->device,
  1398. "identifiers changed for nsid %d\n", ns->head->ns_id);
  1399. ret = -ENODEV;
  1400. }
  1401. out:
  1402. kfree(id);
  1403. return ret;
  1404. }
  1405. static char nvme_pr_type(enum pr_type type)
  1406. {
  1407. switch (type) {
  1408. case PR_WRITE_EXCLUSIVE:
  1409. return 1;
  1410. case PR_EXCLUSIVE_ACCESS:
  1411. return 2;
  1412. case PR_WRITE_EXCLUSIVE_REG_ONLY:
  1413. return 3;
  1414. case PR_EXCLUSIVE_ACCESS_REG_ONLY:
  1415. return 4;
  1416. case PR_WRITE_EXCLUSIVE_ALL_REGS:
  1417. return 5;
  1418. case PR_EXCLUSIVE_ACCESS_ALL_REGS:
  1419. return 6;
  1420. default:
  1421. return 0;
  1422. }
  1423. };
  1424. static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
  1425. u64 key, u64 sa_key, u8 op)
  1426. {
  1427. struct nvme_ns_head *head = NULL;
  1428. struct nvme_ns *ns;
  1429. struct nvme_command c;
  1430. int srcu_idx, ret;
  1431. u8 data[16] = { 0, };
  1432. ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
  1433. if (unlikely(!ns))
  1434. return -EWOULDBLOCK;
  1435. put_unaligned_le64(key, &data[0]);
  1436. put_unaligned_le64(sa_key, &data[8]);
  1437. memset(&c, 0, sizeof(c));
  1438. c.common.opcode = op;
  1439. c.common.nsid = cpu_to_le32(ns->head->ns_id);
  1440. c.common.cdw10[0] = cpu_to_le32(cdw10);
  1441. ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
  1442. nvme_put_ns_from_disk(head, srcu_idx);
  1443. return ret;
  1444. }
  1445. static int nvme_pr_register(struct block_device *bdev, u64 old,
  1446. u64 new, unsigned flags)
  1447. {
  1448. u32 cdw10;
  1449. if (flags & ~PR_FL_IGNORE_KEY)
  1450. return -EOPNOTSUPP;
  1451. cdw10 = old ? 2 : 0;
  1452. cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
  1453. cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
  1454. return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
  1455. }
  1456. static int nvme_pr_reserve(struct block_device *bdev, u64 key,
  1457. enum pr_type type, unsigned flags)
  1458. {
  1459. u32 cdw10;
  1460. if (flags & ~PR_FL_IGNORE_KEY)
  1461. return -EOPNOTSUPP;
  1462. cdw10 = nvme_pr_type(type) << 8;
  1463. cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
  1464. return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
  1465. }
  1466. static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
  1467. enum pr_type type, bool abort)
  1468. {
  1469. u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
  1470. return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
  1471. }
  1472. static int nvme_pr_clear(struct block_device *bdev, u64 key)
  1473. {
  1474. u32 cdw10 = 1 | (key ? 1 << 3 : 0);
  1475. return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
  1476. }
  1477. static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
  1478. {
  1479. u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
  1480. return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
  1481. }
  1482. static const struct pr_ops nvme_pr_ops = {
  1483. .pr_register = nvme_pr_register,
  1484. .pr_reserve = nvme_pr_reserve,
  1485. .pr_release = nvme_pr_release,
  1486. .pr_preempt = nvme_pr_preempt,
  1487. .pr_clear = nvme_pr_clear,
  1488. };
  1489. #ifdef CONFIG_BLK_SED_OPAL
  1490. int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
  1491. bool send)
  1492. {
  1493. struct nvme_ctrl *ctrl = data;
  1494. struct nvme_command cmd;
  1495. memset(&cmd, 0, sizeof(cmd));
  1496. if (send)
  1497. cmd.common.opcode = nvme_admin_security_send;
  1498. else
  1499. cmd.common.opcode = nvme_admin_security_recv;
  1500. cmd.common.nsid = 0;
  1501. cmd.common.cdw10[0] = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
  1502. cmd.common.cdw10[1] = cpu_to_le32(len);
  1503. return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
  1504. ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0);
  1505. }
  1506. EXPORT_SYMBOL_GPL(nvme_sec_submit);
  1507. #endif /* CONFIG_BLK_SED_OPAL */
  1508. static const struct block_device_operations nvme_fops = {
  1509. .owner = THIS_MODULE,
  1510. .ioctl = nvme_ioctl,
  1511. .compat_ioctl = nvme_ioctl,
  1512. .open = nvme_open,
  1513. .release = nvme_release,
  1514. .getgeo = nvme_getgeo,
  1515. .revalidate_disk= nvme_revalidate_disk,
  1516. .pr_ops = &nvme_pr_ops,
  1517. };
  1518. #ifdef CONFIG_NVME_MULTIPATH
  1519. static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
  1520. {
  1521. struct nvme_ns_head *head = bdev->bd_disk->private_data;
  1522. if (!kref_get_unless_zero(&head->ref))
  1523. return -ENXIO;
  1524. return 0;
  1525. }
  1526. static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
  1527. {
  1528. nvme_put_ns_head(disk->private_data);
  1529. }
  1530. const struct block_device_operations nvme_ns_head_ops = {
  1531. .owner = THIS_MODULE,
  1532. .open = nvme_ns_head_open,
  1533. .release = nvme_ns_head_release,
  1534. .ioctl = nvme_ioctl,
  1535. .compat_ioctl = nvme_ioctl,
  1536. .getgeo = nvme_getgeo,
  1537. .pr_ops = &nvme_pr_ops,
  1538. };
  1539. #endif /* CONFIG_NVME_MULTIPATH */
  1540. static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
  1541. {
  1542. unsigned long timeout =
  1543. ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
  1544. u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
  1545. int ret;
  1546. while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
  1547. if (csts == ~0)
  1548. return -ENODEV;
  1549. if ((csts & NVME_CSTS_RDY) == bit)
  1550. break;
  1551. msleep(100);
  1552. if (fatal_signal_pending(current))
  1553. return -EINTR;
  1554. if (time_after(jiffies, timeout)) {
  1555. dev_err(ctrl->device,
  1556. "Device not ready; aborting %s\n", enabled ?
  1557. "initialisation" : "reset");
  1558. return -ENODEV;
  1559. }
  1560. }
  1561. return ret;
  1562. }
  1563. /*
  1564. * If the device has been passed off to us in an enabled state, just clear
  1565. * the enabled bit. The spec says we should set the 'shutdown notification
  1566. * bits', but doing so may cause the device to complete commands to the
  1567. * admin queue ... and we don't know what memory that might be pointing at!
  1568. */
  1569. int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
  1570. {
  1571. int ret;
  1572. ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
  1573. ctrl->ctrl_config &= ~NVME_CC_ENABLE;
  1574. ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
  1575. if (ret)
  1576. return ret;
  1577. if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
  1578. msleep(NVME_QUIRK_DELAY_AMOUNT);
  1579. return nvme_wait_ready(ctrl, cap, false);
  1580. }
  1581. EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
  1582. int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
  1583. {
  1584. /*
  1585. * Default to a 4K page size, with the intention to update this
  1586. * path in the future to accomodate architectures with differing
  1587. * kernel and IO page sizes.
  1588. */
  1589. unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
  1590. int ret;
  1591. if (page_shift < dev_page_min) {
  1592. dev_err(ctrl->device,
  1593. "Minimum device page size %u too large for host (%u)\n",
  1594. 1 << dev_page_min, 1 << page_shift);
  1595. return -ENODEV;
  1596. }
  1597. ctrl->page_size = 1 << page_shift;
  1598. ctrl->ctrl_config = NVME_CC_CSS_NVM;
  1599. ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
  1600. ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
  1601. ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
  1602. ctrl->ctrl_config |= NVME_CC_ENABLE;
  1603. ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
  1604. if (ret)
  1605. return ret;
  1606. return nvme_wait_ready(ctrl, cap, true);
  1607. }
  1608. EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
  1609. int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
  1610. {
  1611. unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
  1612. u32 csts;
  1613. int ret;
  1614. ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
  1615. ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
  1616. ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
  1617. if (ret)
  1618. return ret;
  1619. while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
  1620. if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
  1621. break;
  1622. msleep(100);
  1623. if (fatal_signal_pending(current))
  1624. return -EINTR;
  1625. if (time_after(jiffies, timeout)) {
  1626. dev_err(ctrl->device,
  1627. "Device shutdown incomplete; abort shutdown\n");
  1628. return -ENODEV;
  1629. }
  1630. }
  1631. return ret;
  1632. }
  1633. EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
  1634. static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
  1635. struct request_queue *q)
  1636. {
  1637. bool vwc = false;
  1638. if (ctrl->max_hw_sectors) {
  1639. u32 max_segments =
  1640. (ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;
  1641. max_segments = min_not_zero(max_segments, ctrl->max_segments);
  1642. blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
  1643. blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
  1644. }
  1645. if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
  1646. is_power_of_2(ctrl->max_hw_sectors))
  1647. blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
  1648. blk_queue_virt_boundary(q, ctrl->page_size - 1);
  1649. if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
  1650. vwc = true;
  1651. blk_queue_write_cache(q, vwc, vwc);
  1652. }
  1653. static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
  1654. {
  1655. __le64 ts;
  1656. int ret;
  1657. if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
  1658. return 0;
  1659. ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
  1660. ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
  1661. NULL);
  1662. if (ret)
  1663. dev_warn_once(ctrl->device,
  1664. "could not set timestamp (%d)\n", ret);
  1665. return ret;
  1666. }
  1667. static int nvme_configure_apst(struct nvme_ctrl *ctrl)
  1668. {
  1669. /*
  1670. * APST (Autonomous Power State Transition) lets us program a
  1671. * table of power state transitions that the controller will
  1672. * perform automatically. We configure it with a simple
  1673. * heuristic: we are willing to spend at most 2% of the time
  1674. * transitioning between power states. Therefore, when running
  1675. * in any given state, we will enter the next lower-power
  1676. * non-operational state after waiting 50 * (enlat + exlat)
  1677. * microseconds, as long as that state's exit latency is under
  1678. * the requested maximum latency.
  1679. *
  1680. * We will not autonomously enter any non-operational state for
  1681. * which the total latency exceeds ps_max_latency_us. Users
  1682. * can set ps_max_latency_us to zero to turn off APST.
  1683. */
  1684. unsigned apste;
  1685. struct nvme_feat_auto_pst *table;
  1686. u64 max_lat_us = 0;
  1687. int max_ps = -1;
  1688. int ret;
  1689. /*
  1690. * If APST isn't supported or if we haven't been initialized yet,
  1691. * then don't do anything.
  1692. */
  1693. if (!ctrl->apsta)
  1694. return 0;
  1695. if (ctrl->npss > 31) {
  1696. dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
  1697. return 0;
  1698. }
  1699. table = kzalloc(sizeof(*table), GFP_KERNEL);
  1700. if (!table)
  1701. return 0;
  1702. if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
  1703. /* Turn off APST. */
  1704. apste = 0;
  1705. dev_dbg(ctrl->device, "APST disabled\n");
  1706. } else {
  1707. __le64 target = cpu_to_le64(0);
  1708. int state;
  1709. /*
  1710. * Walk through all states from lowest- to highest-power.
  1711. * According to the spec, lower-numbered states use more
  1712. * power. NPSS, despite the name, is the index of the
  1713. * lowest-power state, not the number of states.
  1714. */
  1715. for (state = (int)ctrl->npss; state >= 0; state--) {
  1716. u64 total_latency_us, exit_latency_us, transition_ms;
  1717. if (target)
  1718. table->entries[state] = target;
  1719. /*
  1720. * Don't allow transitions to the deepest state
  1721. * if it's quirked off.
  1722. */
  1723. if (state == ctrl->npss &&
  1724. (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
  1725. continue;
  1726. /*
  1727. * Is this state a useful non-operational state for
  1728. * higher-power states to autonomously transition to?
  1729. */
  1730. if (!(ctrl->psd[state].flags &
  1731. NVME_PS_FLAGS_NON_OP_STATE))
  1732. continue;
  1733. exit_latency_us =
  1734. (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
  1735. if (exit_latency_us > ctrl->ps_max_latency_us)
  1736. continue;
  1737. total_latency_us =
  1738. exit_latency_us +
  1739. le32_to_cpu(ctrl->psd[state].entry_lat);
  1740. /*
  1741. * This state is good. Use it as the APST idle
  1742. * target for higher power states.
  1743. */
  1744. transition_ms = total_latency_us + 19;
  1745. do_div(transition_ms, 20);
  1746. if (transition_ms > (1 << 24) - 1)
  1747. transition_ms = (1 << 24) - 1;
  1748. target = cpu_to_le64((state << 3) |
  1749. (transition_ms << 8));
  1750. if (max_ps == -1)
  1751. max_ps = state;
  1752. if (total_latency_us > max_lat_us)
  1753. max_lat_us = total_latency_us;
  1754. }
  1755. apste = 1;
  1756. if (max_ps == -1) {
  1757. dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
  1758. } else {
  1759. dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
  1760. max_ps, max_lat_us, (int)sizeof(*table), table);
  1761. }
  1762. }
  1763. ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
  1764. table, sizeof(*table), NULL);
  1765. if (ret)
  1766. dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
  1767. kfree(table);
  1768. return ret;
  1769. }
  1770. static void nvme_set_latency_tolerance(struct device *dev, s32 val)
  1771. {
  1772. struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
  1773. u64 latency;
  1774. switch (val) {
  1775. case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
  1776. case PM_QOS_LATENCY_ANY:
  1777. latency = U64_MAX;
  1778. break;
  1779. default:
  1780. latency = val;
  1781. }
  1782. if (ctrl->ps_max_latency_us != latency) {
  1783. ctrl->ps_max_latency_us = latency;
  1784. if (ctrl->state == NVME_CTRL_LIVE)
  1785. nvme_configure_apst(ctrl);
  1786. }
  1787. }
  1788. struct nvme_core_quirk_entry {
  1789. /*
  1790. * NVMe model and firmware strings are padded with spaces. For
  1791. * simplicity, strings in the quirk table are padded with NULLs
  1792. * instead.
  1793. */
  1794. u16 vid;
  1795. const char *mn;
  1796. const char *fr;
  1797. unsigned long quirks;
  1798. };
  1799. static const struct nvme_core_quirk_entry core_quirks[] = {
  1800. {
  1801. /*
  1802. * This Toshiba device seems to die using any APST states. See:
  1803. * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
  1804. */
  1805. .vid = 0x1179,
  1806. .mn = "THNSF5256GPUK TOSHIBA",
  1807. .quirks = NVME_QUIRK_NO_APST,
  1808. }
  1809. };
  1810. /* match is null-terminated but idstr is space-padded. */
  1811. static bool string_matches(const char *idstr, const char *match, size_t len)
  1812. {
  1813. size_t matchlen;
  1814. if (!match)
  1815. return true;
  1816. matchlen = strlen(match);
  1817. WARN_ON_ONCE(matchlen > len);
  1818. if (memcmp(idstr, match, matchlen))
  1819. return false;
  1820. for (; matchlen < len; matchlen++)
  1821. if (idstr[matchlen] != ' ')
  1822. return false;
  1823. return true;
  1824. }
  1825. static bool quirk_matches(const struct nvme_id_ctrl *id,
  1826. const struct nvme_core_quirk_entry *q)
  1827. {
  1828. return q->vid == le16_to_cpu(id->vid) &&
  1829. string_matches(id->mn, q->mn, sizeof(id->mn)) &&
  1830. string_matches(id->fr, q->fr, sizeof(id->fr));
  1831. }
  1832. static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
  1833. struct nvme_id_ctrl *id)
  1834. {
  1835. size_t nqnlen;
  1836. int off;
  1837. nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
  1838. if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
  1839. strncpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
  1840. return;
  1841. }
  1842. if (ctrl->vs >= NVME_VS(1, 2, 1))
  1843. dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
  1844. /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
  1845. off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
  1846. "nqn.2014.08.org.nvmexpress:%04x%04x",
  1847. le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
  1848. memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
  1849. off += sizeof(id->sn);
  1850. memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
  1851. off += sizeof(id->mn);
  1852. memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
  1853. }
  1854. static void __nvme_release_subsystem(struct nvme_subsystem *subsys)
  1855. {
  1856. ida_simple_remove(&nvme_subsystems_ida, subsys->instance);
  1857. kfree(subsys);
  1858. }
  1859. static void nvme_release_subsystem(struct device *dev)
  1860. {
  1861. __nvme_release_subsystem(container_of(dev, struct nvme_subsystem, dev));
  1862. }
  1863. static void nvme_destroy_subsystem(struct kref *ref)
  1864. {
  1865. struct nvme_subsystem *subsys =
  1866. container_of(ref, struct nvme_subsystem, ref);
  1867. mutex_lock(&nvme_subsystems_lock);
  1868. list_del(&subsys->entry);
  1869. mutex_unlock(&nvme_subsystems_lock);
  1870. ida_destroy(&subsys->ns_ida);
  1871. device_del(&subsys->dev);
  1872. put_device(&subsys->dev);
  1873. }
  1874. static void nvme_put_subsystem(struct nvme_subsystem *subsys)
  1875. {
  1876. kref_put(&subsys->ref, nvme_destroy_subsystem);
  1877. }
  1878. static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
  1879. {
  1880. struct nvme_subsystem *subsys;
  1881. lockdep_assert_held(&nvme_subsystems_lock);
  1882. /*
  1883. * Fail matches for discovery subsystems. This results
  1884. * in each discovery controller bound to a unique subsystem.
  1885. * This avoids issues with validating controller values
  1886. * that can only be true when there is a single unique subsystem.
  1887. * There may be multiple and completely independent entities
  1888. * that provide discovery controllers.
  1889. */
  1890. if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
  1891. return NULL;
  1892. list_for_each_entry(subsys, &nvme_subsystems, entry) {
  1893. if (strcmp(subsys->subnqn, subsysnqn))
  1894. continue;
  1895. if (!kref_get_unless_zero(&subsys->ref))
  1896. continue;
  1897. return subsys;
  1898. }
  1899. return NULL;
  1900. }
  1901. #define SUBSYS_ATTR_RO(_name, _mode, _show) \
  1902. struct device_attribute subsys_attr_##_name = \
  1903. __ATTR(_name, _mode, _show, NULL)
  1904. static ssize_t nvme_subsys_show_nqn(struct device *dev,
  1905. struct device_attribute *attr,
  1906. char *buf)
  1907. {
  1908. struct nvme_subsystem *subsys =
  1909. container_of(dev, struct nvme_subsystem, dev);
  1910. return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
  1911. }
  1912. static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
  1913. #define nvme_subsys_show_str_function(field) \
  1914. static ssize_t subsys_##field##_show(struct device *dev, \
  1915. struct device_attribute *attr, char *buf) \
  1916. { \
  1917. struct nvme_subsystem *subsys = \
  1918. container_of(dev, struct nvme_subsystem, dev); \
  1919. return sprintf(buf, "%.*s\n", \
  1920. (int)sizeof(subsys->field), subsys->field); \
  1921. } \
  1922. static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
  1923. nvme_subsys_show_str_function(model);
  1924. nvme_subsys_show_str_function(serial);
  1925. nvme_subsys_show_str_function(firmware_rev);
  1926. static struct attribute *nvme_subsys_attrs[] = {
  1927. &subsys_attr_model.attr,
  1928. &subsys_attr_serial.attr,
  1929. &subsys_attr_firmware_rev.attr,
  1930. &subsys_attr_subsysnqn.attr,
  1931. NULL,
  1932. };
  1933. static struct attribute_group nvme_subsys_attrs_group = {
  1934. .attrs = nvme_subsys_attrs,
  1935. };
  1936. static const struct attribute_group *nvme_subsys_attrs_groups[] = {
  1937. &nvme_subsys_attrs_group,
  1938. NULL,
  1939. };
  1940. static int nvme_active_ctrls(struct nvme_subsystem *subsys)
  1941. {
  1942. int count = 0;
  1943. struct nvme_ctrl *ctrl;
  1944. mutex_lock(&subsys->lock);
  1945. list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
  1946. if (ctrl->state != NVME_CTRL_DELETING &&
  1947. ctrl->state != NVME_CTRL_DEAD)
  1948. count++;
  1949. }
  1950. mutex_unlock(&subsys->lock);
  1951. return count;
  1952. }
  1953. static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
  1954. {
  1955. struct nvme_subsystem *subsys, *found;
  1956. int ret;
  1957. subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
  1958. if (!subsys)
  1959. return -ENOMEM;
  1960. ret = ida_simple_get(&nvme_subsystems_ida, 0, 0, GFP_KERNEL);
  1961. if (ret < 0) {
  1962. kfree(subsys);
  1963. return ret;
  1964. }
  1965. subsys->instance = ret;
  1966. mutex_init(&subsys->lock);
  1967. kref_init(&subsys->ref);
  1968. INIT_LIST_HEAD(&subsys->ctrls);
  1969. INIT_LIST_HEAD(&subsys->nsheads);
  1970. nvme_init_subnqn(subsys, ctrl, id);
  1971. memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
  1972. memcpy(subsys->model, id->mn, sizeof(subsys->model));
  1973. memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
  1974. subsys->vendor_id = le16_to_cpu(id->vid);
  1975. subsys->cmic = id->cmic;
  1976. subsys->dev.class = nvme_subsys_class;
  1977. subsys->dev.release = nvme_release_subsystem;
  1978. subsys->dev.groups = nvme_subsys_attrs_groups;
  1979. dev_set_name(&subsys->dev, "nvme-subsys%d", subsys->instance);
  1980. device_initialize(&subsys->dev);
  1981. mutex_lock(&nvme_subsystems_lock);
  1982. found = __nvme_find_get_subsystem(subsys->subnqn);
  1983. if (found) {
  1984. /*
  1985. * Verify that the subsystem actually supports multiple
  1986. * controllers, else bail out.
  1987. */
  1988. if (!(ctrl->opts && ctrl->opts->discovery_nqn) &&
  1989. nvme_active_ctrls(found) && !(id->cmic & (1 << 1))) {
  1990. dev_err(ctrl->device,
  1991. "ignoring ctrl due to duplicate subnqn (%s).\n",
  1992. found->subnqn);
  1993. nvme_put_subsystem(found);
  1994. ret = -EINVAL;
  1995. goto out_unlock;
  1996. }
  1997. __nvme_release_subsystem(subsys);
  1998. subsys = found;
  1999. } else {
  2000. ret = device_add(&subsys->dev);
  2001. if (ret) {
  2002. dev_err(ctrl->device,
  2003. "failed to register subsystem device.\n");
  2004. goto out_unlock;
  2005. }
  2006. ida_init(&subsys->ns_ida);
  2007. list_add_tail(&subsys->entry, &nvme_subsystems);
  2008. }
  2009. ctrl->subsys = subsys;
  2010. mutex_unlock(&nvme_subsystems_lock);
  2011. if (sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
  2012. dev_name(ctrl->device))) {
  2013. dev_err(ctrl->device,
  2014. "failed to create sysfs link from subsystem.\n");
  2015. /* the transport driver will eventually put the subsystem */
  2016. return -EINVAL;
  2017. }
  2018. mutex_lock(&subsys->lock);
  2019. list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
  2020. mutex_unlock(&subsys->lock);
  2021. return 0;
  2022. out_unlock:
  2023. mutex_unlock(&nvme_subsystems_lock);
  2024. put_device(&subsys->dev);
  2025. return ret;
  2026. }
  2027. int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
  2028. void *log, size_t size, u64 offset)
  2029. {
  2030. struct nvme_command c = { };
  2031. unsigned long dwlen = size / 4 - 1;
  2032. c.get_log_page.opcode = nvme_admin_get_log_page;
  2033. c.get_log_page.nsid = cpu_to_le32(nsid);
  2034. c.get_log_page.lid = log_page;
  2035. c.get_log_page.lsp = lsp;
  2036. c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
  2037. c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
  2038. c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
  2039. c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
  2040. return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
  2041. }
  2042. static int nvme_get_effects_log(struct nvme_ctrl *ctrl)
  2043. {
  2044. int ret;
  2045. if (!ctrl->effects)
  2046. ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
  2047. if (!ctrl->effects)
  2048. return 0;
  2049. ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0,
  2050. ctrl->effects, sizeof(*ctrl->effects), 0);
  2051. if (ret) {
  2052. kfree(ctrl->effects);
  2053. ctrl->effects = NULL;
  2054. }
  2055. return ret;
  2056. }
  2057. /*
  2058. * Initialize the cached copies of the Identify data and various controller
  2059. * register in our nvme_ctrl structure. This should be called as soon as
  2060. * the admin queue is fully up and running.
  2061. */
  2062. int nvme_init_identify(struct nvme_ctrl *ctrl)
  2063. {
  2064. struct nvme_id_ctrl *id;
  2065. u64 cap;
  2066. int ret, page_shift;
  2067. u32 max_hw_sectors;
  2068. bool prev_apst_enabled;
  2069. ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
  2070. if (ret) {
  2071. dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
  2072. return ret;
  2073. }
  2074. ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
  2075. if (ret) {
  2076. dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
  2077. return ret;
  2078. }
  2079. page_shift = NVME_CAP_MPSMIN(cap) + 12;
  2080. if (ctrl->vs >= NVME_VS(1, 1, 0))
  2081. ctrl->subsystem = NVME_CAP_NSSRC(cap);
  2082. ret = nvme_identify_ctrl(ctrl, &id);
  2083. if (ret) {
  2084. dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
  2085. return -EIO;
  2086. }
  2087. if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
  2088. ret = nvme_get_effects_log(ctrl);
  2089. if (ret < 0)
  2090. goto out_free;
  2091. }
  2092. if (!ctrl->identified) {
  2093. int i;
  2094. ret = nvme_init_subsystem(ctrl, id);
  2095. if (ret)
  2096. goto out_free;
  2097. /*
  2098. * Check for quirks. Quirk can depend on firmware version,
  2099. * so, in principle, the set of quirks present can change
  2100. * across a reset. As a possible future enhancement, we
  2101. * could re-scan for quirks every time we reinitialize
  2102. * the device, but we'd have to make sure that the driver
  2103. * behaves intelligently if the quirks change.
  2104. */
  2105. for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
  2106. if (quirk_matches(id, &core_quirks[i]))
  2107. ctrl->quirks |= core_quirks[i].quirks;
  2108. }
  2109. }
  2110. if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
  2111. dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
  2112. ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
  2113. }
  2114. ctrl->oacs = le16_to_cpu(id->oacs);
  2115. ctrl->oncs = le16_to_cpup(&id->oncs);
  2116. ctrl->oaes = le32_to_cpu(id->oaes);
  2117. atomic_set(&ctrl->abort_limit, id->acl + 1);
  2118. ctrl->vwc = id->vwc;
  2119. ctrl->cntlid = le16_to_cpup(&id->cntlid);
  2120. if (id->mdts)
  2121. max_hw_sectors = 1 << (id->mdts + page_shift - 9);
  2122. else
  2123. max_hw_sectors = UINT_MAX;
  2124. ctrl->max_hw_sectors =
  2125. min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
  2126. nvme_set_queue_limits(ctrl, ctrl->admin_q);
  2127. ctrl->sgls = le32_to_cpu(id->sgls);
  2128. ctrl->kas = le16_to_cpu(id->kas);
  2129. ctrl->max_namespaces = le32_to_cpu(id->mnan);
  2130. if (id->rtd3e) {
  2131. /* us -> s */
  2132. u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000;
  2133. ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
  2134. shutdown_timeout, 60);
  2135. if (ctrl->shutdown_timeout != shutdown_timeout)
  2136. dev_info(ctrl->device,
  2137. "Shutdown timeout set to %u seconds\n",
  2138. ctrl->shutdown_timeout);
  2139. } else
  2140. ctrl->shutdown_timeout = shutdown_timeout;
  2141. ctrl->npss = id->npss;
  2142. ctrl->apsta = id->apsta;
  2143. prev_apst_enabled = ctrl->apst_enabled;
  2144. if (ctrl->quirks & NVME_QUIRK_NO_APST) {
  2145. if (force_apst && id->apsta) {
  2146. dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
  2147. ctrl->apst_enabled = true;
  2148. } else {
  2149. ctrl->apst_enabled = false;
  2150. }
  2151. } else {
  2152. ctrl->apst_enabled = id->apsta;
  2153. }
  2154. memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
  2155. if (ctrl->ops->flags & NVME_F_FABRICS) {
  2156. ctrl->icdoff = le16_to_cpu(id->icdoff);
  2157. ctrl->ioccsz = le32_to_cpu(id->ioccsz);
  2158. ctrl->iorcsz = le32_to_cpu(id->iorcsz);
  2159. ctrl->maxcmd = le16_to_cpu(id->maxcmd);
  2160. /*
  2161. * In fabrics we need to verify the cntlid matches the
  2162. * admin connect
  2163. */
  2164. if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
  2165. ret = -EINVAL;
  2166. goto out_free;
  2167. }
  2168. if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
  2169. dev_err(ctrl->device,
  2170. "keep-alive support is mandatory for fabrics\n");
  2171. ret = -EINVAL;
  2172. goto out_free;
  2173. }
  2174. } else {
  2175. ctrl->cntlid = le16_to_cpu(id->cntlid);
  2176. ctrl->hmpre = le32_to_cpu(id->hmpre);
  2177. ctrl->hmmin = le32_to_cpu(id->hmmin);
  2178. ctrl->hmminds = le32_to_cpu(id->hmminds);
  2179. ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
  2180. }
  2181. ret = nvme_mpath_init(ctrl, id);
  2182. kfree(id);
  2183. if (ret < 0)
  2184. return ret;
  2185. if (ctrl->apst_enabled && !prev_apst_enabled)
  2186. dev_pm_qos_expose_latency_tolerance(ctrl->device);
  2187. else if (!ctrl->apst_enabled && prev_apst_enabled)
  2188. dev_pm_qos_hide_latency_tolerance(ctrl->device);
  2189. ret = nvme_configure_apst(ctrl);
  2190. if (ret < 0)
  2191. return ret;
  2192. ret = nvme_configure_timestamp(ctrl);
  2193. if (ret < 0)
  2194. return ret;
  2195. ret = nvme_configure_directives(ctrl);
  2196. if (ret < 0)
  2197. return ret;
  2198. ctrl->identified = true;
  2199. return 0;
  2200. out_free:
  2201. kfree(id);
  2202. return ret;
  2203. }
  2204. EXPORT_SYMBOL_GPL(nvme_init_identify);
  2205. static int nvme_dev_open(struct inode *inode, struct file *file)
  2206. {
  2207. struct nvme_ctrl *ctrl =
  2208. container_of(inode->i_cdev, struct nvme_ctrl, cdev);
  2209. switch (ctrl->state) {
  2210. case NVME_CTRL_LIVE:
  2211. case NVME_CTRL_ADMIN_ONLY:
  2212. break;
  2213. default:
  2214. return -EWOULDBLOCK;
  2215. }
  2216. nvme_get_ctrl(ctrl);
  2217. if (!try_module_get(ctrl->ops->module)) {
  2218. nvme_put_ctrl(ctrl);
  2219. return -EINVAL;
  2220. }
  2221. file->private_data = ctrl;
  2222. return 0;
  2223. }
  2224. static int nvme_dev_release(struct inode *inode, struct file *file)
  2225. {
  2226. struct nvme_ctrl *ctrl =
  2227. container_of(inode->i_cdev, struct nvme_ctrl, cdev);
  2228. module_put(ctrl->ops->module);
  2229. nvme_put_ctrl(ctrl);
  2230. return 0;
  2231. }
  2232. static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
  2233. {
  2234. struct nvme_ns *ns;
  2235. int ret;
  2236. down_read(&ctrl->namespaces_rwsem);
  2237. if (list_empty(&ctrl->namespaces)) {
  2238. ret = -ENOTTY;
  2239. goto out_unlock;
  2240. }
  2241. ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
  2242. if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
  2243. dev_warn(ctrl->device,
  2244. "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
  2245. ret = -EINVAL;
  2246. goto out_unlock;
  2247. }
  2248. dev_warn(ctrl->device,
  2249. "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
  2250. kref_get(&ns->kref);
  2251. up_read(&ctrl->namespaces_rwsem);
  2252. ret = nvme_user_cmd(ctrl, ns, argp);
  2253. nvme_put_ns(ns);
  2254. return ret;
  2255. out_unlock:
  2256. up_read(&ctrl->namespaces_rwsem);
  2257. return ret;
  2258. }
  2259. static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
  2260. unsigned long arg)
  2261. {
  2262. struct nvme_ctrl *ctrl = file->private_data;
  2263. void __user *argp = (void __user *)arg;
  2264. switch (cmd) {
  2265. case NVME_IOCTL_ADMIN_CMD:
  2266. return nvme_user_cmd(ctrl, NULL, argp);
  2267. case NVME_IOCTL_IO_CMD:
  2268. return nvme_dev_user_cmd(ctrl, argp);
  2269. case NVME_IOCTL_RESET:
  2270. dev_warn(ctrl->device, "resetting controller\n");
  2271. return nvme_reset_ctrl_sync(ctrl);
  2272. case NVME_IOCTL_SUBSYS_RESET:
  2273. return nvme_reset_subsystem(ctrl);
  2274. case NVME_IOCTL_RESCAN:
  2275. nvme_queue_scan(ctrl);
  2276. return 0;
  2277. default:
  2278. return -ENOTTY;
  2279. }
  2280. }
  2281. static const struct file_operations nvme_dev_fops = {
  2282. .owner = THIS_MODULE,
  2283. .open = nvme_dev_open,
  2284. .release = nvme_dev_release,
  2285. .unlocked_ioctl = nvme_dev_ioctl,
  2286. .compat_ioctl = nvme_dev_ioctl,
  2287. };
  2288. static ssize_t nvme_sysfs_reset(struct device *dev,
  2289. struct device_attribute *attr, const char *buf,
  2290. size_t count)
  2291. {
  2292. struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
  2293. int ret;
  2294. ret = nvme_reset_ctrl_sync(ctrl);
  2295. if (ret < 0)
  2296. return ret;
  2297. return count;
  2298. }
  2299. static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
  2300. static ssize_t nvme_sysfs_rescan(struct device *dev,
  2301. struct device_attribute *attr, const char *buf,
  2302. size_t count)
  2303. {
  2304. struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
  2305. nvme_queue_scan(ctrl);
  2306. return count;
  2307. }
  2308. static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
  2309. static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
  2310. {
  2311. struct gendisk *disk = dev_to_disk(dev);
  2312. if (disk->fops == &nvme_fops)
  2313. return nvme_get_ns_from_dev(dev)->head;
  2314. else
  2315. return disk->private_data;
  2316. }
  2317. static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
  2318. char *buf)
  2319. {
  2320. struct nvme_ns_head *head = dev_to_ns_head(dev);
  2321. struct nvme_ns_ids *ids = &head->ids;
  2322. struct nvme_subsystem *subsys = head->subsys;
  2323. int serial_len = sizeof(subsys->serial);
  2324. int model_len = sizeof(subsys->model);
  2325. if (!uuid_is_null(&ids->uuid))
  2326. return sprintf(buf, "uuid.%pU\n", &ids->uuid);
  2327. if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
  2328. return sprintf(buf, "eui.%16phN\n", ids->nguid);
  2329. if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
  2330. return sprintf(buf, "eui.%8phN\n", ids->eui64);
  2331. while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
  2332. subsys->serial[serial_len - 1] == '\0'))
  2333. serial_len--;
  2334. while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
  2335. subsys->model[model_len - 1] == '\0'))
  2336. model_len--;
  2337. return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
  2338. serial_len, subsys->serial, model_len, subsys->model,
  2339. head->ns_id);
  2340. }
  2341. static DEVICE_ATTR_RO(wwid);
  2342. static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
  2343. char *buf)
  2344. {
  2345. return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
  2346. }
  2347. static DEVICE_ATTR_RO(nguid);
  2348. static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
  2349. char *buf)
  2350. {
  2351. struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
  2352. /* For backward compatibility expose the NGUID to userspace if
  2353. * we have no UUID set
  2354. */
  2355. if (uuid_is_null(&ids->uuid)) {
  2356. printk_ratelimited(KERN_WARNING
  2357. "No UUID available providing old NGUID\n");
  2358. return sprintf(buf, "%pU\n", ids->nguid);
  2359. }
  2360. return sprintf(buf, "%pU\n", &ids->uuid);
  2361. }
  2362. static DEVICE_ATTR_RO(uuid);
  2363. static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
  2364. char *buf)
  2365. {
  2366. return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
  2367. }
  2368. static DEVICE_ATTR_RO(eui);
  2369. static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
  2370. char *buf)
  2371. {
  2372. return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
  2373. }
  2374. static DEVICE_ATTR_RO(nsid);
  2375. static struct attribute *nvme_ns_id_attrs[] = {
  2376. &dev_attr_wwid.attr,
  2377. &dev_attr_uuid.attr,
  2378. &dev_attr_nguid.attr,
  2379. &dev_attr_eui.attr,
  2380. &dev_attr_nsid.attr,
  2381. #ifdef CONFIG_NVME_MULTIPATH
  2382. &dev_attr_ana_grpid.attr,
  2383. &dev_attr_ana_state.attr,
  2384. #endif
  2385. NULL,
  2386. };
  2387. static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
  2388. struct attribute *a, int n)
  2389. {
  2390. struct device *dev = container_of(kobj, struct device, kobj);
  2391. struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
  2392. if (a == &dev_attr_uuid.attr) {
  2393. if (uuid_is_null(&ids->uuid) &&
  2394. !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
  2395. return 0;
  2396. }
  2397. if (a == &dev_attr_nguid.attr) {
  2398. if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
  2399. return 0;
  2400. }
  2401. if (a == &dev_attr_eui.attr) {
  2402. if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
  2403. return 0;
  2404. }
  2405. #ifdef CONFIG_NVME_MULTIPATH
  2406. if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
  2407. if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
  2408. return 0;
  2409. if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
  2410. return 0;
  2411. }
  2412. #endif
  2413. return a->mode;
  2414. }
  2415. const struct attribute_group nvme_ns_id_attr_group = {
  2416. .attrs = nvme_ns_id_attrs,
  2417. .is_visible = nvme_ns_id_attrs_are_visible,
  2418. };
  2419. const struct attribute_group *nvme_ns_id_attr_groups[] = {
  2420. &nvme_ns_id_attr_group,
  2421. #ifdef CONFIG_NVM
  2422. &nvme_nvm_attr_group,
  2423. #endif
  2424. NULL,
  2425. };
  2426. #define nvme_show_str_function(field) \
  2427. static ssize_t field##_show(struct device *dev, \
  2428. struct device_attribute *attr, char *buf) \
  2429. { \
  2430. struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
  2431. return sprintf(buf, "%.*s\n", \
  2432. (int)sizeof(ctrl->subsys->field), ctrl->subsys->field); \
  2433. } \
  2434. static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
  2435. nvme_show_str_function(model);
  2436. nvme_show_str_function(serial);
  2437. nvme_show_str_function(firmware_rev);
  2438. #define nvme_show_int_function(field) \
  2439. static ssize_t field##_show(struct device *dev, \
  2440. struct device_attribute *attr, char *buf) \
  2441. { \
  2442. struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
  2443. return sprintf(buf, "%d\n", ctrl->field); \
  2444. } \
  2445. static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
  2446. nvme_show_int_function(cntlid);
  2447. static ssize_t nvme_sysfs_delete(struct device *dev,
  2448. struct device_attribute *attr, const char *buf,
  2449. size_t count)
  2450. {
  2451. struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
  2452. /* Can't delete non-created controllers */
  2453. if (!ctrl->created)
  2454. return -EBUSY;
  2455. if (device_remove_file_self(dev, attr))
  2456. nvme_delete_ctrl_sync(ctrl);
  2457. return count;
  2458. }
  2459. static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
  2460. static ssize_t nvme_sysfs_show_transport(struct device *dev,
  2461. struct device_attribute *attr,
  2462. char *buf)
  2463. {
  2464. struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
  2465. return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
  2466. }
  2467. static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
  2468. static ssize_t nvme_sysfs_show_state(struct device *dev,
  2469. struct device_attribute *attr,
  2470. char *buf)
  2471. {
  2472. struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
  2473. static const char *const state_name[] = {
  2474. [NVME_CTRL_NEW] = "new",
  2475. [NVME_CTRL_LIVE] = "live",
  2476. [NVME_CTRL_ADMIN_ONLY] = "only-admin",
  2477. [NVME_CTRL_RESETTING] = "resetting",
  2478. [NVME_CTRL_CONNECTING] = "connecting",
  2479. [NVME_CTRL_DELETING] = "deleting",
  2480. [NVME_CTRL_DEAD] = "dead",
  2481. };
  2482. if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
  2483. state_name[ctrl->state])
  2484. return sprintf(buf, "%s\n", state_name[ctrl->state]);
  2485. return sprintf(buf, "unknown state\n");
  2486. }
  2487. static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
  2488. static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
  2489. struct device_attribute *attr,
  2490. char *buf)
  2491. {
  2492. struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
  2493. return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
  2494. }
  2495. static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
  2496. static ssize_t nvme_sysfs_show_address(struct device *dev,
  2497. struct device_attribute *attr,
  2498. char *buf)
  2499. {
  2500. struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
  2501. return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
  2502. }
  2503. static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
  2504. static struct attribute *nvme_dev_attrs[] = {
  2505. &dev_attr_reset_controller.attr,
  2506. &dev_attr_rescan_controller.attr,
  2507. &dev_attr_model.attr,
  2508. &dev_attr_serial.attr,
  2509. &dev_attr_firmware_rev.attr,
  2510. &dev_attr_cntlid.attr,
  2511. &dev_attr_delete_controller.attr,
  2512. &dev_attr_transport.attr,
  2513. &dev_attr_subsysnqn.attr,
  2514. &dev_attr_address.attr,
  2515. &dev_attr_state.attr,
  2516. NULL
  2517. };
  2518. static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
  2519. struct attribute *a, int n)
  2520. {
  2521. struct device *dev = container_of(kobj, struct device, kobj);
  2522. struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
  2523. if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
  2524. return 0;
  2525. if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
  2526. return 0;
  2527. return a->mode;
  2528. }
  2529. static struct attribute_group nvme_dev_attrs_group = {
  2530. .attrs = nvme_dev_attrs,
  2531. .is_visible = nvme_dev_attrs_are_visible,
  2532. };
  2533. static const struct attribute_group *nvme_dev_attr_groups[] = {
  2534. &nvme_dev_attrs_group,
  2535. NULL,
  2536. };
  2537. static struct nvme_ns_head *__nvme_find_ns_head(struct nvme_subsystem *subsys,
  2538. unsigned nsid)
  2539. {
  2540. struct nvme_ns_head *h;
  2541. lockdep_assert_held(&subsys->lock);
  2542. list_for_each_entry(h, &subsys->nsheads, entry) {
  2543. if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
  2544. return h;
  2545. }
  2546. return NULL;
  2547. }
  2548. static int __nvme_check_ids(struct nvme_subsystem *subsys,
  2549. struct nvme_ns_head *new)
  2550. {
  2551. struct nvme_ns_head *h;
  2552. lockdep_assert_held(&subsys->lock);
  2553. list_for_each_entry(h, &subsys->nsheads, entry) {
  2554. if (nvme_ns_ids_valid(&new->ids) &&
  2555. !list_empty(&h->list) &&
  2556. nvme_ns_ids_equal(&new->ids, &h->ids))
  2557. return -EINVAL;
  2558. }
  2559. return 0;
  2560. }
  2561. static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
  2562. unsigned nsid, struct nvme_id_ns *id)
  2563. {
  2564. struct nvme_ns_head *head;
  2565. int ret = -ENOMEM;
  2566. head = kzalloc(sizeof(*head), GFP_KERNEL);
  2567. if (!head)
  2568. goto out;
  2569. ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
  2570. if (ret < 0)
  2571. goto out_free_head;
  2572. head->instance = ret;
  2573. INIT_LIST_HEAD(&head->list);
  2574. ret = init_srcu_struct(&head->srcu);
  2575. if (ret)
  2576. goto out_ida_remove;
  2577. head->subsys = ctrl->subsys;
  2578. head->ns_id = nsid;
  2579. kref_init(&head->ref);
  2580. nvme_report_ns_ids(ctrl, nsid, id, &head->ids);
  2581. ret = __nvme_check_ids(ctrl->subsys, head);
  2582. if (ret) {
  2583. dev_err(ctrl->device,
  2584. "duplicate IDs for nsid %d\n", nsid);
  2585. goto out_cleanup_srcu;
  2586. }
  2587. ret = nvme_mpath_alloc_disk(ctrl, head);
  2588. if (ret)
  2589. goto out_cleanup_srcu;
  2590. list_add_tail(&head->entry, &ctrl->subsys->nsheads);
  2591. kref_get(&ctrl->subsys->ref);
  2592. return head;
  2593. out_cleanup_srcu:
  2594. cleanup_srcu_struct(&head->srcu);
  2595. out_ida_remove:
  2596. ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
  2597. out_free_head:
  2598. kfree(head);
  2599. out:
  2600. return ERR_PTR(ret);
  2601. }
  2602. static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
  2603. struct nvme_id_ns *id)
  2604. {
  2605. struct nvme_ctrl *ctrl = ns->ctrl;
  2606. bool is_shared = id->nmic & (1 << 0);
  2607. struct nvme_ns_head *head = NULL;
  2608. int ret = 0;
  2609. mutex_lock(&ctrl->subsys->lock);
  2610. if (is_shared)
  2611. head = __nvme_find_ns_head(ctrl->subsys, nsid);
  2612. if (!head) {
  2613. head = nvme_alloc_ns_head(ctrl, nsid, id);
  2614. if (IS_ERR(head)) {
  2615. ret = PTR_ERR(head);
  2616. goto out_unlock;
  2617. }
  2618. } else {
  2619. struct nvme_ns_ids ids;
  2620. nvme_report_ns_ids(ctrl, nsid, id, &ids);
  2621. if (!nvme_ns_ids_equal(&head->ids, &ids)) {
  2622. dev_err(ctrl->device,
  2623. "IDs don't match for shared namespace %d\n",
  2624. nsid);
  2625. ret = -EINVAL;
  2626. goto out_unlock;
  2627. }
  2628. }
  2629. list_add_tail(&ns->siblings, &head->list);
  2630. ns->head = head;
  2631. out_unlock:
  2632. mutex_unlock(&ctrl->subsys->lock);
  2633. return ret;
  2634. }
  2635. static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
  2636. {
  2637. struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
  2638. struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
  2639. return nsa->head->ns_id - nsb->head->ns_id;
  2640. }
  2641. static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
  2642. {
  2643. struct nvme_ns *ns, *ret = NULL;
  2644. down_read(&ctrl->namespaces_rwsem);
  2645. list_for_each_entry(ns, &ctrl->namespaces, list) {
  2646. if (ns->head->ns_id == nsid) {
  2647. if (!kref_get_unless_zero(&ns->kref))
  2648. continue;
  2649. ret = ns;
  2650. break;
  2651. }
  2652. if (ns->head->ns_id > nsid)
  2653. break;
  2654. }
  2655. up_read(&ctrl->namespaces_rwsem);
  2656. return ret;
  2657. }
  2658. static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
  2659. {
  2660. struct streams_directive_params s;
  2661. int ret;
  2662. if (!ctrl->nr_streams)
  2663. return 0;
  2664. ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
  2665. if (ret)
  2666. return ret;
  2667. ns->sws = le32_to_cpu(s.sws);
  2668. ns->sgs = le16_to_cpu(s.sgs);
  2669. if (ns->sws) {
  2670. unsigned int bs = 1 << ns->lba_shift;
  2671. blk_queue_io_min(ns->queue, bs * ns->sws);
  2672. if (ns->sgs)
  2673. blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs);
  2674. }
  2675. return 0;
  2676. }
  2677. static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
  2678. {
  2679. struct nvme_ns *ns;
  2680. struct gendisk *disk;
  2681. struct nvme_id_ns *id;
  2682. char disk_name[DISK_NAME_LEN];
  2683. int node = dev_to_node(ctrl->dev), flags = GENHD_FL_EXT_DEVT;
  2684. ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
  2685. if (!ns)
  2686. return;
  2687. ns->queue = blk_mq_init_queue(ctrl->tagset);
  2688. if (IS_ERR(ns->queue))
  2689. goto out_free_ns;
  2690. blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
  2691. ns->queue->queuedata = ns;
  2692. ns->ctrl = ctrl;
  2693. kref_init(&ns->kref);
  2694. ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
  2695. blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
  2696. nvme_set_queue_limits(ctrl, ns->queue);
  2697. id = nvme_identify_ns(ctrl, nsid);
  2698. if (!id)
  2699. goto out_free_queue;
  2700. if (id->ncap == 0)
  2701. goto out_free_id;
  2702. if (nvme_init_ns_head(ns, nsid, id))
  2703. goto out_free_id;
  2704. nvme_setup_streams_ns(ctrl, ns);
  2705. nvme_set_disk_name(disk_name, ns, ctrl, &flags);
  2706. if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
  2707. if (nvme_nvm_register(ns, disk_name, node)) {
  2708. dev_warn(ctrl->device, "LightNVM init failure\n");
  2709. goto out_unlink_ns;
  2710. }
  2711. }
  2712. disk = alloc_disk_node(0, node);
  2713. if (!disk)
  2714. goto out_unlink_ns;
  2715. disk->fops = &nvme_fops;
  2716. disk->private_data = ns;
  2717. disk->queue = ns->queue;
  2718. disk->flags = flags;
  2719. memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
  2720. ns->disk = disk;
  2721. __nvme_revalidate_disk(disk, id);
  2722. down_write(&ctrl->namespaces_rwsem);
  2723. list_add_tail(&ns->list, &ctrl->namespaces);
  2724. up_write(&ctrl->namespaces_rwsem);
  2725. nvme_get_ctrl(ctrl);
  2726. device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
  2727. nvme_mpath_add_disk(ns, id);
  2728. nvme_fault_inject_init(ns);
  2729. kfree(id);
  2730. return;
  2731. out_unlink_ns:
  2732. mutex_lock(&ctrl->subsys->lock);
  2733. list_del_rcu(&ns->siblings);
  2734. mutex_unlock(&ctrl->subsys->lock);
  2735. out_free_id:
  2736. kfree(id);
  2737. out_free_queue:
  2738. blk_cleanup_queue(ns->queue);
  2739. out_free_ns:
  2740. kfree(ns);
  2741. }
  2742. static void nvme_ns_remove(struct nvme_ns *ns)
  2743. {
  2744. if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
  2745. return;
  2746. nvme_fault_inject_fini(ns);
  2747. mutex_lock(&ns->ctrl->subsys->lock);
  2748. list_del_rcu(&ns->siblings);
  2749. mutex_unlock(&ns->ctrl->subsys->lock);
  2750. synchronize_rcu(); /* guarantee not available in head->list */
  2751. nvme_mpath_clear_current_path(ns);
  2752. synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
  2753. if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
  2754. del_gendisk(ns->disk);
  2755. blk_cleanup_queue(ns->queue);
  2756. if (blk_get_integrity(ns->disk))
  2757. blk_integrity_unregister(ns->disk);
  2758. }
  2759. down_write(&ns->ctrl->namespaces_rwsem);
  2760. list_del_init(&ns->list);
  2761. up_write(&ns->ctrl->namespaces_rwsem);
  2762. nvme_mpath_check_last_path(ns);
  2763. nvme_put_ns(ns);
  2764. }
  2765. static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
  2766. {
  2767. struct nvme_ns *ns;
  2768. ns = nvme_find_get_ns(ctrl, nsid);
  2769. if (ns) {
  2770. if (ns->disk && revalidate_disk(ns->disk))
  2771. nvme_ns_remove(ns);
  2772. nvme_put_ns(ns);
  2773. } else
  2774. nvme_alloc_ns(ctrl, nsid);
  2775. }
  2776. static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
  2777. unsigned nsid)
  2778. {
  2779. struct nvme_ns *ns, *next;
  2780. LIST_HEAD(rm_list);
  2781. down_write(&ctrl->namespaces_rwsem);
  2782. list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
  2783. if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
  2784. list_move_tail(&ns->list, &rm_list);
  2785. }
  2786. up_write(&ctrl->namespaces_rwsem);
  2787. list_for_each_entry_safe(ns, next, &rm_list, list)
  2788. nvme_ns_remove(ns);
  2789. }
  2790. static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
  2791. {
  2792. struct nvme_ns *ns;
  2793. __le32 *ns_list;
  2794. unsigned i, j, nsid, prev = 0;
  2795. unsigned num_lists = DIV_ROUND_UP_ULL((u64)nn, 1024);
  2796. int ret = 0;
  2797. ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
  2798. if (!ns_list)
  2799. return -ENOMEM;
  2800. for (i = 0; i < num_lists; i++) {
  2801. ret = nvme_identify_ns_list(ctrl, prev, ns_list);
  2802. if (ret)
  2803. goto free;
  2804. for (j = 0; j < min(nn, 1024U); j++) {
  2805. nsid = le32_to_cpu(ns_list[j]);
  2806. if (!nsid)
  2807. goto out;
  2808. nvme_validate_ns(ctrl, nsid);
  2809. while (++prev < nsid) {
  2810. ns = nvme_find_get_ns(ctrl, prev);
  2811. if (ns) {
  2812. nvme_ns_remove(ns);
  2813. nvme_put_ns(ns);
  2814. }
  2815. }
  2816. }
  2817. nn -= j;
  2818. }
  2819. out:
  2820. nvme_remove_invalid_namespaces(ctrl, prev);
  2821. free:
  2822. kfree(ns_list);
  2823. return ret;
  2824. }
  2825. static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
  2826. {
  2827. unsigned i;
  2828. for (i = 1; i <= nn; i++)
  2829. nvme_validate_ns(ctrl, i);
  2830. nvme_remove_invalid_namespaces(ctrl, nn);
  2831. }
  2832. static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
  2833. {
  2834. size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
  2835. __le32 *log;
  2836. int error;
  2837. log = kzalloc(log_size, GFP_KERNEL);
  2838. if (!log)
  2839. return;
  2840. /*
  2841. * We need to read the log to clear the AEN, but we don't want to rely
  2842. * on it for the changed namespace information as userspace could have
  2843. * raced with us in reading the log page, which could cause us to miss
  2844. * updates.
  2845. */
  2846. error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0, log,
  2847. log_size, 0);
  2848. if (error)
  2849. dev_warn(ctrl->device,
  2850. "reading changed ns log failed: %d\n", error);
  2851. kfree(log);
  2852. }
  2853. static void nvme_scan_work(struct work_struct *work)
  2854. {
  2855. struct nvme_ctrl *ctrl =
  2856. container_of(work, struct nvme_ctrl, scan_work);
  2857. struct nvme_id_ctrl *id;
  2858. unsigned nn;
  2859. if (ctrl->state != NVME_CTRL_LIVE)
  2860. return;
  2861. WARN_ON_ONCE(!ctrl->tagset);
  2862. if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
  2863. dev_info(ctrl->device, "rescanning namespaces.\n");
  2864. nvme_clear_changed_ns_log(ctrl);
  2865. }
  2866. if (nvme_identify_ctrl(ctrl, &id))
  2867. return;
  2868. mutex_lock(&ctrl->scan_lock);
  2869. nn = le32_to_cpu(id->nn);
  2870. if (!nvme_ctrl_limited_cns(ctrl)) {
  2871. if (!nvme_scan_ns_list(ctrl, nn))
  2872. goto out_free_id;
  2873. }
  2874. nvme_scan_ns_sequential(ctrl, nn);
  2875. out_free_id:
  2876. mutex_unlock(&ctrl->scan_lock);
  2877. kfree(id);
  2878. down_write(&ctrl->namespaces_rwsem);
  2879. list_sort(NULL, &ctrl->namespaces, ns_cmp);
  2880. up_write(&ctrl->namespaces_rwsem);
  2881. }
  2882. /*
  2883. * This function iterates the namespace list unlocked to allow recovery from
  2884. * controller failure. It is up to the caller to ensure the namespace list is
  2885. * not modified by scan work while this function is executing.
  2886. */
  2887. void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
  2888. {
  2889. struct nvme_ns *ns, *next;
  2890. LIST_HEAD(ns_list);
  2891. /* prevent racing with ns scanning */
  2892. flush_work(&ctrl->scan_work);
  2893. /*
  2894. * The dead states indicates the controller was not gracefully
  2895. * disconnected. In that case, we won't be able to flush any data while
  2896. * removing the namespaces' disks; fail all the queues now to avoid
  2897. * potentially having to clean up the failed sync later.
  2898. */
  2899. if (ctrl->state == NVME_CTRL_DEAD)
  2900. nvme_kill_queues(ctrl);
  2901. down_write(&ctrl->namespaces_rwsem);
  2902. list_splice_init(&ctrl->namespaces, &ns_list);
  2903. up_write(&ctrl->namespaces_rwsem);
  2904. list_for_each_entry_safe(ns, next, &ns_list, list)
  2905. nvme_ns_remove(ns);
  2906. }
  2907. EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
  2908. static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
  2909. {
  2910. char *envp[2] = { NULL, NULL };
  2911. u32 aen_result = ctrl->aen_result;
  2912. ctrl->aen_result = 0;
  2913. if (!aen_result)
  2914. return;
  2915. envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
  2916. if (!envp[0])
  2917. return;
  2918. kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
  2919. kfree(envp[0]);
  2920. }
  2921. static void nvme_async_event_work(struct work_struct *work)
  2922. {
  2923. struct nvme_ctrl *ctrl =
  2924. container_of(work, struct nvme_ctrl, async_event_work);
  2925. nvme_aen_uevent(ctrl);
  2926. ctrl->ops->submit_async_event(ctrl);
  2927. }
  2928. static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
  2929. {
  2930. u32 csts;
  2931. if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
  2932. return false;
  2933. if (csts == ~0)
  2934. return false;
  2935. return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
  2936. }
  2937. static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
  2938. {
  2939. struct nvme_fw_slot_info_log *log;
  2940. log = kmalloc(sizeof(*log), GFP_KERNEL);
  2941. if (!log)
  2942. return;
  2943. if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, log,
  2944. sizeof(*log), 0))
  2945. dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
  2946. kfree(log);
  2947. }
  2948. static void nvme_fw_act_work(struct work_struct *work)
  2949. {
  2950. struct nvme_ctrl *ctrl = container_of(work,
  2951. struct nvme_ctrl, fw_act_work);
  2952. unsigned long fw_act_timeout;
  2953. if (ctrl->mtfa)
  2954. fw_act_timeout = jiffies +
  2955. msecs_to_jiffies(ctrl->mtfa * 100);
  2956. else
  2957. fw_act_timeout = jiffies +
  2958. msecs_to_jiffies(admin_timeout * 1000);
  2959. nvme_stop_queues(ctrl);
  2960. while (nvme_ctrl_pp_status(ctrl)) {
  2961. if (time_after(jiffies, fw_act_timeout)) {
  2962. dev_warn(ctrl->device,
  2963. "Fw activation timeout, reset controller\n");
  2964. nvme_reset_ctrl(ctrl);
  2965. break;
  2966. }
  2967. msleep(100);
  2968. }
  2969. if (ctrl->state != NVME_CTRL_LIVE)
  2970. return;
  2971. nvme_start_queues(ctrl);
  2972. /* read FW slot information to clear the AER */
  2973. nvme_get_fw_slot_info(ctrl);
  2974. }
  2975. static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
  2976. {
  2977. switch ((result & 0xff00) >> 8) {
  2978. case NVME_AER_NOTICE_NS_CHANGED:
  2979. set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
  2980. nvme_queue_scan(ctrl);
  2981. break;
  2982. case NVME_AER_NOTICE_FW_ACT_STARTING:
  2983. queue_work(nvme_wq, &ctrl->fw_act_work);
  2984. break;
  2985. #ifdef CONFIG_NVME_MULTIPATH
  2986. case NVME_AER_NOTICE_ANA:
  2987. if (!ctrl->ana_log_buf)
  2988. break;
  2989. queue_work(nvme_wq, &ctrl->ana_work);
  2990. break;
  2991. #endif
  2992. default:
  2993. dev_warn(ctrl->device, "async event result %08x\n", result);
  2994. }
  2995. }
  2996. void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
  2997. volatile union nvme_result *res)
  2998. {
  2999. u32 result = le32_to_cpu(res->u32);
  3000. if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
  3001. return;
  3002. switch (result & 0x7) {
  3003. case NVME_AER_NOTICE:
  3004. nvme_handle_aen_notice(ctrl, result);
  3005. break;
  3006. case NVME_AER_ERROR:
  3007. case NVME_AER_SMART:
  3008. case NVME_AER_CSS:
  3009. case NVME_AER_VS:
  3010. ctrl->aen_result = result;
  3011. break;
  3012. default:
  3013. break;
  3014. }
  3015. queue_work(nvme_wq, &ctrl->async_event_work);
  3016. }
  3017. EXPORT_SYMBOL_GPL(nvme_complete_async_event);
  3018. void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
  3019. {
  3020. nvme_mpath_stop(ctrl);
  3021. nvme_stop_keep_alive(ctrl);
  3022. flush_work(&ctrl->async_event_work);
  3023. cancel_work_sync(&ctrl->fw_act_work);
  3024. if (ctrl->ops->stop_ctrl)
  3025. ctrl->ops->stop_ctrl(ctrl);
  3026. }
  3027. EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
  3028. void nvme_start_ctrl(struct nvme_ctrl *ctrl)
  3029. {
  3030. if (ctrl->kato)
  3031. nvme_start_keep_alive(ctrl);
  3032. if (ctrl->queue_count > 1) {
  3033. nvme_queue_scan(ctrl);
  3034. nvme_enable_aen(ctrl);
  3035. queue_work(nvme_wq, &ctrl->async_event_work);
  3036. nvme_start_queues(ctrl);
  3037. }
  3038. ctrl->created = true;
  3039. }
  3040. EXPORT_SYMBOL_GPL(nvme_start_ctrl);
  3041. void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
  3042. {
  3043. dev_pm_qos_hide_latency_tolerance(ctrl->device);
  3044. cdev_device_del(&ctrl->cdev, ctrl->device);
  3045. }
  3046. EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
  3047. static void nvme_free_ctrl(struct device *dev)
  3048. {
  3049. struct nvme_ctrl *ctrl =
  3050. container_of(dev, struct nvme_ctrl, ctrl_device);
  3051. struct nvme_subsystem *subsys = ctrl->subsys;
  3052. ida_simple_remove(&nvme_instance_ida, ctrl->instance);
  3053. kfree(ctrl->effects);
  3054. nvme_mpath_uninit(ctrl);
  3055. __free_page(ctrl->discard_page);
  3056. if (subsys) {
  3057. mutex_lock(&subsys->lock);
  3058. list_del(&ctrl->subsys_entry);
  3059. mutex_unlock(&subsys->lock);
  3060. sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
  3061. }
  3062. ctrl->ops->free_ctrl(ctrl);
  3063. if (subsys)
  3064. nvme_put_subsystem(subsys);
  3065. }
  3066. /*
  3067. * Initialize a NVMe controller structures. This needs to be called during
  3068. * earliest initialization so that we have the initialized structured around
  3069. * during probing.
  3070. */
  3071. int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
  3072. const struct nvme_ctrl_ops *ops, unsigned long quirks)
  3073. {
  3074. int ret;
  3075. ctrl->state = NVME_CTRL_NEW;
  3076. spin_lock_init(&ctrl->lock);
  3077. mutex_init(&ctrl->scan_lock);
  3078. INIT_LIST_HEAD(&ctrl->namespaces);
  3079. init_rwsem(&ctrl->namespaces_rwsem);
  3080. ctrl->dev = dev;
  3081. ctrl->ops = ops;
  3082. ctrl->quirks = quirks;
  3083. INIT_WORK(&ctrl->scan_work, nvme_scan_work);
  3084. INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
  3085. INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
  3086. INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
  3087. INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
  3088. memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
  3089. ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
  3090. BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
  3091. PAGE_SIZE);
  3092. ctrl->discard_page = alloc_page(GFP_KERNEL);
  3093. if (!ctrl->discard_page) {
  3094. ret = -ENOMEM;
  3095. goto out;
  3096. }
  3097. ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
  3098. if (ret < 0)
  3099. goto out;
  3100. ctrl->instance = ret;
  3101. device_initialize(&ctrl->ctrl_device);
  3102. ctrl->device = &ctrl->ctrl_device;
  3103. ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
  3104. ctrl->device->class = nvme_class;
  3105. ctrl->device->parent = ctrl->dev;
  3106. ctrl->device->groups = nvme_dev_attr_groups;
  3107. ctrl->device->release = nvme_free_ctrl;
  3108. dev_set_drvdata(ctrl->device, ctrl);
  3109. ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
  3110. if (ret)
  3111. goto out_release_instance;
  3112. cdev_init(&ctrl->cdev, &nvme_dev_fops);
  3113. ctrl->cdev.owner = ops->module;
  3114. ret = cdev_device_add(&ctrl->cdev, ctrl->device);
  3115. if (ret)
  3116. goto out_free_name;
  3117. /*
  3118. * Initialize latency tolerance controls. The sysfs files won't
  3119. * be visible to userspace unless the device actually supports APST.
  3120. */
  3121. ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
  3122. dev_pm_qos_update_user_latency_tolerance(ctrl->device,
  3123. min(default_ps_max_latency_us, (unsigned long)S32_MAX));
  3124. return 0;
  3125. out_free_name:
  3126. kfree_const(ctrl->device->kobj.name);
  3127. out_release_instance:
  3128. ida_simple_remove(&nvme_instance_ida, ctrl->instance);
  3129. out:
  3130. if (ctrl->discard_page)
  3131. __free_page(ctrl->discard_page);
  3132. return ret;
  3133. }
  3134. EXPORT_SYMBOL_GPL(nvme_init_ctrl);
  3135. /**
  3136. * nvme_kill_queues(): Ends all namespace queues
  3137. * @ctrl: the dead controller that needs to end
  3138. *
  3139. * Call this function when the driver determines it is unable to get the
  3140. * controller in a state capable of servicing IO.
  3141. */
  3142. void nvme_kill_queues(struct nvme_ctrl *ctrl)
  3143. {
  3144. struct nvme_ns *ns;
  3145. down_read(&ctrl->namespaces_rwsem);
  3146. /* Forcibly unquiesce queues to avoid blocking dispatch */
  3147. if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
  3148. blk_mq_unquiesce_queue(ctrl->admin_q);
  3149. list_for_each_entry(ns, &ctrl->namespaces, list)
  3150. nvme_set_queue_dying(ns);
  3151. up_read(&ctrl->namespaces_rwsem);
  3152. }
  3153. EXPORT_SYMBOL_GPL(nvme_kill_queues);
  3154. void nvme_unfreeze(struct nvme_ctrl *ctrl)
  3155. {
  3156. struct nvme_ns *ns;
  3157. down_read(&ctrl->namespaces_rwsem);
  3158. list_for_each_entry(ns, &ctrl->namespaces, list)
  3159. blk_mq_unfreeze_queue(ns->queue);
  3160. up_read(&ctrl->namespaces_rwsem);
  3161. }
  3162. EXPORT_SYMBOL_GPL(nvme_unfreeze);
  3163. void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
  3164. {
  3165. struct nvme_ns *ns;
  3166. down_read(&ctrl->namespaces_rwsem);
  3167. list_for_each_entry(ns, &ctrl->namespaces, list) {
  3168. timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
  3169. if (timeout <= 0)
  3170. break;
  3171. }
  3172. up_read(&ctrl->namespaces_rwsem);
  3173. }
  3174. EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
  3175. void nvme_wait_freeze(struct nvme_ctrl *ctrl)
  3176. {
  3177. struct nvme_ns *ns;
  3178. down_read(&ctrl->namespaces_rwsem);
  3179. list_for_each_entry(ns, &ctrl->namespaces, list)
  3180. blk_mq_freeze_queue_wait(ns->queue);
  3181. up_read(&ctrl->namespaces_rwsem);
  3182. }
  3183. EXPORT_SYMBOL_GPL(nvme_wait_freeze);
  3184. void nvme_start_freeze(struct nvme_ctrl *ctrl)
  3185. {
  3186. struct nvme_ns *ns;
  3187. down_read(&ctrl->namespaces_rwsem);
  3188. list_for_each_entry(ns, &ctrl->namespaces, list)
  3189. blk_freeze_queue_start(ns->queue);
  3190. up_read(&ctrl->namespaces_rwsem);
  3191. }
  3192. EXPORT_SYMBOL_GPL(nvme_start_freeze);
  3193. void nvme_stop_queues(struct nvme_ctrl *ctrl)
  3194. {
  3195. struct nvme_ns *ns;
  3196. down_read(&ctrl->namespaces_rwsem);
  3197. list_for_each_entry(ns, &ctrl->namespaces, list)
  3198. blk_mq_quiesce_queue(ns->queue);
  3199. up_read(&ctrl->namespaces_rwsem);
  3200. }
  3201. EXPORT_SYMBOL_GPL(nvme_stop_queues);
  3202. void nvme_start_queues(struct nvme_ctrl *ctrl)
  3203. {
  3204. struct nvme_ns *ns;
  3205. down_read(&ctrl->namespaces_rwsem);
  3206. list_for_each_entry(ns, &ctrl->namespaces, list)
  3207. blk_mq_unquiesce_queue(ns->queue);
  3208. up_read(&ctrl->namespaces_rwsem);
  3209. }
  3210. EXPORT_SYMBOL_GPL(nvme_start_queues);
  3211. int __init nvme_core_init(void)
  3212. {
  3213. int result = -ENOMEM;
  3214. nvme_wq = alloc_workqueue("nvme-wq",
  3215. WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
  3216. if (!nvme_wq)
  3217. goto out;
  3218. nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
  3219. WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
  3220. if (!nvme_reset_wq)
  3221. goto destroy_wq;
  3222. nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
  3223. WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
  3224. if (!nvme_delete_wq)
  3225. goto destroy_reset_wq;
  3226. result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
  3227. if (result < 0)
  3228. goto destroy_delete_wq;
  3229. nvme_class = class_create(THIS_MODULE, "nvme");
  3230. if (IS_ERR(nvme_class)) {
  3231. result = PTR_ERR(nvme_class);
  3232. goto unregister_chrdev;
  3233. }
  3234. nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
  3235. if (IS_ERR(nvme_subsys_class)) {
  3236. result = PTR_ERR(nvme_subsys_class);
  3237. goto destroy_class;
  3238. }
  3239. return 0;
  3240. destroy_class:
  3241. class_destroy(nvme_class);
  3242. unregister_chrdev:
  3243. unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
  3244. destroy_delete_wq:
  3245. destroy_workqueue(nvme_delete_wq);
  3246. destroy_reset_wq:
  3247. destroy_workqueue(nvme_reset_wq);
  3248. destroy_wq:
  3249. destroy_workqueue(nvme_wq);
  3250. out:
  3251. return result;
  3252. }
  3253. void nvme_core_exit(void)
  3254. {
  3255. ida_destroy(&nvme_subsystems_ida);
  3256. class_destroy(nvme_subsys_class);
  3257. class_destroy(nvme_class);
  3258. unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
  3259. destroy_workqueue(nvme_delete_wq);
  3260. destroy_workqueue(nvme_reset_wq);
  3261. destroy_workqueue(nvme_wq);
  3262. }
  3263. MODULE_LICENSE("GPL");
  3264. MODULE_VERSION("1.0");
  3265. module_init(nvme_core_init);
  3266. module_exit(nvme_core_exit);