xhci-ring.c 124 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * xHCI host controller driver
  4. *
  5. * Copyright (C) 2008 Intel Corp.
  6. *
  7. * Author: Sarah Sharp
  8. * Some code borrowed from the Linux EHCI driver.
  9. */
  10. /*
  11. * Ring initialization rules:
  12. * 1. Each segment is initialized to zero, except for link TRBs.
  13. * 2. Ring cycle state = 0. This represents Producer Cycle State (PCS) or
  14. * Consumer Cycle State (CCS), depending on ring function.
  15. * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment.
  16. *
  17. * Ring behavior rules:
  18. * 1. A ring is empty if enqueue == dequeue. This means there will always be at
  19. * least one free TRB in the ring. This is useful if you want to turn that
  20. * into a link TRB and expand the ring.
  21. * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a
  22. * link TRB, then load the pointer with the address in the link TRB. If the
  23. * link TRB had its toggle bit set, you may need to update the ring cycle
  24. * state (see cycle bit rules). You may have to do this multiple times
  25. * until you reach a non-link TRB.
  26. * 3. A ring is full if enqueue++ (for the definition of increment above)
  27. * equals the dequeue pointer.
  28. *
  29. * Cycle bit rules:
  30. * 1. When a consumer increments a dequeue pointer and encounters a toggle bit
  31. * in a link TRB, it must toggle the ring cycle state.
  32. * 2. When a producer increments an enqueue pointer and encounters a toggle bit
  33. * in a link TRB, it must toggle the ring cycle state.
  34. *
  35. * Producer rules:
  36. * 1. Check if ring is full before you enqueue.
  37. * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing.
  38. * Update enqueue pointer between each write (which may update the ring
  39. * cycle state).
  40. * 3. Notify consumer. If SW is producer, it rings the doorbell for command
  41. * and endpoint rings. If HC is the producer for the event ring,
  42. * and it generates an interrupt according to interrupt modulation rules.
  43. *
  44. * Consumer rules:
  45. * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state,
  46. * the TRB is owned by the consumer.
  47. * 2. Update dequeue pointer (which may update the ring cycle state) and
  48. * continue processing TRBs until you reach a TRB which is not owned by you.
  49. * 3. Notify the producer. SW is the consumer for the event ring, and it
  50. * updates event ring dequeue pointer. HC is the consumer for the command and
  51. * endpoint rings; it generates events on the event ring for these.
  52. */
  53. #include <linux/scatterlist.h>
  54. #include <linux/slab.h>
  55. #include <linux/dma-mapping.h>
  56. #include "xhci.h"
  57. #include "xhci-trace.h"
  58. #include "xhci-mtk.h"
  59. /*
  60. * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
  61. * address of the TRB.
  62. */
  63. dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg,
  64. union xhci_trb *trb)
  65. {
  66. unsigned long segment_offset;
  67. if (!seg || !trb || trb < seg->trbs)
  68. return 0;
  69. /* offset in TRBs */
  70. segment_offset = trb - seg->trbs;
  71. if (segment_offset >= TRBS_PER_SEGMENT)
  72. return 0;
  73. return seg->dma + (segment_offset * sizeof(*trb));
  74. }
  75. static bool trb_is_noop(union xhci_trb *trb)
  76. {
  77. return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
  78. }
  79. static bool trb_is_link(union xhci_trb *trb)
  80. {
  81. return TRB_TYPE_LINK_LE32(trb->link.control);
  82. }
  83. static bool last_trb_on_seg(struct xhci_segment *seg, union xhci_trb *trb)
  84. {
  85. return trb == &seg->trbs[TRBS_PER_SEGMENT - 1];
  86. }
  87. static bool last_trb_on_ring(struct xhci_ring *ring,
  88. struct xhci_segment *seg, union xhci_trb *trb)
  89. {
  90. return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg);
  91. }
  92. static bool link_trb_toggles_cycle(union xhci_trb *trb)
  93. {
  94. return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
  95. }
  96. static bool last_td_in_urb(struct xhci_td *td)
  97. {
  98. struct urb_priv *urb_priv = td->urb->hcpriv;
  99. return urb_priv->num_tds_done == urb_priv->num_tds;
  100. }
  101. static void inc_td_cnt(struct urb *urb)
  102. {
  103. struct urb_priv *urb_priv = urb->hcpriv;
  104. urb_priv->num_tds_done++;
  105. }
  106. static void trb_to_noop(union xhci_trb *trb, u32 noop_type)
  107. {
  108. if (trb_is_link(trb)) {
  109. /* unchain chained link TRBs */
  110. trb->link.control &= cpu_to_le32(~TRB_CHAIN);
  111. } else {
  112. trb->generic.field[0] = 0;
  113. trb->generic.field[1] = 0;
  114. trb->generic.field[2] = 0;
  115. /* Preserve only the cycle bit of this TRB */
  116. trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
  117. trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type));
  118. }
  119. }
  120. /* Updates trb to point to the next TRB in the ring, and updates seg if the next
  121. * TRB is in a new segment. This does not skip over link TRBs, and it does not
  122. * effect the ring dequeue or enqueue pointers.
  123. */
  124. static void next_trb(struct xhci_hcd *xhci,
  125. struct xhci_ring *ring,
  126. struct xhci_segment **seg,
  127. union xhci_trb **trb)
  128. {
  129. if (trb_is_link(*trb)) {
  130. *seg = (*seg)->next;
  131. *trb = ((*seg)->trbs);
  132. } else {
  133. (*trb)++;
  134. }
  135. }
  136. /*
  137. * See Cycle bit rules. SW is the consumer for the event ring only.
  138. * Don't make a ring full of link TRBs. That would be dumb and this would loop.
  139. */
  140. void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring)
  141. {
  142. /* event ring doesn't have link trbs, check for last trb */
  143. if (ring->type == TYPE_EVENT) {
  144. if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
  145. ring->dequeue++;
  146. goto out;
  147. }
  148. if (last_trb_on_ring(ring, ring->deq_seg, ring->dequeue))
  149. ring->cycle_state ^= 1;
  150. ring->deq_seg = ring->deq_seg->next;
  151. ring->dequeue = ring->deq_seg->trbs;
  152. goto out;
  153. }
  154. /* All other rings have link trbs */
  155. if (!trb_is_link(ring->dequeue)) {
  156. ring->dequeue++;
  157. ring->num_trbs_free++;
  158. }
  159. while (trb_is_link(ring->dequeue)) {
  160. ring->deq_seg = ring->deq_seg->next;
  161. ring->dequeue = ring->deq_seg->trbs;
  162. }
  163. out:
  164. trace_xhci_inc_deq(ring);
  165. return;
  166. }
  167. /*
  168. * See Cycle bit rules. SW is the consumer for the event ring only.
  169. * Don't make a ring full of link TRBs. That would be dumb and this would loop.
  170. *
  171. * If we've just enqueued a TRB that is in the middle of a TD (meaning the
  172. * chain bit is set), then set the chain bit in all the following link TRBs.
  173. * If we've enqueued the last TRB in a TD, make sure the following link TRBs
  174. * have their chain bit cleared (so that each Link TRB is a separate TD).
  175. *
  176. * Section 6.4.4.1 of the 0.95 spec says link TRBs cannot have the chain bit
  177. * set, but other sections talk about dealing with the chain bit set. This was
  178. * fixed in the 0.96 specification errata, but we have to assume that all 0.95
  179. * xHCI hardware can't handle the chain bit being cleared on a link TRB.
  180. *
  181. * @more_trbs_coming: Will you enqueue more TRBs before calling
  182. * prepare_transfer()?
  183. */
  184. static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
  185. bool more_trbs_coming)
  186. {
  187. u32 chain;
  188. union xhci_trb *next;
  189. chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
  190. /* If this is not event ring, there is one less usable TRB */
  191. if (!trb_is_link(ring->enqueue))
  192. ring->num_trbs_free--;
  193. next = ++(ring->enqueue);
  194. /* Update the dequeue pointer further if that was a link TRB */
  195. while (trb_is_link(next)) {
  196. /*
  197. * If the caller doesn't plan on enqueueing more TDs before
  198. * ringing the doorbell, then we don't want to give the link TRB
  199. * to the hardware just yet. We'll give the link TRB back in
  200. * prepare_ring() just before we enqueue the TD at the top of
  201. * the ring.
  202. */
  203. if (!chain && !more_trbs_coming)
  204. break;
  205. /* If we're not dealing with 0.95 hardware or isoc rings on
  206. * AMD 0.96 host, carry over the chain bit of the previous TRB
  207. * (which may mean the chain bit is cleared).
  208. */
  209. if (!(ring->type == TYPE_ISOC &&
  210. (xhci->quirks & XHCI_AMD_0x96_HOST)) &&
  211. !xhci_link_trb_quirk(xhci)) {
  212. next->link.control &= cpu_to_le32(~TRB_CHAIN);
  213. next->link.control |= cpu_to_le32(chain);
  214. }
  215. /* Give this link TRB to the hardware */
  216. wmb();
  217. next->link.control ^= cpu_to_le32(TRB_CYCLE);
  218. /* Toggle the cycle bit after the last ring segment. */
  219. if (link_trb_toggles_cycle(next))
  220. ring->cycle_state ^= 1;
  221. ring->enq_seg = ring->enq_seg->next;
  222. ring->enqueue = ring->enq_seg->trbs;
  223. next = ring->enqueue;
  224. }
  225. trace_xhci_inc_enq(ring);
  226. }
  227. /*
  228. * Check to see if there's room to enqueue num_trbs on the ring and make sure
  229. * enqueue pointer will not advance into dequeue segment. See rules above.
  230. */
  231. static inline int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring,
  232. unsigned int num_trbs)
  233. {
  234. int num_trbs_in_deq_seg;
  235. if (ring->num_trbs_free < num_trbs)
  236. return 0;
  237. if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) {
  238. num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs;
  239. if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg)
  240. return 0;
  241. }
  242. return 1;
  243. }
  244. /* Ring the host controller doorbell after placing a command on the ring */
  245. void xhci_ring_cmd_db(struct xhci_hcd *xhci)
  246. {
  247. if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING))
  248. return;
  249. xhci_dbg(xhci, "// Ding dong!\n");
  250. writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]);
  251. /* Flush PCI posted writes */
  252. readl(&xhci->dba->doorbell[0]);
  253. }
  254. static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay)
  255. {
  256. return mod_delayed_work(system_wq, &xhci->cmd_timer, delay);
  257. }
  258. static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci)
  259. {
  260. return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command,
  261. cmd_list);
  262. }
  263. /*
  264. * Turn all commands on command ring with status set to "aborted" to no-op trbs.
  265. * If there are other commands waiting then restart the ring and kick the timer.
  266. * This must be called with command ring stopped and xhci->lock held.
  267. */
  268. static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci,
  269. struct xhci_command *cur_cmd)
  270. {
  271. struct xhci_command *i_cmd;
  272. /* Turn all aborted commands in list to no-ops, then restart */
  273. list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) {
  274. if (i_cmd->status != COMP_COMMAND_ABORTED)
  275. continue;
  276. i_cmd->status = COMP_COMMAND_RING_STOPPED;
  277. xhci_dbg(xhci, "Turn aborted command %p to no-op\n",
  278. i_cmd->command_trb);
  279. trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP);
  280. /*
  281. * caller waiting for completion is called when command
  282. * completion event is received for these no-op commands
  283. */
  284. }
  285. xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
  286. /* ring command ring doorbell to restart the command ring */
  287. if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) &&
  288. !(xhci->xhc_state & XHCI_STATE_DYING)) {
  289. xhci->current_cmd = cur_cmd;
  290. xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
  291. xhci_ring_cmd_db(xhci);
  292. }
  293. }
  294. /* Must be called with xhci->lock held, releases and aquires lock back */
  295. static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags)
  296. {
  297. u64 temp_64;
  298. int ret;
  299. xhci_dbg(xhci, "Abort command ring\n");
  300. reinit_completion(&xhci->cmd_ring_stop_completion);
  301. temp_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
  302. xhci_write_64(xhci, temp_64 | CMD_RING_ABORT,
  303. &xhci->op_regs->cmd_ring);
  304. /* Section 4.6.1.2 of xHCI 1.0 spec says software should also time the
  305. * completion of the Command Abort operation. If CRR is not negated in 5
  306. * seconds then driver handles it as if host died (-ENODEV).
  307. * In the future we should distinguish between -ENODEV and -ETIMEDOUT
  308. * and try to recover a -ETIMEDOUT with a host controller reset.
  309. */
  310. ret = xhci_handshake(&xhci->op_regs->cmd_ring,
  311. CMD_RING_RUNNING, 0, 5 * 1000 * 1000);
  312. if (ret < 0) {
  313. xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret);
  314. xhci_halt(xhci);
  315. xhci_hc_died(xhci);
  316. return ret;
  317. }
  318. /*
  319. * Writing the CMD_RING_ABORT bit should cause a cmd completion event,
  320. * however on some host hw the CMD_RING_RUNNING bit is correctly cleared
  321. * but the completion event in never sent. Wait 2 secs (arbitrary
  322. * number) to handle those cases after negation of CMD_RING_RUNNING.
  323. */
  324. spin_unlock_irqrestore(&xhci->lock, flags);
  325. ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion,
  326. msecs_to_jiffies(2000));
  327. spin_lock_irqsave(&xhci->lock, flags);
  328. if (!ret) {
  329. xhci_dbg(xhci, "No stop event for abort, ring start fail?\n");
  330. xhci_cleanup_command_queue(xhci);
  331. } else {
  332. xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci));
  333. }
  334. return 0;
  335. }
  336. void xhci_ring_ep_doorbell(struct xhci_hcd *xhci,
  337. unsigned int slot_id,
  338. unsigned int ep_index,
  339. unsigned int stream_id)
  340. {
  341. __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
  342. struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
  343. unsigned int ep_state = ep->ep_state;
  344. /* Don't ring the doorbell for this endpoint if there are pending
  345. * cancellations because we don't want to interrupt processing.
  346. * We don't want to restart any stream rings if there's a set dequeue
  347. * pointer command pending because the device can choose to start any
  348. * stream once the endpoint is on the HW schedule.
  349. */
  350. if ((ep_state & EP_STOP_CMD_PENDING) || (ep_state & SET_DEQ_PENDING) ||
  351. (ep_state & EP_HALTED))
  352. return;
  353. writel(DB_VALUE(ep_index, stream_id), db_addr);
  354. /* The CPU has better things to do at this point than wait for a
  355. * write-posting flush. It'll get there soon enough.
  356. */
  357. }
  358. /* Ring the doorbell for any rings with pending URBs */
  359. static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
  360. unsigned int slot_id,
  361. unsigned int ep_index)
  362. {
  363. unsigned int stream_id;
  364. struct xhci_virt_ep *ep;
  365. ep = &xhci->devs[slot_id]->eps[ep_index];
  366. /* A ring has pending URBs if its TD list is not empty */
  367. if (!(ep->ep_state & EP_HAS_STREAMS)) {
  368. if (ep->ring && !(list_empty(&ep->ring->td_list)))
  369. xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0);
  370. return;
  371. }
  372. for (stream_id = 1; stream_id < ep->stream_info->num_streams;
  373. stream_id++) {
  374. struct xhci_stream_info *stream_info = ep->stream_info;
  375. if (!list_empty(&stream_info->stream_rings[stream_id]->td_list))
  376. xhci_ring_ep_doorbell(xhci, slot_id, ep_index,
  377. stream_id);
  378. }
  379. }
  380. /* Get the right ring for the given slot_id, ep_index and stream_id.
  381. * If the endpoint supports streams, boundary check the URB's stream ID.
  382. * If the endpoint doesn't support streams, return the singular endpoint ring.
  383. */
  384. struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci,
  385. unsigned int slot_id, unsigned int ep_index,
  386. unsigned int stream_id)
  387. {
  388. struct xhci_virt_ep *ep;
  389. ep = &xhci->devs[slot_id]->eps[ep_index];
  390. /* Common case: no streams */
  391. if (!(ep->ep_state & EP_HAS_STREAMS))
  392. return ep->ring;
  393. if (stream_id == 0) {
  394. xhci_warn(xhci,
  395. "WARN: Slot ID %u, ep index %u has streams, "
  396. "but URB has no stream ID.\n",
  397. slot_id, ep_index);
  398. return NULL;
  399. }
  400. if (stream_id < ep->stream_info->num_streams)
  401. return ep->stream_info->stream_rings[stream_id];
  402. xhci_warn(xhci,
  403. "WARN: Slot ID %u, ep index %u has "
  404. "stream IDs 1 to %u allocated, "
  405. "but stream ID %u is requested.\n",
  406. slot_id, ep_index,
  407. ep->stream_info->num_streams - 1,
  408. stream_id);
  409. return NULL;
  410. }
  411. /*
  412. * Get the hw dequeue pointer xHC stopped on, either directly from the
  413. * endpoint context, or if streams are in use from the stream context.
  414. * The returned hw_dequeue contains the lowest four bits with cycle state
  415. * and possbile stream context type.
  416. */
  417. static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev,
  418. unsigned int ep_index, unsigned int stream_id)
  419. {
  420. struct xhci_ep_ctx *ep_ctx;
  421. struct xhci_stream_ctx *st_ctx;
  422. struct xhci_virt_ep *ep;
  423. ep = &vdev->eps[ep_index];
  424. if (ep->ep_state & EP_HAS_STREAMS) {
  425. st_ctx = &ep->stream_info->stream_ctx_array[stream_id];
  426. return le64_to_cpu(st_ctx->stream_ring);
  427. }
  428. ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
  429. return le64_to_cpu(ep_ctx->deq);
  430. }
  431. /*
  432. * Move the xHC's endpoint ring dequeue pointer past cur_td.
  433. * Record the new state of the xHC's endpoint ring dequeue segment,
  434. * dequeue pointer, stream id, and new consumer cycle state in state.
  435. * Update our internal representation of the ring's dequeue pointer.
  436. *
  437. * We do this in three jumps:
  438. * - First we update our new ring state to be the same as when the xHC stopped.
  439. * - Then we traverse the ring to find the segment that contains
  440. * the last TRB in the TD. We toggle the xHC's new cycle state when we pass
  441. * any link TRBs with the toggle cycle bit set.
  442. * - Finally we move the dequeue state one TRB further, toggling the cycle bit
  443. * if we've moved it past a link TRB with the toggle cycle bit set.
  444. *
  445. * Some of the uses of xhci_generic_trb are grotty, but if they're done
  446. * with correct __le32 accesses they should work fine. Only users of this are
  447. * in here.
  448. */
  449. void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
  450. unsigned int slot_id, unsigned int ep_index,
  451. unsigned int stream_id, struct xhci_td *cur_td,
  452. struct xhci_dequeue_state *state)
  453. {
  454. struct xhci_virt_device *dev = xhci->devs[slot_id];
  455. struct xhci_virt_ep *ep = &dev->eps[ep_index];
  456. struct xhci_ring *ep_ring;
  457. struct xhci_segment *new_seg;
  458. union xhci_trb *new_deq;
  459. dma_addr_t addr;
  460. u64 hw_dequeue;
  461. bool cycle_found = false;
  462. bool td_last_trb_found = false;
  463. ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
  464. ep_index, stream_id);
  465. if (!ep_ring) {
  466. xhci_warn(xhci, "WARN can't find new dequeue state "
  467. "for invalid stream ID %u.\n",
  468. stream_id);
  469. return;
  470. }
  471. /* Dig out the cycle state saved by the xHC during the stop ep cmd */
  472. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  473. "Finding endpoint context");
  474. hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id);
  475. new_seg = ep_ring->deq_seg;
  476. new_deq = ep_ring->dequeue;
  477. state->new_cycle_state = hw_dequeue & 0x1;
  478. state->stream_id = stream_id;
  479. /*
  480. * We want to find the pointer, segment and cycle state of the new trb
  481. * (the one after current TD's last_trb). We know the cycle state at
  482. * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are
  483. * found.
  484. */
  485. do {
  486. if (!cycle_found && xhci_trb_virt_to_dma(new_seg, new_deq)
  487. == (dma_addr_t)(hw_dequeue & ~0xf)) {
  488. cycle_found = true;
  489. if (td_last_trb_found)
  490. break;
  491. }
  492. if (new_deq == cur_td->last_trb)
  493. td_last_trb_found = true;
  494. if (cycle_found && trb_is_link(new_deq) &&
  495. link_trb_toggles_cycle(new_deq))
  496. state->new_cycle_state ^= 0x1;
  497. next_trb(xhci, ep_ring, &new_seg, &new_deq);
  498. /* Search wrapped around, bail out */
  499. if (new_deq == ep->ring->dequeue) {
  500. xhci_err(xhci, "Error: Failed finding new dequeue state\n");
  501. state->new_deq_seg = NULL;
  502. state->new_deq_ptr = NULL;
  503. return;
  504. }
  505. } while (!cycle_found || !td_last_trb_found);
  506. state->new_deq_seg = new_seg;
  507. state->new_deq_ptr = new_deq;
  508. /* Don't update the ring cycle state for the producer (us). */
  509. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  510. "Cycle state = 0x%x", state->new_cycle_state);
  511. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  512. "New dequeue segment = %p (virtual)",
  513. state->new_deq_seg);
  514. addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr);
  515. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  516. "New dequeue pointer = 0x%llx (DMA)",
  517. (unsigned long long) addr);
  518. }
  519. /* flip_cycle means flip the cycle bit of all but the first and last TRB.
  520. * (The last TRB actually points to the ring enqueue pointer, which is not part
  521. * of this TD.) This is used to remove partially enqueued isoc TDs from a ring.
  522. */
  523. static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
  524. struct xhci_td *td, bool flip_cycle)
  525. {
  526. struct xhci_segment *seg = td->start_seg;
  527. union xhci_trb *trb = td->first_trb;
  528. while (1) {
  529. trb_to_noop(trb, TRB_TR_NOOP);
  530. /* flip cycle if asked to */
  531. if (flip_cycle && trb != td->first_trb && trb != td->last_trb)
  532. trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
  533. if (trb == td->last_trb)
  534. break;
  535. next_trb(xhci, ep_ring, &seg, &trb);
  536. }
  537. }
  538. static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci,
  539. struct xhci_virt_ep *ep)
  540. {
  541. ep->ep_state &= ~EP_STOP_CMD_PENDING;
  542. /* Can't del_timer_sync in interrupt */
  543. del_timer(&ep->stop_cmd_timer);
  544. }
  545. /*
  546. * Must be called with xhci->lock held in interrupt context,
  547. * releases and re-acquires xhci->lock
  548. */
  549. static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
  550. struct xhci_td *cur_td, int status)
  551. {
  552. struct urb *urb = cur_td->urb;
  553. struct urb_priv *urb_priv = urb->hcpriv;
  554. struct usb_hcd *hcd = bus_to_hcd(urb->dev->bus);
  555. if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
  556. xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
  557. if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
  558. if (xhci->quirks & XHCI_AMD_PLL_FIX)
  559. usb_amd_quirk_pll_enable();
  560. }
  561. }
  562. xhci_urb_free_priv(urb_priv);
  563. usb_hcd_unlink_urb_from_ep(hcd, urb);
  564. spin_unlock(&xhci->lock);
  565. trace_xhci_urb_giveback(urb);
  566. usb_hcd_giveback_urb(hcd, urb, status);
  567. spin_lock(&xhci->lock);
  568. }
  569. static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
  570. struct xhci_ring *ring, struct xhci_td *td)
  571. {
  572. struct device *dev = xhci_to_hcd(xhci)->self.controller;
  573. struct xhci_segment *seg = td->bounce_seg;
  574. struct urb *urb = td->urb;
  575. size_t len;
  576. if (!ring || !seg || !urb)
  577. return;
  578. if (usb_urb_dir_out(urb)) {
  579. dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
  580. DMA_TO_DEVICE);
  581. return;
  582. }
  583. dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
  584. DMA_FROM_DEVICE);
  585. /* for in tranfers we need to copy the data from bounce to sg */
  586. if (urb->num_sgs) {
  587. len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs, seg->bounce_buf,
  588. seg->bounce_len, seg->bounce_offs);
  589. if (len != seg->bounce_len)
  590. xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n",
  591. len, seg->bounce_len);
  592. } else {
  593. memcpy(urb->transfer_buffer + seg->bounce_offs, seg->bounce_buf,
  594. seg->bounce_len);
  595. }
  596. seg->bounce_len = 0;
  597. seg->bounce_offs = 0;
  598. }
  599. /*
  600. * When we get a command completion for a Stop Endpoint Command, we need to
  601. * unlink any cancelled TDs from the ring. There are two ways to do that:
  602. *
  603. * 1. If the HW was in the middle of processing the TD that needs to be
  604. * cancelled, then we must move the ring's dequeue pointer past the last TRB
  605. * in the TD with a Set Dequeue Pointer Command.
  606. * 2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain
  607. * bit cleared) so that the HW will skip over them.
  608. */
  609. static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
  610. union xhci_trb *trb, struct xhci_event_cmd *event)
  611. {
  612. unsigned int ep_index;
  613. struct xhci_ring *ep_ring;
  614. struct xhci_virt_ep *ep;
  615. struct xhci_td *cur_td = NULL;
  616. struct xhci_td *last_unlinked_td;
  617. struct xhci_ep_ctx *ep_ctx;
  618. struct xhci_virt_device *vdev;
  619. u64 hw_deq;
  620. struct xhci_dequeue_state deq_state;
  621. if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) {
  622. if (!xhci->devs[slot_id])
  623. xhci_warn(xhci, "Stop endpoint command "
  624. "completion for disabled slot %u\n",
  625. slot_id);
  626. return;
  627. }
  628. memset(&deq_state, 0, sizeof(deq_state));
  629. ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
  630. vdev = xhci->devs[slot_id];
  631. ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
  632. trace_xhci_handle_cmd_stop_ep(ep_ctx);
  633. ep = &xhci->devs[slot_id]->eps[ep_index];
  634. last_unlinked_td = list_last_entry(&ep->cancelled_td_list,
  635. struct xhci_td, cancelled_td_list);
  636. if (list_empty(&ep->cancelled_td_list)) {
  637. xhci_stop_watchdog_timer_in_irq(xhci, ep);
  638. ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
  639. return;
  640. }
  641. /* Fix up the ep ring first, so HW stops executing cancelled TDs.
  642. * We have the xHCI lock, so nothing can modify this list until we drop
  643. * it. We're also in the event handler, so we can't get re-interrupted
  644. * if another Stop Endpoint command completes
  645. */
  646. list_for_each_entry(cur_td, &ep->cancelled_td_list, cancelled_td_list) {
  647. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  648. "Removing canceled TD starting at 0x%llx (dma).",
  649. (unsigned long long)xhci_trb_virt_to_dma(
  650. cur_td->start_seg, cur_td->first_trb));
  651. ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
  652. if (!ep_ring) {
  653. /* This shouldn't happen unless a driver is mucking
  654. * with the stream ID after submission. This will
  655. * leave the TD on the hardware ring, and the hardware
  656. * will try to execute it, and may access a buffer
  657. * that has already been freed. In the best case, the
  658. * hardware will execute it, and the event handler will
  659. * ignore the completion event for that TD, since it was
  660. * removed from the td_list for that endpoint. In
  661. * short, don't muck with the stream ID after
  662. * submission.
  663. */
  664. xhci_warn(xhci, "WARN Cancelled URB %p "
  665. "has invalid stream ID %u.\n",
  666. cur_td->urb,
  667. cur_td->urb->stream_id);
  668. goto remove_finished_td;
  669. }
  670. /*
  671. * If we stopped on the TD we need to cancel, then we have to
  672. * move the xHC endpoint ring dequeue pointer past this TD.
  673. */
  674. hw_deq = xhci_get_hw_deq(xhci, vdev, ep_index,
  675. cur_td->urb->stream_id);
  676. hw_deq &= ~0xf;
  677. if (trb_in_td(xhci, cur_td->start_seg, cur_td->first_trb,
  678. cur_td->last_trb, hw_deq, false)) {
  679. xhci_find_new_dequeue_state(xhci, slot_id, ep_index,
  680. cur_td->urb->stream_id,
  681. cur_td, &deq_state);
  682. } else {
  683. td_to_noop(xhci, ep_ring, cur_td, false);
  684. }
  685. remove_finished_td:
  686. /*
  687. * The event handler won't see a completion for this TD anymore,
  688. * so remove it from the endpoint ring's TD list. Keep it in
  689. * the cancelled TD list for URB completion later.
  690. */
  691. list_del_init(&cur_td->td_list);
  692. }
  693. xhci_stop_watchdog_timer_in_irq(xhci, ep);
  694. /* If necessary, queue a Set Transfer Ring Dequeue Pointer command */
  695. if (deq_state.new_deq_ptr && deq_state.new_deq_seg) {
  696. xhci_queue_new_dequeue_state(xhci, slot_id, ep_index,
  697. &deq_state);
  698. xhci_ring_cmd_db(xhci);
  699. } else {
  700. /* Otherwise ring the doorbell(s) to restart queued transfers */
  701. ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
  702. }
  703. /*
  704. * Drop the lock and complete the URBs in the cancelled TD list.
  705. * New TDs to be cancelled might be added to the end of the list before
  706. * we can complete all the URBs for the TDs we already unlinked.
  707. * So stop when we've completed the URB for the last TD we unlinked.
  708. */
  709. do {
  710. cur_td = list_first_entry(&ep->cancelled_td_list,
  711. struct xhci_td, cancelled_td_list);
  712. list_del_init(&cur_td->cancelled_td_list);
  713. /* Clean up the cancelled URB */
  714. /* Doesn't matter what we pass for status, since the core will
  715. * just overwrite it (because the URB has been unlinked).
  716. */
  717. ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
  718. xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td);
  719. inc_td_cnt(cur_td->urb);
  720. if (last_td_in_urb(cur_td))
  721. xhci_giveback_urb_in_irq(xhci, cur_td, 0);
  722. /* Stop processing the cancelled list if the watchdog timer is
  723. * running.
  724. */
  725. if (xhci->xhc_state & XHCI_STATE_DYING)
  726. return;
  727. } while (cur_td != last_unlinked_td);
  728. /* Return to the event handler with xhci->lock re-acquired */
  729. }
  730. static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring)
  731. {
  732. struct xhci_td *cur_td;
  733. struct xhci_td *tmp;
  734. list_for_each_entry_safe(cur_td, tmp, &ring->td_list, td_list) {
  735. list_del_init(&cur_td->td_list);
  736. if (!list_empty(&cur_td->cancelled_td_list))
  737. list_del_init(&cur_td->cancelled_td_list);
  738. xhci_unmap_td_bounce_buffer(xhci, ring, cur_td);
  739. inc_td_cnt(cur_td->urb);
  740. if (last_td_in_urb(cur_td))
  741. xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
  742. }
  743. }
  744. static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci,
  745. int slot_id, int ep_index)
  746. {
  747. struct xhci_td *cur_td;
  748. struct xhci_td *tmp;
  749. struct xhci_virt_ep *ep;
  750. struct xhci_ring *ring;
  751. ep = &xhci->devs[slot_id]->eps[ep_index];
  752. if ((ep->ep_state & EP_HAS_STREAMS) ||
  753. (ep->ep_state & EP_GETTING_NO_STREAMS)) {
  754. int stream_id;
  755. for (stream_id = 1; stream_id < ep->stream_info->num_streams;
  756. stream_id++) {
  757. ring = ep->stream_info->stream_rings[stream_id];
  758. if (!ring)
  759. continue;
  760. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  761. "Killing URBs for slot ID %u, ep index %u, stream %u",
  762. slot_id, ep_index, stream_id);
  763. xhci_kill_ring_urbs(xhci, ring);
  764. }
  765. } else {
  766. ring = ep->ring;
  767. if (!ring)
  768. return;
  769. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  770. "Killing URBs for slot ID %u, ep index %u",
  771. slot_id, ep_index);
  772. xhci_kill_ring_urbs(xhci, ring);
  773. }
  774. list_for_each_entry_safe(cur_td, tmp, &ep->cancelled_td_list,
  775. cancelled_td_list) {
  776. list_del_init(&cur_td->cancelled_td_list);
  777. inc_td_cnt(cur_td->urb);
  778. if (last_td_in_urb(cur_td))
  779. xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
  780. }
  781. }
  782. /*
  783. * host controller died, register read returns 0xffffffff
  784. * Complete pending commands, mark them ABORTED.
  785. * URBs need to be given back as usb core might be waiting with device locks
  786. * held for the URBs to finish during device disconnect, blocking host remove.
  787. *
  788. * Call with xhci->lock held.
  789. * lock is relased and re-acquired while giving back urb.
  790. */
  791. void xhci_hc_died(struct xhci_hcd *xhci)
  792. {
  793. int i, j;
  794. if (xhci->xhc_state & XHCI_STATE_DYING)
  795. return;
  796. xhci_err(xhci, "xHCI host controller not responding, assume dead\n");
  797. xhci->xhc_state |= XHCI_STATE_DYING;
  798. xhci_cleanup_command_queue(xhci);
  799. /* return any pending urbs, remove may be waiting for them */
  800. for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) {
  801. if (!xhci->devs[i])
  802. continue;
  803. for (j = 0; j < 31; j++)
  804. xhci_kill_endpoint_urbs(xhci, i, j);
  805. }
  806. /* inform usb core hc died if PCI remove isn't already handling it */
  807. if (!(xhci->xhc_state & XHCI_STATE_REMOVING))
  808. usb_hc_died(xhci_to_hcd(xhci));
  809. }
  810. /* Watchdog timer function for when a stop endpoint command fails to complete.
  811. * In this case, we assume the host controller is broken or dying or dead. The
  812. * host may still be completing some other events, so we have to be careful to
  813. * let the event ring handler and the URB dequeueing/enqueueing functions know
  814. * through xhci->state.
  815. *
  816. * The timer may also fire if the host takes a very long time to respond to the
  817. * command, and the stop endpoint command completion handler cannot delete the
  818. * timer before the timer function is called. Another endpoint cancellation may
  819. * sneak in before the timer function can grab the lock, and that may queue
  820. * another stop endpoint command and add the timer back. So we cannot use a
  821. * simple flag to say whether there is a pending stop endpoint command for a
  822. * particular endpoint.
  823. *
  824. * Instead we use a combination of that flag and checking if a new timer is
  825. * pending.
  826. */
  827. void xhci_stop_endpoint_command_watchdog(struct timer_list *t)
  828. {
  829. struct xhci_virt_ep *ep = from_timer(ep, t, stop_cmd_timer);
  830. struct xhci_hcd *xhci = ep->xhci;
  831. unsigned long flags;
  832. spin_lock_irqsave(&xhci->lock, flags);
  833. /* bail out if cmd completed but raced with stop ep watchdog timer.*/
  834. if (!(ep->ep_state & EP_STOP_CMD_PENDING) ||
  835. timer_pending(&ep->stop_cmd_timer)) {
  836. spin_unlock_irqrestore(&xhci->lock, flags);
  837. xhci_dbg(xhci, "Stop EP timer raced with cmd completion, exit");
  838. return;
  839. }
  840. xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n");
  841. ep->ep_state &= ~EP_STOP_CMD_PENDING;
  842. xhci_halt(xhci);
  843. /*
  844. * handle a stop endpoint cmd timeout as if host died (-ENODEV).
  845. * In the future we could distinguish between -ENODEV and -ETIMEDOUT
  846. * and try to recover a -ETIMEDOUT with a host controller reset
  847. */
  848. xhci_hc_died(xhci);
  849. spin_unlock_irqrestore(&xhci->lock, flags);
  850. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  851. "xHCI host controller is dead.");
  852. }
  853. static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci,
  854. struct xhci_virt_device *dev,
  855. struct xhci_ring *ep_ring,
  856. unsigned int ep_index)
  857. {
  858. union xhci_trb *dequeue_temp;
  859. int num_trbs_free_temp;
  860. bool revert = false;
  861. num_trbs_free_temp = ep_ring->num_trbs_free;
  862. dequeue_temp = ep_ring->dequeue;
  863. /* If we get two back-to-back stalls, and the first stalled transfer
  864. * ends just before a link TRB, the dequeue pointer will be left on
  865. * the link TRB by the code in the while loop. So we have to update
  866. * the dequeue pointer one segment further, or we'll jump off
  867. * the segment into la-la-land.
  868. */
  869. if (trb_is_link(ep_ring->dequeue)) {
  870. ep_ring->deq_seg = ep_ring->deq_seg->next;
  871. ep_ring->dequeue = ep_ring->deq_seg->trbs;
  872. }
  873. while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {
  874. /* We have more usable TRBs */
  875. ep_ring->num_trbs_free++;
  876. ep_ring->dequeue++;
  877. if (trb_is_link(ep_ring->dequeue)) {
  878. if (ep_ring->dequeue ==
  879. dev->eps[ep_index].queued_deq_ptr)
  880. break;
  881. ep_ring->deq_seg = ep_ring->deq_seg->next;
  882. ep_ring->dequeue = ep_ring->deq_seg->trbs;
  883. }
  884. if (ep_ring->dequeue == dequeue_temp) {
  885. revert = true;
  886. break;
  887. }
  888. }
  889. if (revert) {
  890. xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
  891. ep_ring->num_trbs_free = num_trbs_free_temp;
  892. }
  893. }
  894. /*
  895. * When we get a completion for a Set Transfer Ring Dequeue Pointer command,
  896. * we need to clear the set deq pending flag in the endpoint ring state, so that
  897. * the TD queueing code can ring the doorbell again. We also need to ring the
  898. * endpoint doorbell to restart the ring, but only if there aren't more
  899. * cancellations pending.
  900. */
  901. static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
  902. union xhci_trb *trb, u32 cmd_comp_code)
  903. {
  904. unsigned int ep_index;
  905. unsigned int stream_id;
  906. struct xhci_ring *ep_ring;
  907. struct xhci_virt_device *dev;
  908. struct xhci_virt_ep *ep;
  909. struct xhci_ep_ctx *ep_ctx;
  910. struct xhci_slot_ctx *slot_ctx;
  911. ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
  912. stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
  913. dev = xhci->devs[slot_id];
  914. ep = &dev->eps[ep_index];
  915. ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id);
  916. if (!ep_ring) {
  917. xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
  918. stream_id);
  919. /* XXX: Harmless??? */
  920. goto cleanup;
  921. }
  922. ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
  923. slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
  924. trace_xhci_handle_cmd_set_deq(slot_ctx);
  925. trace_xhci_handle_cmd_set_deq_ep(ep_ctx);
  926. if (cmd_comp_code != COMP_SUCCESS) {
  927. unsigned int ep_state;
  928. unsigned int slot_state;
  929. switch (cmd_comp_code) {
  930. case COMP_TRB_ERROR:
  931. xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n");
  932. break;
  933. case COMP_CONTEXT_STATE_ERROR:
  934. xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n");
  935. ep_state = GET_EP_CTX_STATE(ep_ctx);
  936. slot_state = le32_to_cpu(slot_ctx->dev_state);
  937. slot_state = GET_SLOT_STATE(slot_state);
  938. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  939. "Slot state = %u, EP state = %u",
  940. slot_state, ep_state);
  941. break;
  942. case COMP_SLOT_NOT_ENABLED_ERROR:
  943. xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n",
  944. slot_id);
  945. break;
  946. default:
  947. xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n",
  948. cmd_comp_code);
  949. break;
  950. }
  951. /* OK what do we do now? The endpoint state is hosed, and we
  952. * should never get to this point if the synchronization between
  953. * queueing, and endpoint state are correct. This might happen
  954. * if the device gets disconnected after we've finished
  955. * cancelling URBs, which might not be an error...
  956. */
  957. } else {
  958. u64 deq;
  959. /* 4.6.10 deq ptr is written to the stream ctx for streams */
  960. if (ep->ep_state & EP_HAS_STREAMS) {
  961. struct xhci_stream_ctx *ctx =
  962. &ep->stream_info->stream_ctx_array[stream_id];
  963. deq = le64_to_cpu(ctx->stream_ring) & SCTX_DEQ_MASK;
  964. } else {
  965. deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK;
  966. }
  967. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  968. "Successful Set TR Deq Ptr cmd, deq = @%08llx", deq);
  969. if (xhci_trb_virt_to_dma(ep->queued_deq_seg,
  970. ep->queued_deq_ptr) == deq) {
  971. /* Update the ring's dequeue segment and dequeue pointer
  972. * to reflect the new position.
  973. */
  974. update_ring_for_set_deq_completion(xhci, dev,
  975. ep_ring, ep_index);
  976. } else {
  977. xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
  978. xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n",
  979. ep->queued_deq_seg, ep->queued_deq_ptr);
  980. }
  981. }
  982. cleanup:
  983. dev->eps[ep_index].ep_state &= ~SET_DEQ_PENDING;
  984. dev->eps[ep_index].queued_deq_seg = NULL;
  985. dev->eps[ep_index].queued_deq_ptr = NULL;
  986. /* Restart any rings with pending URBs */
  987. ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
  988. }
  989. static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
  990. union xhci_trb *trb, u32 cmd_comp_code)
  991. {
  992. struct xhci_virt_device *vdev;
  993. struct xhci_ep_ctx *ep_ctx;
  994. unsigned int ep_index;
  995. ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
  996. vdev = xhci->devs[slot_id];
  997. ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
  998. trace_xhci_handle_cmd_reset_ep(ep_ctx);
  999. /* This command will only fail if the endpoint wasn't halted,
  1000. * but we don't care.
  1001. */
  1002. xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
  1003. "Ignoring reset ep completion code of %u", cmd_comp_code);
  1004. /* HW with the reset endpoint quirk needs to have a configure endpoint
  1005. * command complete before the endpoint can be used. Queue that here
  1006. * because the HW can't handle two commands being queued in a row.
  1007. */
  1008. if (xhci->quirks & XHCI_RESET_EP_QUIRK) {
  1009. struct xhci_command *command;
  1010. command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
  1011. if (!command)
  1012. return;
  1013. xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
  1014. "Queueing configure endpoint command");
  1015. xhci_queue_configure_endpoint(xhci, command,
  1016. xhci->devs[slot_id]->in_ctx->dma, slot_id,
  1017. false);
  1018. xhci_ring_cmd_db(xhci);
  1019. } else {
  1020. /* Clear our internal halted state */
  1021. xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_HALTED;
  1022. }
  1023. }
  1024. static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
  1025. struct xhci_command *command, u32 cmd_comp_code)
  1026. {
  1027. if (cmd_comp_code == COMP_SUCCESS)
  1028. command->slot_id = slot_id;
  1029. else
  1030. command->slot_id = 0;
  1031. }
  1032. static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
  1033. {
  1034. struct xhci_virt_device *virt_dev;
  1035. struct xhci_slot_ctx *slot_ctx;
  1036. virt_dev = xhci->devs[slot_id];
  1037. if (!virt_dev)
  1038. return;
  1039. slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
  1040. trace_xhci_handle_cmd_disable_slot(slot_ctx);
  1041. if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
  1042. /* Delete default control endpoint resources */
  1043. xhci_free_device_endpoint_resources(xhci, virt_dev, true);
  1044. xhci_free_virt_device(xhci, slot_id);
  1045. }
  1046. static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
  1047. struct xhci_event_cmd *event, u32 cmd_comp_code)
  1048. {
  1049. struct xhci_virt_device *virt_dev;
  1050. struct xhci_input_control_ctx *ctrl_ctx;
  1051. struct xhci_ep_ctx *ep_ctx;
  1052. unsigned int ep_index;
  1053. unsigned int ep_state;
  1054. u32 add_flags, drop_flags;
  1055. /*
  1056. * Configure endpoint commands can come from the USB core
  1057. * configuration or alt setting changes, or because the HW
  1058. * needed an extra configure endpoint command after a reset
  1059. * endpoint command or streams were being configured.
  1060. * If the command was for a halted endpoint, the xHCI driver
  1061. * is not waiting on the configure endpoint command.
  1062. */
  1063. virt_dev = xhci->devs[slot_id];
  1064. ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
  1065. if (!ctrl_ctx) {
  1066. xhci_warn(xhci, "Could not get input context, bad type.\n");
  1067. return;
  1068. }
  1069. add_flags = le32_to_cpu(ctrl_ctx->add_flags);
  1070. drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
  1071. /* Input ctx add_flags are the endpoint index plus one */
  1072. ep_index = xhci_last_valid_endpoint(add_flags) - 1;
  1073. ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index);
  1074. trace_xhci_handle_cmd_config_ep(ep_ctx);
  1075. /* A usb_set_interface() call directly after clearing a halted
  1076. * condition may race on this quirky hardware. Not worth
  1077. * worrying about, since this is prototype hardware. Not sure
  1078. * if this will work for streams, but streams support was
  1079. * untested on this prototype.
  1080. */
  1081. if (xhci->quirks & XHCI_RESET_EP_QUIRK &&
  1082. ep_index != (unsigned int) -1 &&
  1083. add_flags - SLOT_FLAG == drop_flags) {
  1084. ep_state = virt_dev->eps[ep_index].ep_state;
  1085. if (!(ep_state & EP_HALTED))
  1086. return;
  1087. xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
  1088. "Completed config ep cmd - "
  1089. "last ep index = %d, state = %d",
  1090. ep_index, ep_state);
  1091. /* Clear internal halted state and restart ring(s) */
  1092. virt_dev->eps[ep_index].ep_state &= ~EP_HALTED;
  1093. ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
  1094. return;
  1095. }
  1096. return;
  1097. }
  1098. static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id)
  1099. {
  1100. struct xhci_virt_device *vdev;
  1101. struct xhci_slot_ctx *slot_ctx;
  1102. vdev = xhci->devs[slot_id];
  1103. slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
  1104. trace_xhci_handle_cmd_addr_dev(slot_ctx);
  1105. }
  1106. static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
  1107. struct xhci_event_cmd *event)
  1108. {
  1109. struct xhci_virt_device *vdev;
  1110. struct xhci_slot_ctx *slot_ctx;
  1111. vdev = xhci->devs[slot_id];
  1112. slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
  1113. trace_xhci_handle_cmd_reset_dev(slot_ctx);
  1114. xhci_dbg(xhci, "Completed reset device command.\n");
  1115. if (!xhci->devs[slot_id])
  1116. xhci_warn(xhci, "Reset device command completion "
  1117. "for disabled slot %u\n", slot_id);
  1118. }
  1119. static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
  1120. struct xhci_event_cmd *event)
  1121. {
  1122. if (!(xhci->quirks & XHCI_NEC_HOST)) {
  1123. xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n");
  1124. return;
  1125. }
  1126. xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
  1127. "NEC firmware version %2x.%02x",
  1128. NEC_FW_MAJOR(le32_to_cpu(event->status)),
  1129. NEC_FW_MINOR(le32_to_cpu(event->status)));
  1130. }
  1131. static void xhci_complete_del_and_free_cmd(struct xhci_command *cmd, u32 status)
  1132. {
  1133. list_del(&cmd->cmd_list);
  1134. if (cmd->completion) {
  1135. cmd->status = status;
  1136. complete(cmd->completion);
  1137. } else {
  1138. kfree(cmd);
  1139. }
  1140. }
  1141. void xhci_cleanup_command_queue(struct xhci_hcd *xhci)
  1142. {
  1143. struct xhci_command *cur_cmd, *tmp_cmd;
  1144. xhci->current_cmd = NULL;
  1145. list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list)
  1146. xhci_complete_del_and_free_cmd(cur_cmd, COMP_COMMAND_ABORTED);
  1147. }
  1148. void xhci_handle_command_timeout(struct work_struct *work)
  1149. {
  1150. struct xhci_hcd *xhci;
  1151. unsigned long flags;
  1152. u64 hw_ring_state;
  1153. xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer);
  1154. spin_lock_irqsave(&xhci->lock, flags);
  1155. /*
  1156. * If timeout work is pending, or current_cmd is NULL, it means we
  1157. * raced with command completion. Command is handled so just return.
  1158. */
  1159. if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) {
  1160. spin_unlock_irqrestore(&xhci->lock, flags);
  1161. return;
  1162. }
  1163. /* mark this command to be cancelled */
  1164. xhci->current_cmd->status = COMP_COMMAND_ABORTED;
  1165. /* Make sure command ring is running before aborting it */
  1166. hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
  1167. if (hw_ring_state == ~(u64)0) {
  1168. xhci_hc_died(xhci);
  1169. goto time_out_completed;
  1170. }
  1171. if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) &&
  1172. (hw_ring_state & CMD_RING_RUNNING)) {
  1173. /* Prevent new doorbell, and start command abort */
  1174. xhci->cmd_ring_state = CMD_RING_STATE_ABORTED;
  1175. xhci_dbg(xhci, "Command timeout\n");
  1176. xhci_abort_cmd_ring(xhci, flags);
  1177. goto time_out_completed;
  1178. }
  1179. /* host removed. Bail out */
  1180. if (xhci->xhc_state & XHCI_STATE_REMOVING) {
  1181. xhci_dbg(xhci, "host removed, ring start fail?\n");
  1182. xhci_cleanup_command_queue(xhci);
  1183. goto time_out_completed;
  1184. }
  1185. /* command timeout on stopped ring, ring can't be aborted */
  1186. xhci_dbg(xhci, "Command timeout on stopped ring\n");
  1187. xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd);
  1188. time_out_completed:
  1189. spin_unlock_irqrestore(&xhci->lock, flags);
  1190. return;
  1191. }
  1192. static void handle_cmd_completion(struct xhci_hcd *xhci,
  1193. struct xhci_event_cmd *event)
  1194. {
  1195. int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
  1196. u64 cmd_dma;
  1197. dma_addr_t cmd_dequeue_dma;
  1198. u32 cmd_comp_code;
  1199. union xhci_trb *cmd_trb;
  1200. struct xhci_command *cmd;
  1201. u32 cmd_type;
  1202. cmd_dma = le64_to_cpu(event->cmd_trb);
  1203. cmd_trb = xhci->cmd_ring->dequeue;
  1204. trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic);
  1205. cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
  1206. cmd_trb);
  1207. /*
  1208. * Check whether the completion event is for our internal kept
  1209. * command.
  1210. */
  1211. if (!cmd_dequeue_dma || cmd_dma != (u64)cmd_dequeue_dma) {
  1212. xhci_warn(xhci,
  1213. "ERROR mismatched command completion event\n");
  1214. return;
  1215. }
  1216. cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list);
  1217. cancel_delayed_work(&xhci->cmd_timer);
  1218. cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status));
  1219. /* If CMD ring stopped we own the trbs between enqueue and dequeue */
  1220. if (cmd_comp_code == COMP_COMMAND_RING_STOPPED) {
  1221. complete_all(&xhci->cmd_ring_stop_completion);
  1222. return;
  1223. }
  1224. if (cmd->command_trb != xhci->cmd_ring->dequeue) {
  1225. xhci_err(xhci,
  1226. "Command completion event does not match command\n");
  1227. return;
  1228. }
  1229. /*
  1230. * Host aborted the command ring, check if the current command was
  1231. * supposed to be aborted, otherwise continue normally.
  1232. * The command ring is stopped now, but the xHC will issue a Command
  1233. * Ring Stopped event which will cause us to restart it.
  1234. */
  1235. if (cmd_comp_code == COMP_COMMAND_ABORTED) {
  1236. xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
  1237. if (cmd->status == COMP_COMMAND_ABORTED) {
  1238. if (xhci->current_cmd == cmd)
  1239. xhci->current_cmd = NULL;
  1240. goto event_handled;
  1241. }
  1242. }
  1243. cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3]));
  1244. switch (cmd_type) {
  1245. case TRB_ENABLE_SLOT:
  1246. xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code);
  1247. break;
  1248. case TRB_DISABLE_SLOT:
  1249. xhci_handle_cmd_disable_slot(xhci, slot_id);
  1250. break;
  1251. case TRB_CONFIG_EP:
  1252. if (!cmd->completion)
  1253. xhci_handle_cmd_config_ep(xhci, slot_id, event,
  1254. cmd_comp_code);
  1255. break;
  1256. case TRB_EVAL_CONTEXT:
  1257. break;
  1258. case TRB_ADDR_DEV:
  1259. xhci_handle_cmd_addr_dev(xhci, slot_id);
  1260. break;
  1261. case TRB_STOP_RING:
  1262. WARN_ON(slot_id != TRB_TO_SLOT_ID(
  1263. le32_to_cpu(cmd_trb->generic.field[3])));
  1264. if (!cmd->completion)
  1265. xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event);
  1266. break;
  1267. case TRB_SET_DEQ:
  1268. WARN_ON(slot_id != TRB_TO_SLOT_ID(
  1269. le32_to_cpu(cmd_trb->generic.field[3])));
  1270. xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code);
  1271. break;
  1272. case TRB_CMD_NOOP:
  1273. /* Is this an aborted command turned to NO-OP? */
  1274. if (cmd->status == COMP_COMMAND_RING_STOPPED)
  1275. cmd_comp_code = COMP_COMMAND_RING_STOPPED;
  1276. break;
  1277. case TRB_RESET_EP:
  1278. WARN_ON(slot_id != TRB_TO_SLOT_ID(
  1279. le32_to_cpu(cmd_trb->generic.field[3])));
  1280. xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code);
  1281. break;
  1282. case TRB_RESET_DEV:
  1283. /* SLOT_ID field in reset device cmd completion event TRB is 0.
  1284. * Use the SLOT_ID from the command TRB instead (xhci 4.6.11)
  1285. */
  1286. slot_id = TRB_TO_SLOT_ID(
  1287. le32_to_cpu(cmd_trb->generic.field[3]));
  1288. xhci_handle_cmd_reset_dev(xhci, slot_id, event);
  1289. break;
  1290. case TRB_NEC_GET_FW:
  1291. xhci_handle_cmd_nec_get_fw(xhci, event);
  1292. break;
  1293. default:
  1294. /* Skip over unknown commands on the event ring */
  1295. xhci_info(xhci, "INFO unknown command type %d\n", cmd_type);
  1296. break;
  1297. }
  1298. /* restart timer if this wasn't the last command */
  1299. if (!list_is_singular(&xhci->cmd_list)) {
  1300. xhci->current_cmd = list_first_entry(&cmd->cmd_list,
  1301. struct xhci_command, cmd_list);
  1302. xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
  1303. } else if (xhci->current_cmd == cmd) {
  1304. xhci->current_cmd = NULL;
  1305. }
  1306. event_handled:
  1307. xhci_complete_del_and_free_cmd(cmd, cmd_comp_code);
  1308. inc_deq(xhci, xhci->cmd_ring);
  1309. }
  1310. static void handle_vendor_event(struct xhci_hcd *xhci,
  1311. union xhci_trb *event)
  1312. {
  1313. u32 trb_type;
  1314. trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3]));
  1315. xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
  1316. if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
  1317. handle_cmd_completion(xhci, &event->event_cmd);
  1318. }
  1319. static void handle_device_notification(struct xhci_hcd *xhci,
  1320. union xhci_trb *event)
  1321. {
  1322. u32 slot_id;
  1323. struct usb_device *udev;
  1324. slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3]));
  1325. if (!xhci->devs[slot_id]) {
  1326. xhci_warn(xhci, "Device Notification event for "
  1327. "unused slot %u\n", slot_id);
  1328. return;
  1329. }
  1330. xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n",
  1331. slot_id);
  1332. udev = xhci->devs[slot_id]->udev;
  1333. if (udev && udev->parent)
  1334. usb_wakeup_notification(udev->parent, udev->portnum);
  1335. }
  1336. /*
  1337. * Quirk hanlder for errata seen on Cavium ThunderX2 processor XHCI
  1338. * Controller.
  1339. * As per ThunderX2errata-129 USB 2 device may come up as USB 1
  1340. * If a connection to a USB 1 device is followed by another connection
  1341. * to a USB 2 device.
  1342. *
  1343. * Reset the PHY after the USB device is disconnected if device speed
  1344. * is less than HCD_USB3.
  1345. * Retry the reset sequence max of 4 times checking the PLL lock status.
  1346. *
  1347. */
  1348. static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci)
  1349. {
  1350. struct usb_hcd *hcd = xhci_to_hcd(xhci);
  1351. u32 pll_lock_check;
  1352. u32 retry_count = 4;
  1353. do {
  1354. /* Assert PHY reset */
  1355. writel(0x6F, hcd->regs + 0x1048);
  1356. udelay(10);
  1357. /* De-assert the PHY reset */
  1358. writel(0x7F, hcd->regs + 0x1048);
  1359. udelay(200);
  1360. pll_lock_check = readl(hcd->regs + 0x1070);
  1361. } while (!(pll_lock_check & 0x1) && --retry_count);
  1362. }
  1363. static void handle_port_status(struct xhci_hcd *xhci,
  1364. union xhci_trb *event)
  1365. {
  1366. struct usb_hcd *hcd;
  1367. u32 port_id;
  1368. u32 portsc, cmd_reg;
  1369. int max_ports;
  1370. int slot_id;
  1371. unsigned int hcd_portnum;
  1372. struct xhci_bus_state *bus_state;
  1373. bool bogus_port_status = false;
  1374. struct xhci_port *port;
  1375. /* Port status change events always have a successful completion code */
  1376. if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS)
  1377. xhci_warn(xhci,
  1378. "WARN: xHC returned failed port status event\n");
  1379. port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
  1380. xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id);
  1381. max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
  1382. if ((port_id <= 0) || (port_id > max_ports)) {
  1383. xhci_warn(xhci, "Invalid port id %d\n", port_id);
  1384. inc_deq(xhci, xhci->event_ring);
  1385. return;
  1386. }
  1387. port = &xhci->hw_ports[port_id - 1];
  1388. if (!port || !port->rhub || port->hcd_portnum == DUPLICATE_ENTRY) {
  1389. xhci_warn(xhci, "Event for invalid port %u\n", port_id);
  1390. bogus_port_status = true;
  1391. goto cleanup;
  1392. }
  1393. /* We might get interrupts after shared_hcd is removed */
  1394. if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) {
  1395. xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n");
  1396. bogus_port_status = true;
  1397. goto cleanup;
  1398. }
  1399. hcd = port->rhub->hcd;
  1400. bus_state = &xhci->bus_state[hcd_index(hcd)];
  1401. hcd_portnum = port->hcd_portnum;
  1402. portsc = readl(port->addr);
  1403. trace_xhci_handle_port_status(hcd_portnum, portsc);
  1404. if (hcd->state == HC_STATE_SUSPENDED) {
  1405. xhci_dbg(xhci, "resume root hub\n");
  1406. usb_hcd_resume_root_hub(hcd);
  1407. }
  1408. if (hcd->speed >= HCD_USB3 &&
  1409. (portsc & PORT_PLS_MASK) == XDEV_INACTIVE) {
  1410. slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1);
  1411. if (slot_id && xhci->devs[slot_id])
  1412. xhci->devs[slot_id]->flags |= VDEV_PORT_ERROR;
  1413. }
  1414. if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) {
  1415. xhci_dbg(xhci, "port resume event for port %d\n", port_id);
  1416. cmd_reg = readl(&xhci->op_regs->command);
  1417. if (!(cmd_reg & CMD_RUN)) {
  1418. xhci_warn(xhci, "xHC is not running.\n");
  1419. goto cleanup;
  1420. }
  1421. if (DEV_SUPERSPEED_ANY(portsc)) {
  1422. xhci_dbg(xhci, "remote wake SS port %d\n", port_id);
  1423. /* Set a flag to say the port signaled remote wakeup,
  1424. * so we can tell the difference between the end of
  1425. * device and host initiated resume.
  1426. */
  1427. bus_state->port_remote_wakeup |= 1 << hcd_portnum;
  1428. xhci_test_and_clear_bit(xhci, port, PORT_PLC);
  1429. xhci_set_link_state(xhci, port, XDEV_U0);
  1430. usb_hcd_start_port_resume(&hcd->self, hcd_portnum);
  1431. /* Need to wait until the next link state change
  1432. * indicates the device is actually in U0.
  1433. */
  1434. bogus_port_status = true;
  1435. goto cleanup;
  1436. } else if (!test_bit(hcd_portnum, &bus_state->resuming_ports)) {
  1437. xhci_dbg(xhci, "resume HS port %d\n", port_id);
  1438. bus_state->resume_done[hcd_portnum] = jiffies +
  1439. msecs_to_jiffies(USB_RESUME_TIMEOUT);
  1440. set_bit(hcd_portnum, &bus_state->resuming_ports);
  1441. /* Do the rest in GetPortStatus after resume time delay.
  1442. * Avoid polling roothub status before that so that a
  1443. * usb device auto-resume latency around ~40ms.
  1444. */
  1445. set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
  1446. mod_timer(&hcd->rh_timer,
  1447. bus_state->resume_done[hcd_portnum]);
  1448. usb_hcd_start_port_resume(&hcd->self, hcd_portnum);
  1449. bogus_port_status = true;
  1450. }
  1451. }
  1452. if ((portsc & PORT_PLC) &&
  1453. DEV_SUPERSPEED_ANY(portsc) &&
  1454. ((portsc & PORT_PLS_MASK) == XDEV_U0 ||
  1455. (portsc & PORT_PLS_MASK) == XDEV_U1 ||
  1456. (portsc & PORT_PLS_MASK) == XDEV_U2)) {
  1457. xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
  1458. /* We've just brought the device into U0/1/2 through either the
  1459. * Resume state after a device remote wakeup, or through the
  1460. * U3Exit state after a host-initiated resume. If it's a device
  1461. * initiated remote wake, don't pass up the link state change,
  1462. * so the roothub behavior is consistent with external
  1463. * USB 3.0 hub behavior.
  1464. */
  1465. slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1);
  1466. if (slot_id && xhci->devs[slot_id])
  1467. xhci_ring_device(xhci, slot_id);
  1468. if (bus_state->port_remote_wakeup & (1 << hcd_portnum)) {
  1469. xhci_test_and_clear_bit(xhci, port, PORT_PLC);
  1470. usb_wakeup_notification(hcd->self.root_hub,
  1471. hcd_portnum + 1);
  1472. bogus_port_status = true;
  1473. goto cleanup;
  1474. }
  1475. }
  1476. /*
  1477. * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or
  1478. * RExit to a disconnect state). If so, let the the driver know it's
  1479. * out of the RExit state.
  1480. */
  1481. if (!DEV_SUPERSPEED_ANY(portsc) && hcd->speed < HCD_USB3 &&
  1482. test_and_clear_bit(hcd_portnum,
  1483. &bus_state->rexit_ports)) {
  1484. complete(&bus_state->rexit_done[hcd_portnum]);
  1485. bogus_port_status = true;
  1486. goto cleanup;
  1487. }
  1488. if (hcd->speed < HCD_USB3) {
  1489. xhci_test_and_clear_bit(xhci, port, PORT_PLC);
  1490. if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) &&
  1491. (portsc & PORT_CSC) && !(portsc & PORT_CONNECT))
  1492. xhci_cavium_reset_phy_quirk(xhci);
  1493. }
  1494. cleanup:
  1495. /* Update event ring dequeue pointer before dropping the lock */
  1496. inc_deq(xhci, xhci->event_ring);
  1497. /* Don't make the USB core poll the roothub if we got a bad port status
  1498. * change event. Besides, at that point we can't tell which roothub
  1499. * (USB 2.0 or USB 3.0) to kick.
  1500. */
  1501. if (bogus_port_status)
  1502. return;
  1503. /*
  1504. * xHCI port-status-change events occur when the "or" of all the
  1505. * status-change bits in the portsc register changes from 0 to 1.
  1506. * New status changes won't cause an event if any other change
  1507. * bits are still set. When an event occurs, switch over to
  1508. * polling to avoid losing status changes.
  1509. */
  1510. xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
  1511. set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
  1512. spin_unlock(&xhci->lock);
  1513. /* Pass this up to the core */
  1514. usb_hcd_poll_rh_status(hcd);
  1515. spin_lock(&xhci->lock);
  1516. }
  1517. /*
  1518. * This TD is defined by the TRBs starting at start_trb in start_seg and ending
  1519. * at end_trb, which may be in another segment. If the suspect DMA address is a
  1520. * TRB in this TD, this function returns that TRB's segment. Otherwise it
  1521. * returns 0.
  1522. */
  1523. struct xhci_segment *trb_in_td(struct xhci_hcd *xhci,
  1524. struct xhci_segment *start_seg,
  1525. union xhci_trb *start_trb,
  1526. union xhci_trb *end_trb,
  1527. dma_addr_t suspect_dma,
  1528. bool debug)
  1529. {
  1530. dma_addr_t start_dma;
  1531. dma_addr_t end_seg_dma;
  1532. dma_addr_t end_trb_dma;
  1533. struct xhci_segment *cur_seg;
  1534. start_dma = xhci_trb_virt_to_dma(start_seg, start_trb);
  1535. cur_seg = start_seg;
  1536. do {
  1537. if (start_dma == 0)
  1538. return NULL;
  1539. /* We may get an event for a Link TRB in the middle of a TD */
  1540. end_seg_dma = xhci_trb_virt_to_dma(cur_seg,
  1541. &cur_seg->trbs[TRBS_PER_SEGMENT - 1]);
  1542. /* If the end TRB isn't in this segment, this is set to 0 */
  1543. end_trb_dma = xhci_trb_virt_to_dma(cur_seg, end_trb);
  1544. if (debug)
  1545. xhci_warn(xhci,
  1546. "Looking for event-dma %016llx trb-start %016llx trb-end %016llx seg-start %016llx seg-end %016llx\n",
  1547. (unsigned long long)suspect_dma,
  1548. (unsigned long long)start_dma,
  1549. (unsigned long long)end_trb_dma,
  1550. (unsigned long long)cur_seg->dma,
  1551. (unsigned long long)end_seg_dma);
  1552. if (end_trb_dma > 0) {
  1553. /* The end TRB is in this segment, so suspect should be here */
  1554. if (start_dma <= end_trb_dma) {
  1555. if (suspect_dma >= start_dma && suspect_dma <= end_trb_dma)
  1556. return cur_seg;
  1557. } else {
  1558. /* Case for one segment with
  1559. * a TD wrapped around to the top
  1560. */
  1561. if ((suspect_dma >= start_dma &&
  1562. suspect_dma <= end_seg_dma) ||
  1563. (suspect_dma >= cur_seg->dma &&
  1564. suspect_dma <= end_trb_dma))
  1565. return cur_seg;
  1566. }
  1567. return NULL;
  1568. } else {
  1569. /* Might still be somewhere in this segment */
  1570. if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma)
  1571. return cur_seg;
  1572. }
  1573. cur_seg = cur_seg->next;
  1574. start_dma = xhci_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]);
  1575. } while (cur_seg != start_seg);
  1576. return NULL;
  1577. }
  1578. static void xhci_cleanup_halted_endpoint(struct xhci_hcd *xhci,
  1579. unsigned int slot_id, unsigned int ep_index,
  1580. unsigned int stream_id, struct xhci_td *td,
  1581. enum xhci_ep_reset_type reset_type)
  1582. {
  1583. struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
  1584. struct xhci_command *command;
  1585. /*
  1586. * Avoid resetting endpoint if link is inactive. Can cause host hang.
  1587. * Device will be reset soon to recover the link so don't do anything
  1588. */
  1589. if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR)
  1590. return;
  1591. command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
  1592. if (!command)
  1593. return;
  1594. ep->ep_state |= EP_HALTED;
  1595. xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type);
  1596. if (reset_type == EP_HARD_RESET) {
  1597. ep->ep_state |= EP_HARD_CLEAR_TOGGLE;
  1598. xhci_cleanup_stalled_ring(xhci, ep_index, stream_id, td);
  1599. }
  1600. xhci_ring_cmd_db(xhci);
  1601. }
  1602. /* Check if an error has halted the endpoint ring. The class driver will
  1603. * cleanup the halt for a non-default control endpoint if we indicate a stall.
  1604. * However, a babble and other errors also halt the endpoint ring, and the class
  1605. * driver won't clear the halt in that case, so we need to issue a Set Transfer
  1606. * Ring Dequeue Pointer command manually.
  1607. */
  1608. static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
  1609. struct xhci_ep_ctx *ep_ctx,
  1610. unsigned int trb_comp_code)
  1611. {
  1612. /* TRB completion codes that may require a manual halt cleanup */
  1613. if (trb_comp_code == COMP_USB_TRANSACTION_ERROR ||
  1614. trb_comp_code == COMP_BABBLE_DETECTED_ERROR ||
  1615. trb_comp_code == COMP_SPLIT_TRANSACTION_ERROR)
  1616. /* The 0.95 spec says a babbling control endpoint
  1617. * is not halted. The 0.96 spec says it is. Some HW
  1618. * claims to be 0.95 compliant, but it halts the control
  1619. * endpoint anyway. Check if a babble halted the
  1620. * endpoint.
  1621. */
  1622. if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_HALTED)
  1623. return 1;
  1624. return 0;
  1625. }
  1626. int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code)
  1627. {
  1628. if (trb_comp_code >= 224 && trb_comp_code <= 255) {
  1629. /* Vendor defined "informational" completion code,
  1630. * treat as not-an-error.
  1631. */
  1632. xhci_dbg(xhci, "Vendor defined info completion code %u\n",
  1633. trb_comp_code);
  1634. xhci_dbg(xhci, "Treating code as success.\n");
  1635. return 1;
  1636. }
  1637. return 0;
  1638. }
  1639. static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td,
  1640. struct xhci_ring *ep_ring, int *status)
  1641. {
  1642. struct urb *urb = NULL;
  1643. /* Clean up the endpoint's TD list */
  1644. urb = td->urb;
  1645. /* if a bounce buffer was used to align this td then unmap it */
  1646. xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
  1647. /* Do one last check of the actual transfer length.
  1648. * If the host controller said we transferred more data than the buffer
  1649. * length, urb->actual_length will be a very big number (since it's
  1650. * unsigned). Play it safe and say we didn't transfer anything.
  1651. */
  1652. if (urb->actual_length > urb->transfer_buffer_length) {
  1653. xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
  1654. urb->transfer_buffer_length, urb->actual_length);
  1655. urb->actual_length = 0;
  1656. *status = 0;
  1657. }
  1658. list_del_init(&td->td_list);
  1659. /* Was this TD slated to be cancelled but completed anyway? */
  1660. if (!list_empty(&td->cancelled_td_list))
  1661. list_del_init(&td->cancelled_td_list);
  1662. inc_td_cnt(urb);
  1663. /* Giveback the urb when all the tds are completed */
  1664. if (last_td_in_urb(td)) {
  1665. if ((urb->actual_length != urb->transfer_buffer_length &&
  1666. (urb->transfer_flags & URB_SHORT_NOT_OK)) ||
  1667. (*status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc)))
  1668. xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
  1669. urb, urb->actual_length,
  1670. urb->transfer_buffer_length, *status);
  1671. /* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */
  1672. if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
  1673. *status = 0;
  1674. xhci_giveback_urb_in_irq(xhci, td, *status);
  1675. }
  1676. return 0;
  1677. }
  1678. static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
  1679. struct xhci_transfer_event *event,
  1680. struct xhci_virt_ep *ep, int *status)
  1681. {
  1682. struct xhci_virt_device *xdev;
  1683. struct xhci_ep_ctx *ep_ctx;
  1684. struct xhci_ring *ep_ring;
  1685. unsigned int slot_id;
  1686. u32 trb_comp_code;
  1687. int ep_index;
  1688. slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
  1689. xdev = xhci->devs[slot_id];
  1690. ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
  1691. ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
  1692. ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
  1693. trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
  1694. if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
  1695. trb_comp_code == COMP_STOPPED ||
  1696. trb_comp_code == COMP_STOPPED_SHORT_PACKET) {
  1697. /* The Endpoint Stop Command completion will take care of any
  1698. * stopped TDs. A stopped TD may be restarted, so don't update
  1699. * the ring dequeue pointer or take this TD off any lists yet.
  1700. */
  1701. return 0;
  1702. }
  1703. if (trb_comp_code == COMP_STALL_ERROR ||
  1704. xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
  1705. trb_comp_code)) {
  1706. /* Issue a reset endpoint command to clear the host side
  1707. * halt, followed by a set dequeue command to move the
  1708. * dequeue pointer past the TD.
  1709. * The class driver clears the device side halt later.
  1710. */
  1711. xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
  1712. ep_ring->stream_id, td, EP_HARD_RESET);
  1713. } else {
  1714. /* Update ring dequeue pointer */
  1715. while (ep_ring->dequeue != td->last_trb)
  1716. inc_deq(xhci, ep_ring);
  1717. inc_deq(xhci, ep_ring);
  1718. }
  1719. return xhci_td_cleanup(xhci, td, ep_ring, status);
  1720. }
  1721. /* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */
  1722. static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring,
  1723. union xhci_trb *stop_trb)
  1724. {
  1725. u32 sum;
  1726. union xhci_trb *trb = ring->dequeue;
  1727. struct xhci_segment *seg = ring->deq_seg;
  1728. for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) {
  1729. if (!trb_is_noop(trb) && !trb_is_link(trb))
  1730. sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
  1731. }
  1732. return sum;
  1733. }
  1734. /*
  1735. * Process control tds, update urb status and actual_length.
  1736. */
  1737. static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
  1738. union xhci_trb *ep_trb, struct xhci_transfer_event *event,
  1739. struct xhci_virt_ep *ep, int *status)
  1740. {
  1741. struct xhci_virt_device *xdev;
  1742. unsigned int slot_id;
  1743. int ep_index;
  1744. struct xhci_ep_ctx *ep_ctx;
  1745. u32 trb_comp_code;
  1746. u32 remaining, requested;
  1747. u32 trb_type;
  1748. trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3]));
  1749. slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
  1750. xdev = xhci->devs[slot_id];
  1751. ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
  1752. ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
  1753. trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
  1754. requested = td->urb->transfer_buffer_length;
  1755. remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
  1756. switch (trb_comp_code) {
  1757. case COMP_SUCCESS:
  1758. if (trb_type != TRB_STATUS) {
  1759. xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
  1760. (trb_type == TRB_DATA) ? "data" : "setup");
  1761. *status = -ESHUTDOWN;
  1762. break;
  1763. }
  1764. *status = 0;
  1765. break;
  1766. case COMP_SHORT_PACKET:
  1767. *status = 0;
  1768. break;
  1769. case COMP_STOPPED_SHORT_PACKET:
  1770. if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
  1771. td->urb->actual_length = remaining;
  1772. else
  1773. xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
  1774. goto finish_td;
  1775. case COMP_STOPPED:
  1776. switch (trb_type) {
  1777. case TRB_SETUP:
  1778. td->urb->actual_length = 0;
  1779. goto finish_td;
  1780. case TRB_DATA:
  1781. case TRB_NORMAL:
  1782. td->urb->actual_length = requested - remaining;
  1783. goto finish_td;
  1784. case TRB_STATUS:
  1785. td->urb->actual_length = requested;
  1786. goto finish_td;
  1787. default:
  1788. xhci_warn(xhci, "WARN: unexpected TRB Type %d\n",
  1789. trb_type);
  1790. goto finish_td;
  1791. }
  1792. case COMP_STOPPED_LENGTH_INVALID:
  1793. goto finish_td;
  1794. default:
  1795. if (!xhci_requires_manual_halt_cleanup(xhci,
  1796. ep_ctx, trb_comp_code))
  1797. break;
  1798. xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
  1799. trb_comp_code, ep_index);
  1800. /* else fall through */
  1801. case COMP_STALL_ERROR:
  1802. /* Did we transfer part of the data (middle) phase? */
  1803. if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
  1804. td->urb->actual_length = requested - remaining;
  1805. else if (!td->urb_length_set)
  1806. td->urb->actual_length = 0;
  1807. goto finish_td;
  1808. }
  1809. /* stopped at setup stage, no data transferred */
  1810. if (trb_type == TRB_SETUP)
  1811. goto finish_td;
  1812. /*
  1813. * if on data stage then update the actual_length of the URB and flag it
  1814. * as set, so it won't be overwritten in the event for the last TRB.
  1815. */
  1816. if (trb_type == TRB_DATA ||
  1817. trb_type == TRB_NORMAL) {
  1818. td->urb_length_set = true;
  1819. td->urb->actual_length = requested - remaining;
  1820. xhci_dbg(xhci, "Waiting for status stage event\n");
  1821. return 0;
  1822. }
  1823. /* at status stage */
  1824. if (!td->urb_length_set)
  1825. td->urb->actual_length = requested;
  1826. finish_td:
  1827. return finish_td(xhci, td, event, ep, status);
  1828. }
  1829. /*
  1830. * Process isochronous tds, update urb packet status and actual_length.
  1831. */
  1832. static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
  1833. union xhci_trb *ep_trb, struct xhci_transfer_event *event,
  1834. struct xhci_virt_ep *ep, int *status)
  1835. {
  1836. struct xhci_ring *ep_ring;
  1837. struct urb_priv *urb_priv;
  1838. int idx;
  1839. struct usb_iso_packet_descriptor *frame;
  1840. u32 trb_comp_code;
  1841. bool sum_trbs_for_length = false;
  1842. u32 remaining, requested, ep_trb_len;
  1843. int short_framestatus;
  1844. ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
  1845. trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
  1846. urb_priv = td->urb->hcpriv;
  1847. idx = urb_priv->num_tds_done;
  1848. frame = &td->urb->iso_frame_desc[idx];
  1849. requested = frame->length;
  1850. remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
  1851. ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
  1852. short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
  1853. -EREMOTEIO : 0;
  1854. /* handle completion code */
  1855. switch (trb_comp_code) {
  1856. case COMP_SUCCESS:
  1857. if (remaining) {
  1858. frame->status = short_framestatus;
  1859. if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
  1860. sum_trbs_for_length = true;
  1861. break;
  1862. }
  1863. frame->status = 0;
  1864. break;
  1865. case COMP_SHORT_PACKET:
  1866. frame->status = short_framestatus;
  1867. sum_trbs_for_length = true;
  1868. break;
  1869. case COMP_BANDWIDTH_OVERRUN_ERROR:
  1870. frame->status = -ECOMM;
  1871. break;
  1872. case COMP_ISOCH_BUFFER_OVERRUN:
  1873. case COMP_BABBLE_DETECTED_ERROR:
  1874. frame->status = -EOVERFLOW;
  1875. break;
  1876. case COMP_INCOMPATIBLE_DEVICE_ERROR:
  1877. case COMP_STALL_ERROR:
  1878. frame->status = -EPROTO;
  1879. break;
  1880. case COMP_USB_TRANSACTION_ERROR:
  1881. frame->status = -EPROTO;
  1882. if (ep_trb != td->last_trb)
  1883. return 0;
  1884. break;
  1885. case COMP_STOPPED:
  1886. sum_trbs_for_length = true;
  1887. break;
  1888. case COMP_STOPPED_SHORT_PACKET:
  1889. /* field normally containing residue now contains tranferred */
  1890. frame->status = short_framestatus;
  1891. requested = remaining;
  1892. break;
  1893. case COMP_STOPPED_LENGTH_INVALID:
  1894. requested = 0;
  1895. remaining = 0;
  1896. break;
  1897. default:
  1898. sum_trbs_for_length = true;
  1899. frame->status = -1;
  1900. break;
  1901. }
  1902. if (sum_trbs_for_length)
  1903. frame->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb) +
  1904. ep_trb_len - remaining;
  1905. else
  1906. frame->actual_length = requested;
  1907. td->urb->actual_length += frame->actual_length;
  1908. return finish_td(xhci, td, event, ep, status);
  1909. }
  1910. static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
  1911. struct xhci_transfer_event *event,
  1912. struct xhci_virt_ep *ep, int *status)
  1913. {
  1914. struct xhci_ring *ep_ring;
  1915. struct urb_priv *urb_priv;
  1916. struct usb_iso_packet_descriptor *frame;
  1917. int idx;
  1918. ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
  1919. urb_priv = td->urb->hcpriv;
  1920. idx = urb_priv->num_tds_done;
  1921. frame = &td->urb->iso_frame_desc[idx];
  1922. /* The transfer is partly done. */
  1923. frame->status = -EXDEV;
  1924. /* calc actual length */
  1925. frame->actual_length = 0;
  1926. /* Update ring dequeue pointer */
  1927. while (ep_ring->dequeue != td->last_trb)
  1928. inc_deq(xhci, ep_ring);
  1929. inc_deq(xhci, ep_ring);
  1930. return xhci_td_cleanup(xhci, td, ep_ring, status);
  1931. }
  1932. /*
  1933. * Process bulk and interrupt tds, update urb status and actual_length.
  1934. */
  1935. static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
  1936. union xhci_trb *ep_trb, struct xhci_transfer_event *event,
  1937. struct xhci_virt_ep *ep, int *status)
  1938. {
  1939. struct xhci_ring *ep_ring;
  1940. u32 trb_comp_code;
  1941. u32 remaining, requested, ep_trb_len;
  1942. ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
  1943. trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
  1944. remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
  1945. ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
  1946. requested = td->urb->transfer_buffer_length;
  1947. switch (trb_comp_code) {
  1948. case COMP_SUCCESS:
  1949. /* handle success with untransferred data as short packet */
  1950. if (ep_trb != td->last_trb || remaining) {
  1951. xhci_warn(xhci, "WARN Successful completion on short TX\n");
  1952. xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
  1953. td->urb->ep->desc.bEndpointAddress,
  1954. requested, remaining);
  1955. }
  1956. *status = 0;
  1957. break;
  1958. case COMP_SHORT_PACKET:
  1959. xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
  1960. td->urb->ep->desc.bEndpointAddress,
  1961. requested, remaining);
  1962. *status = 0;
  1963. break;
  1964. case COMP_STOPPED_SHORT_PACKET:
  1965. td->urb->actual_length = remaining;
  1966. goto finish_td;
  1967. case COMP_STOPPED_LENGTH_INVALID:
  1968. /* stopped on ep trb with invalid length, exclude it */
  1969. ep_trb_len = 0;
  1970. remaining = 0;
  1971. break;
  1972. default:
  1973. /* do nothing */
  1974. break;
  1975. }
  1976. if (ep_trb == td->last_trb)
  1977. td->urb->actual_length = requested - remaining;
  1978. else
  1979. td->urb->actual_length =
  1980. sum_trb_lengths(xhci, ep_ring, ep_trb) +
  1981. ep_trb_len - remaining;
  1982. finish_td:
  1983. if (remaining > requested) {
  1984. xhci_warn(xhci, "bad transfer trb length %d in event trb\n",
  1985. remaining);
  1986. td->urb->actual_length = 0;
  1987. }
  1988. return finish_td(xhci, td, event, ep, status);
  1989. }
  1990. /*
  1991. * If this function returns an error condition, it means it got a Transfer
  1992. * event with a corrupted Slot ID, Endpoint ID, or TRB DMA address.
  1993. * At this point, the host controller is probably hosed and should be reset.
  1994. */
  1995. static int handle_tx_event(struct xhci_hcd *xhci,
  1996. struct xhci_transfer_event *event)
  1997. {
  1998. struct xhci_virt_device *xdev;
  1999. struct xhci_virt_ep *ep;
  2000. struct xhci_ring *ep_ring;
  2001. unsigned int slot_id;
  2002. int ep_index;
  2003. struct xhci_td *td = NULL;
  2004. dma_addr_t ep_trb_dma;
  2005. struct xhci_segment *ep_seg;
  2006. union xhci_trb *ep_trb;
  2007. int status = -EINPROGRESS;
  2008. struct xhci_ep_ctx *ep_ctx;
  2009. struct list_head *tmp;
  2010. u32 trb_comp_code;
  2011. int td_num = 0;
  2012. bool handling_skipped_tds = false;
  2013. slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
  2014. ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
  2015. trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
  2016. ep_trb_dma = le64_to_cpu(event->buffer);
  2017. xdev = xhci->devs[slot_id];
  2018. if (!xdev) {
  2019. xhci_err(xhci, "ERROR Transfer event pointed to bad slot %u\n",
  2020. slot_id);
  2021. goto err_out;
  2022. }
  2023. ep = &xdev->eps[ep_index];
  2024. ep_ring = xhci_dma_to_transfer_ring(ep, ep_trb_dma);
  2025. ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
  2026. if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) {
  2027. xhci_err(xhci,
  2028. "ERROR Transfer event for disabled endpoint slot %u ep %u\n",
  2029. slot_id, ep_index);
  2030. goto err_out;
  2031. }
  2032. /* Some transfer events don't always point to a trb, see xhci 4.17.4 */
  2033. if (!ep_ring) {
  2034. switch (trb_comp_code) {
  2035. case COMP_STALL_ERROR:
  2036. case COMP_USB_TRANSACTION_ERROR:
  2037. case COMP_INVALID_STREAM_TYPE_ERROR:
  2038. case COMP_INVALID_STREAM_ID_ERROR:
  2039. xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index, 0,
  2040. NULL, EP_SOFT_RESET);
  2041. goto cleanup;
  2042. case COMP_RING_UNDERRUN:
  2043. case COMP_RING_OVERRUN:
  2044. case COMP_STOPPED_LENGTH_INVALID:
  2045. goto cleanup;
  2046. default:
  2047. xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n",
  2048. slot_id, ep_index);
  2049. goto err_out;
  2050. }
  2051. }
  2052. /* Count current td numbers if ep->skip is set */
  2053. if (ep->skip) {
  2054. list_for_each(tmp, &ep_ring->td_list)
  2055. td_num++;
  2056. }
  2057. /* Look for common error cases */
  2058. switch (trb_comp_code) {
  2059. /* Skip codes that require special handling depending on
  2060. * transfer type
  2061. */
  2062. case COMP_SUCCESS:
  2063. if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0)
  2064. break;
  2065. if (xhci->quirks & XHCI_TRUST_TX_LENGTH ||
  2066. ep_ring->last_td_was_short)
  2067. trb_comp_code = COMP_SHORT_PACKET;
  2068. else
  2069. xhci_warn_ratelimited(xhci,
  2070. "WARN Successful completion on short TX for slot %u ep %u: needs XHCI_TRUST_TX_LENGTH quirk?\n",
  2071. slot_id, ep_index);
  2072. case COMP_SHORT_PACKET:
  2073. break;
  2074. /* Completion codes for endpoint stopped state */
  2075. case COMP_STOPPED:
  2076. xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n",
  2077. slot_id, ep_index);
  2078. break;
  2079. case COMP_STOPPED_LENGTH_INVALID:
  2080. xhci_dbg(xhci,
  2081. "Stopped on No-op or Link TRB for slot %u ep %u\n",
  2082. slot_id, ep_index);
  2083. break;
  2084. case COMP_STOPPED_SHORT_PACKET:
  2085. xhci_dbg(xhci,
  2086. "Stopped with short packet transfer detected for slot %u ep %u\n",
  2087. slot_id, ep_index);
  2088. break;
  2089. /* Completion codes for endpoint halted state */
  2090. case COMP_STALL_ERROR:
  2091. xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id,
  2092. ep_index);
  2093. ep->ep_state |= EP_HALTED;
  2094. status = -EPIPE;
  2095. break;
  2096. case COMP_SPLIT_TRANSACTION_ERROR:
  2097. case COMP_USB_TRANSACTION_ERROR:
  2098. xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n",
  2099. slot_id, ep_index);
  2100. status = -EPROTO;
  2101. break;
  2102. case COMP_BABBLE_DETECTED_ERROR:
  2103. xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n",
  2104. slot_id, ep_index);
  2105. status = -EOVERFLOW;
  2106. break;
  2107. /* Completion codes for endpoint error state */
  2108. case COMP_TRB_ERROR:
  2109. xhci_warn(xhci,
  2110. "WARN: TRB error for slot %u ep %u on endpoint\n",
  2111. slot_id, ep_index);
  2112. status = -EILSEQ;
  2113. break;
  2114. /* completion codes not indicating endpoint state change */
  2115. case COMP_DATA_BUFFER_ERROR:
  2116. xhci_warn(xhci,
  2117. "WARN: HC couldn't access mem fast enough for slot %u ep %u\n",
  2118. slot_id, ep_index);
  2119. status = -ENOSR;
  2120. break;
  2121. case COMP_BANDWIDTH_OVERRUN_ERROR:
  2122. xhci_warn(xhci,
  2123. "WARN: bandwidth overrun event for slot %u ep %u on endpoint\n",
  2124. slot_id, ep_index);
  2125. break;
  2126. case COMP_ISOCH_BUFFER_OVERRUN:
  2127. xhci_warn(xhci,
  2128. "WARN: buffer overrun event for slot %u ep %u on endpoint",
  2129. slot_id, ep_index);
  2130. break;
  2131. case COMP_RING_UNDERRUN:
  2132. /*
  2133. * When the Isoch ring is empty, the xHC will generate
  2134. * a Ring Overrun Event for IN Isoch endpoint or Ring
  2135. * Underrun Event for OUT Isoch endpoint.
  2136. */
  2137. xhci_dbg(xhci, "underrun event on endpoint\n");
  2138. if (!list_empty(&ep_ring->td_list))
  2139. xhci_dbg(xhci, "Underrun Event for slot %d ep %d "
  2140. "still with TDs queued?\n",
  2141. TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
  2142. ep_index);
  2143. goto cleanup;
  2144. case COMP_RING_OVERRUN:
  2145. xhci_dbg(xhci, "overrun event on endpoint\n");
  2146. if (!list_empty(&ep_ring->td_list))
  2147. xhci_dbg(xhci, "Overrun Event for slot %d ep %d "
  2148. "still with TDs queued?\n",
  2149. TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
  2150. ep_index);
  2151. goto cleanup;
  2152. case COMP_MISSED_SERVICE_ERROR:
  2153. /*
  2154. * When encounter missed service error, one or more isoc tds
  2155. * may be missed by xHC.
  2156. * Set skip flag of the ep_ring; Complete the missed tds as
  2157. * short transfer when process the ep_ring next time.
  2158. */
  2159. ep->skip = true;
  2160. xhci_dbg(xhci,
  2161. "Miss service interval error for slot %u ep %u, set skip flag\n",
  2162. slot_id, ep_index);
  2163. goto cleanup;
  2164. case COMP_NO_PING_RESPONSE_ERROR:
  2165. ep->skip = true;
  2166. xhci_dbg(xhci,
  2167. "No Ping response error for slot %u ep %u, Skip one Isoc TD\n",
  2168. slot_id, ep_index);
  2169. goto cleanup;
  2170. case COMP_INCOMPATIBLE_DEVICE_ERROR:
  2171. /* needs disable slot command to recover */
  2172. xhci_warn(xhci,
  2173. "WARN: detect an incompatible device for slot %u ep %u",
  2174. slot_id, ep_index);
  2175. status = -EPROTO;
  2176. break;
  2177. default:
  2178. if (xhci_is_vendor_info_code(xhci, trb_comp_code)) {
  2179. status = 0;
  2180. break;
  2181. }
  2182. xhci_warn(xhci,
  2183. "ERROR Unknown event condition %u for slot %u ep %u , HC probably busted\n",
  2184. trb_comp_code, slot_id, ep_index);
  2185. goto cleanup;
  2186. }
  2187. do {
  2188. /* This TRB should be in the TD at the head of this ring's
  2189. * TD list.
  2190. */
  2191. if (list_empty(&ep_ring->td_list)) {
  2192. /*
  2193. * Don't print wanings if it's due to a stopped endpoint
  2194. * generating an extra completion event if the device
  2195. * was suspended. Or, a event for the last TRB of a
  2196. * short TD we already got a short event for.
  2197. * The short TD is already removed from the TD list.
  2198. */
  2199. if (!(trb_comp_code == COMP_STOPPED ||
  2200. trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
  2201. ep_ring->last_td_was_short)) {
  2202. xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n",
  2203. TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
  2204. ep_index);
  2205. }
  2206. if (ep->skip) {
  2207. ep->skip = false;
  2208. xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n",
  2209. slot_id, ep_index);
  2210. }
  2211. goto cleanup;
  2212. }
  2213. /* We've skipped all the TDs on the ep ring when ep->skip set */
  2214. if (ep->skip && td_num == 0) {
  2215. ep->skip = false;
  2216. xhci_dbg(xhci, "All tds on the ep_ring skipped. Clear skip flag for slot %u ep %u.\n",
  2217. slot_id, ep_index);
  2218. goto cleanup;
  2219. }
  2220. td = list_first_entry(&ep_ring->td_list, struct xhci_td,
  2221. td_list);
  2222. if (ep->skip)
  2223. td_num--;
  2224. /* Is this a TRB in the currently executing TD? */
  2225. ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,
  2226. td->last_trb, ep_trb_dma, false);
  2227. /*
  2228. * Skip the Force Stopped Event. The event_trb(event_dma) of FSE
  2229. * is not in the current TD pointed by ep_ring->dequeue because
  2230. * that the hardware dequeue pointer still at the previous TRB
  2231. * of the current TD. The previous TRB maybe a Link TD or the
  2232. * last TRB of the previous TD. The command completion handle
  2233. * will take care the rest.
  2234. */
  2235. if (!ep_seg && (trb_comp_code == COMP_STOPPED ||
  2236. trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) {
  2237. goto cleanup;
  2238. }
  2239. if (!ep_seg) {
  2240. if (!ep->skip ||
  2241. !usb_endpoint_xfer_isoc(&td->urb->ep->desc)) {
  2242. /* Some host controllers give a spurious
  2243. * successful event after a short transfer.
  2244. * Ignore it.
  2245. */
  2246. if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) &&
  2247. ep_ring->last_td_was_short) {
  2248. ep_ring->last_td_was_short = false;
  2249. goto cleanup;
  2250. }
  2251. /* HC is busted, give up! */
  2252. xhci_err(xhci,
  2253. "ERROR Transfer event TRB DMA ptr not "
  2254. "part of current TD ep_index %d "
  2255. "comp_code %u\n", ep_index,
  2256. trb_comp_code);
  2257. trb_in_td(xhci, ep_ring->deq_seg,
  2258. ep_ring->dequeue, td->last_trb,
  2259. ep_trb_dma, true);
  2260. return -ESHUTDOWN;
  2261. }
  2262. skip_isoc_td(xhci, td, event, ep, &status);
  2263. goto cleanup;
  2264. }
  2265. if (trb_comp_code == COMP_SHORT_PACKET)
  2266. ep_ring->last_td_was_short = true;
  2267. else
  2268. ep_ring->last_td_was_short = false;
  2269. if (ep->skip) {
  2270. xhci_dbg(xhci,
  2271. "Found td. Clear skip flag for slot %u ep %u.\n",
  2272. slot_id, ep_index);
  2273. ep->skip = false;
  2274. }
  2275. ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) /
  2276. sizeof(*ep_trb)];
  2277. trace_xhci_handle_transfer(ep_ring,
  2278. (struct xhci_generic_trb *) ep_trb);
  2279. /*
  2280. * No-op TRB could trigger interrupts in a case where
  2281. * a URB was killed and a STALL_ERROR happens right
  2282. * after the endpoint ring stopped. Reset the halted
  2283. * endpoint. Otherwise, the endpoint remains stalled
  2284. * indefinitely.
  2285. */
  2286. if (trb_is_noop(ep_trb)) {
  2287. if (trb_comp_code == COMP_STALL_ERROR ||
  2288. xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
  2289. trb_comp_code))
  2290. xhci_cleanup_halted_endpoint(xhci, slot_id,
  2291. ep_index,
  2292. ep_ring->stream_id,
  2293. td, EP_HARD_RESET);
  2294. goto cleanup;
  2295. }
  2296. /* update the urb's actual_length and give back to the core */
  2297. if (usb_endpoint_xfer_control(&td->urb->ep->desc))
  2298. process_ctrl_td(xhci, td, ep_trb, event, ep, &status);
  2299. else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc))
  2300. process_isoc_td(xhci, td, ep_trb, event, ep, &status);
  2301. else
  2302. process_bulk_intr_td(xhci, td, ep_trb, event, ep,
  2303. &status);
  2304. cleanup:
  2305. handling_skipped_tds = ep->skip &&
  2306. trb_comp_code != COMP_MISSED_SERVICE_ERROR &&
  2307. trb_comp_code != COMP_NO_PING_RESPONSE_ERROR;
  2308. /*
  2309. * Do not update event ring dequeue pointer if we're in a loop
  2310. * processing missed tds.
  2311. */
  2312. if (!handling_skipped_tds)
  2313. inc_deq(xhci, xhci->event_ring);
  2314. /*
  2315. * If ep->skip is set, it means there are missed tds on the
  2316. * endpoint ring need to take care of.
  2317. * Process them as short transfer until reach the td pointed by
  2318. * the event.
  2319. */
  2320. } while (handling_skipped_tds);
  2321. return 0;
  2322. err_out:
  2323. xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
  2324. (unsigned long long) xhci_trb_virt_to_dma(
  2325. xhci->event_ring->deq_seg,
  2326. xhci->event_ring->dequeue),
  2327. lower_32_bits(le64_to_cpu(event->buffer)),
  2328. upper_32_bits(le64_to_cpu(event->buffer)),
  2329. le32_to_cpu(event->transfer_len),
  2330. le32_to_cpu(event->flags));
  2331. return -ENODEV;
  2332. }
  2333. /*
  2334. * This function handles all OS-owned events on the event ring. It may drop
  2335. * xhci->lock between event processing (e.g. to pass up port status changes).
  2336. * Returns >0 for "possibly more events to process" (caller should call again),
  2337. * otherwise 0 if done. In future, <0 returns should indicate error code.
  2338. */
  2339. static int xhci_handle_event(struct xhci_hcd *xhci)
  2340. {
  2341. union xhci_trb *event;
  2342. int update_ptrs = 1;
  2343. int ret;
  2344. /* Event ring hasn't been allocated yet. */
  2345. if (!xhci->event_ring || !xhci->event_ring->dequeue) {
  2346. xhci_err(xhci, "ERROR event ring not ready\n");
  2347. return -ENOMEM;
  2348. }
  2349. event = xhci->event_ring->dequeue;
  2350. /* Does the HC or OS own the TRB? */
  2351. if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
  2352. xhci->event_ring->cycle_state)
  2353. return 0;
  2354. trace_xhci_handle_event(xhci->event_ring, &event->generic);
  2355. /*
  2356. * Barrier between reading the TRB_CYCLE (valid) flag above and any
  2357. * speculative reads of the event's flags/data below.
  2358. */
  2359. rmb();
  2360. /* FIXME: Handle more event types. */
  2361. switch (le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) {
  2362. case TRB_TYPE(TRB_COMPLETION):
  2363. handle_cmd_completion(xhci, &event->event_cmd);
  2364. break;
  2365. case TRB_TYPE(TRB_PORT_STATUS):
  2366. handle_port_status(xhci, event);
  2367. update_ptrs = 0;
  2368. break;
  2369. case TRB_TYPE(TRB_TRANSFER):
  2370. ret = handle_tx_event(xhci, &event->trans_event);
  2371. if (ret >= 0)
  2372. update_ptrs = 0;
  2373. break;
  2374. case TRB_TYPE(TRB_DEV_NOTE):
  2375. handle_device_notification(xhci, event);
  2376. break;
  2377. default:
  2378. if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >=
  2379. TRB_TYPE(48))
  2380. handle_vendor_event(xhci, event);
  2381. else
  2382. xhci_warn(xhci, "ERROR unknown event type %d\n",
  2383. TRB_FIELD_TO_TYPE(
  2384. le32_to_cpu(event->event_cmd.flags)));
  2385. }
  2386. /* Any of the above functions may drop and re-acquire the lock, so check
  2387. * to make sure a watchdog timer didn't mark the host as non-responsive.
  2388. */
  2389. if (xhci->xhc_state & XHCI_STATE_DYING) {
  2390. xhci_dbg(xhci, "xHCI host dying, returning from "
  2391. "event handler.\n");
  2392. return 0;
  2393. }
  2394. if (update_ptrs)
  2395. /* Update SW event ring dequeue pointer */
  2396. inc_deq(xhci, xhci->event_ring);
  2397. /* Are there more items on the event ring? Caller will call us again to
  2398. * check.
  2399. */
  2400. return 1;
  2401. }
  2402. /*
  2403. * Update Event Ring Dequeue Pointer:
  2404. * - When all events have finished
  2405. * - To avoid "Event Ring Full Error" condition
  2406. */
  2407. static void xhci_update_erst_dequeue(struct xhci_hcd *xhci,
  2408. union xhci_trb *event_ring_deq)
  2409. {
  2410. u64 temp_64;
  2411. dma_addr_t deq;
  2412. temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
  2413. /* If necessary, update the HW's version of the event ring deq ptr. */
  2414. if (event_ring_deq != xhci->event_ring->dequeue) {
  2415. deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
  2416. xhci->event_ring->dequeue);
  2417. if (deq == 0)
  2418. xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n");
  2419. /*
  2420. * Per 4.9.4, Software writes to the ERDP register shall
  2421. * always advance the Event Ring Dequeue Pointer value.
  2422. */
  2423. if ((temp_64 & (u64) ~ERST_PTR_MASK) ==
  2424. ((u64) deq & (u64) ~ERST_PTR_MASK))
  2425. return;
  2426. /* Update HC event ring dequeue pointer */
  2427. temp_64 &= ERST_PTR_MASK;
  2428. temp_64 |= ((u64) deq & (u64) ~ERST_PTR_MASK);
  2429. }
  2430. /* Clear the event handler busy flag (RW1C) */
  2431. temp_64 |= ERST_EHB;
  2432. xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue);
  2433. }
  2434. /*
  2435. * xHCI spec says we can get an interrupt, and if the HC has an error condition,
  2436. * we might get bad data out of the event ring. Section 4.10.2.7 has a list of
  2437. * indicators of an event TRB error, but we check the status *first* to be safe.
  2438. */
  2439. irqreturn_t xhci_irq(struct usb_hcd *hcd)
  2440. {
  2441. struct xhci_hcd *xhci = hcd_to_xhci(hcd);
  2442. union xhci_trb *event_ring_deq;
  2443. irqreturn_t ret = IRQ_NONE;
  2444. unsigned long flags;
  2445. u64 temp_64;
  2446. u32 status;
  2447. int event_loop = 0;
  2448. spin_lock_irqsave(&xhci->lock, flags);
  2449. /* Check if the xHC generated the interrupt, or the irq is shared */
  2450. status = readl(&xhci->op_regs->status);
  2451. if (status == ~(u32)0) {
  2452. xhci_hc_died(xhci);
  2453. ret = IRQ_HANDLED;
  2454. goto out;
  2455. }
  2456. if (!(status & STS_EINT))
  2457. goto out;
  2458. if (status & STS_FATAL) {
  2459. xhci_warn(xhci, "WARNING: Host System Error\n");
  2460. xhci_halt(xhci);
  2461. ret = IRQ_HANDLED;
  2462. goto out;
  2463. }
  2464. /*
  2465. * Clear the op reg interrupt status first,
  2466. * so we can receive interrupts from other MSI-X interrupters.
  2467. * Write 1 to clear the interrupt status.
  2468. */
  2469. status |= STS_EINT;
  2470. writel(status, &xhci->op_regs->status);
  2471. if (!hcd->msi_enabled) {
  2472. u32 irq_pending;
  2473. irq_pending = readl(&xhci->ir_set->irq_pending);
  2474. irq_pending |= IMAN_IP;
  2475. writel(irq_pending, &xhci->ir_set->irq_pending);
  2476. }
  2477. if (xhci->xhc_state & XHCI_STATE_DYING ||
  2478. xhci->xhc_state & XHCI_STATE_HALTED) {
  2479. xhci_dbg(xhci, "xHCI dying, ignoring interrupt. "
  2480. "Shouldn't IRQs be disabled?\n");
  2481. /* Clear the event handler busy flag (RW1C);
  2482. * the event ring should be empty.
  2483. */
  2484. temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
  2485. xhci_write_64(xhci, temp_64 | ERST_EHB,
  2486. &xhci->ir_set->erst_dequeue);
  2487. ret = IRQ_HANDLED;
  2488. goto out;
  2489. }
  2490. event_ring_deq = xhci->event_ring->dequeue;
  2491. /* FIXME this should be a delayed service routine
  2492. * that clears the EHB.
  2493. */
  2494. while (xhci_handle_event(xhci) > 0) {
  2495. if (event_loop++ < TRBS_PER_SEGMENT / 2)
  2496. continue;
  2497. xhci_update_erst_dequeue(xhci, event_ring_deq);
  2498. event_loop = 0;
  2499. }
  2500. xhci_update_erst_dequeue(xhci, event_ring_deq);
  2501. ret = IRQ_HANDLED;
  2502. out:
  2503. spin_unlock_irqrestore(&xhci->lock, flags);
  2504. return ret;
  2505. }
  2506. irqreturn_t xhci_msi_irq(int irq, void *hcd)
  2507. {
  2508. return xhci_irq(hcd);
  2509. }
  2510. /**** Endpoint Ring Operations ****/
  2511. /*
  2512. * Generic function for queueing a TRB on a ring.
  2513. * The caller must have checked to make sure there's room on the ring.
  2514. *
  2515. * @more_trbs_coming: Will you enqueue more TRBs before calling
  2516. * prepare_transfer()?
  2517. */
  2518. static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
  2519. bool more_trbs_coming,
  2520. u32 field1, u32 field2, u32 field3, u32 field4)
  2521. {
  2522. struct xhci_generic_trb *trb;
  2523. trb = &ring->enqueue->generic;
  2524. trb->field[0] = cpu_to_le32(field1);
  2525. trb->field[1] = cpu_to_le32(field2);
  2526. trb->field[2] = cpu_to_le32(field3);
  2527. /* make sure TRB is fully written before giving it to the controller */
  2528. wmb();
  2529. trb->field[3] = cpu_to_le32(field4);
  2530. trace_xhci_queue_trb(ring, trb);
  2531. inc_enq(xhci, ring, more_trbs_coming);
  2532. }
  2533. /*
  2534. * Does various checks on the endpoint ring, and makes it ready to queue num_trbs.
  2535. * FIXME allocate segments if the ring is full.
  2536. */
  2537. static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
  2538. u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
  2539. {
  2540. unsigned int num_trbs_needed;
  2541. /* Make sure the endpoint has been added to xHC schedule */
  2542. switch (ep_state) {
  2543. case EP_STATE_DISABLED:
  2544. /*
  2545. * USB core changed config/interfaces without notifying us,
  2546. * or hardware is reporting the wrong state.
  2547. */
  2548. xhci_warn(xhci, "WARN urb submitted to disabled ep\n");
  2549. return -ENOENT;
  2550. case EP_STATE_ERROR:
  2551. xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n");
  2552. /* FIXME event handling code for error needs to clear it */
  2553. /* XXX not sure if this should be -ENOENT or not */
  2554. return -EINVAL;
  2555. case EP_STATE_HALTED:
  2556. xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n");
  2557. case EP_STATE_STOPPED:
  2558. case EP_STATE_RUNNING:
  2559. break;
  2560. default:
  2561. xhci_err(xhci, "ERROR unknown endpoint state for ep\n");
  2562. /*
  2563. * FIXME issue Configure Endpoint command to try to get the HC
  2564. * back into a known state.
  2565. */
  2566. return -EINVAL;
  2567. }
  2568. while (1) {
  2569. if (room_on_ring(xhci, ep_ring, num_trbs))
  2570. break;
  2571. if (ep_ring == xhci->cmd_ring) {
  2572. xhci_err(xhci, "Do not support expand command ring\n");
  2573. return -ENOMEM;
  2574. }
  2575. xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion,
  2576. "ERROR no room on ep ring, try ring expansion");
  2577. num_trbs_needed = num_trbs - ep_ring->num_trbs_free;
  2578. if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed,
  2579. mem_flags)) {
  2580. xhci_err(xhci, "Ring expansion failed\n");
  2581. return -ENOMEM;
  2582. }
  2583. }
  2584. while (trb_is_link(ep_ring->enqueue)) {
  2585. /* If we're not dealing with 0.95 hardware or isoc rings
  2586. * on AMD 0.96 host, clear the chain bit.
  2587. */
  2588. if (!xhci_link_trb_quirk(xhci) &&
  2589. !(ep_ring->type == TYPE_ISOC &&
  2590. (xhci->quirks & XHCI_AMD_0x96_HOST)))
  2591. ep_ring->enqueue->link.control &=
  2592. cpu_to_le32(~TRB_CHAIN);
  2593. else
  2594. ep_ring->enqueue->link.control |=
  2595. cpu_to_le32(TRB_CHAIN);
  2596. wmb();
  2597. ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);
  2598. /* Toggle the cycle bit after the last ring segment. */
  2599. if (link_trb_toggles_cycle(ep_ring->enqueue))
  2600. ep_ring->cycle_state ^= 1;
  2601. ep_ring->enq_seg = ep_ring->enq_seg->next;
  2602. ep_ring->enqueue = ep_ring->enq_seg->trbs;
  2603. }
  2604. return 0;
  2605. }
  2606. static int prepare_transfer(struct xhci_hcd *xhci,
  2607. struct xhci_virt_device *xdev,
  2608. unsigned int ep_index,
  2609. unsigned int stream_id,
  2610. unsigned int num_trbs,
  2611. struct urb *urb,
  2612. unsigned int td_index,
  2613. gfp_t mem_flags)
  2614. {
  2615. int ret;
  2616. struct urb_priv *urb_priv;
  2617. struct xhci_td *td;
  2618. struct xhci_ring *ep_ring;
  2619. struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
  2620. ep_ring = xhci_stream_id_to_ring(xdev, ep_index, stream_id);
  2621. if (!ep_ring) {
  2622. xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n",
  2623. stream_id);
  2624. return -EINVAL;
  2625. }
  2626. ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
  2627. num_trbs, mem_flags);
  2628. if (ret)
  2629. return ret;
  2630. urb_priv = urb->hcpriv;
  2631. td = &urb_priv->td[td_index];
  2632. INIT_LIST_HEAD(&td->td_list);
  2633. INIT_LIST_HEAD(&td->cancelled_td_list);
  2634. if (td_index == 0) {
  2635. ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb);
  2636. if (unlikely(ret))
  2637. return ret;
  2638. }
  2639. td->urb = urb;
  2640. /* Add this TD to the tail of the endpoint ring's TD list */
  2641. list_add_tail(&td->td_list, &ep_ring->td_list);
  2642. td->start_seg = ep_ring->enq_seg;
  2643. td->first_trb = ep_ring->enqueue;
  2644. return 0;
  2645. }
  2646. unsigned int count_trbs(u64 addr, u64 len)
  2647. {
  2648. unsigned int num_trbs;
  2649. num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)),
  2650. TRB_MAX_BUFF_SIZE);
  2651. if (num_trbs == 0)
  2652. num_trbs++;
  2653. return num_trbs;
  2654. }
  2655. static inline unsigned int count_trbs_needed(struct urb *urb)
  2656. {
  2657. return count_trbs(urb->transfer_dma, urb->transfer_buffer_length);
  2658. }
  2659. static unsigned int count_sg_trbs_needed(struct urb *urb)
  2660. {
  2661. struct scatterlist *sg;
  2662. unsigned int i, len, full_len, num_trbs = 0;
  2663. full_len = urb->transfer_buffer_length;
  2664. for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) {
  2665. len = sg_dma_len(sg);
  2666. num_trbs += count_trbs(sg_dma_address(sg), len);
  2667. len = min_t(unsigned int, len, full_len);
  2668. full_len -= len;
  2669. if (full_len == 0)
  2670. break;
  2671. }
  2672. return num_trbs;
  2673. }
  2674. static unsigned int count_isoc_trbs_needed(struct urb *urb, int i)
  2675. {
  2676. u64 addr, len;
  2677. addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset);
  2678. len = urb->iso_frame_desc[i].length;
  2679. return count_trbs(addr, len);
  2680. }
  2681. static void check_trb_math(struct urb *urb, int running_total)
  2682. {
  2683. if (unlikely(running_total != urb->transfer_buffer_length))
  2684. dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, "
  2685. "queued %#x (%d), asked for %#x (%d)\n",
  2686. __func__,
  2687. urb->ep->desc.bEndpointAddress,
  2688. running_total, running_total,
  2689. urb->transfer_buffer_length,
  2690. urb->transfer_buffer_length);
  2691. }
  2692. static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id,
  2693. unsigned int ep_index, unsigned int stream_id, int start_cycle,
  2694. struct xhci_generic_trb *start_trb)
  2695. {
  2696. /*
  2697. * Pass all the TRBs to the hardware at once and make sure this write
  2698. * isn't reordered.
  2699. */
  2700. wmb();
  2701. if (start_cycle)
  2702. start_trb->field[3] |= cpu_to_le32(start_cycle);
  2703. else
  2704. start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
  2705. xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id);
  2706. }
  2707. static void check_interval(struct xhci_hcd *xhci, struct urb *urb,
  2708. struct xhci_ep_ctx *ep_ctx)
  2709. {
  2710. int xhci_interval;
  2711. int ep_interval;
  2712. xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info));
  2713. ep_interval = urb->interval;
  2714. /* Convert to microframes */
  2715. if (urb->dev->speed == USB_SPEED_LOW ||
  2716. urb->dev->speed == USB_SPEED_FULL)
  2717. ep_interval *= 8;
  2718. /* FIXME change this to a warning and a suggestion to use the new API
  2719. * to set the polling interval (once the API is added).
  2720. */
  2721. if (xhci_interval != ep_interval) {
  2722. dev_dbg_ratelimited(&urb->dev->dev,
  2723. "Driver uses different interval (%d microframe%s) than xHCI (%d microframe%s)\n",
  2724. ep_interval, ep_interval == 1 ? "" : "s",
  2725. xhci_interval, xhci_interval == 1 ? "" : "s");
  2726. urb->interval = xhci_interval;
  2727. /* Convert back to frames for LS/FS devices */
  2728. if (urb->dev->speed == USB_SPEED_LOW ||
  2729. urb->dev->speed == USB_SPEED_FULL)
  2730. urb->interval /= 8;
  2731. }
  2732. }
  2733. /*
  2734. * xHCI uses normal TRBs for both bulk and interrupt. When the interrupt
  2735. * endpoint is to be serviced, the xHC will consume (at most) one TD. A TD
  2736. * (comprised of sg list entries) can take several service intervals to
  2737. * transmit.
  2738. */
  2739. int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
  2740. struct urb *urb, int slot_id, unsigned int ep_index)
  2741. {
  2742. struct xhci_ep_ctx *ep_ctx;
  2743. ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index);
  2744. check_interval(xhci, urb, ep_ctx);
  2745. return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index);
  2746. }
  2747. /*
  2748. * For xHCI 1.0 host controllers, TD size is the number of max packet sized
  2749. * packets remaining in the TD (*not* including this TRB).
  2750. *
  2751. * Total TD packet count = total_packet_count =
  2752. * DIV_ROUND_UP(TD size in bytes / wMaxPacketSize)
  2753. *
  2754. * Packets transferred up to and including this TRB = packets_transferred =
  2755. * rounddown(total bytes transferred including this TRB / wMaxPacketSize)
  2756. *
  2757. * TD size = total_packet_count - packets_transferred
  2758. *
  2759. * For xHCI 0.96 and older, TD size field should be the remaining bytes
  2760. * including this TRB, right shifted by 10
  2761. *
  2762. * For all hosts it must fit in bits 21:17, so it can't be bigger than 31.
  2763. * This is taken care of in the TRB_TD_SIZE() macro
  2764. *
  2765. * The last TRB in a TD must have the TD size set to zero.
  2766. */
  2767. static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred,
  2768. int trb_buff_len, unsigned int td_total_len,
  2769. struct urb *urb, bool more_trbs_coming)
  2770. {
  2771. u32 maxp, total_packet_count;
  2772. /* MTK xHCI 0.96 contains some features from 1.0 */
  2773. if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST))
  2774. return ((td_total_len - transferred) >> 10);
  2775. /* One TRB with a zero-length data packet. */
  2776. if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
  2777. trb_buff_len == td_total_len)
  2778. return 0;
  2779. /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */
  2780. if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100))
  2781. trb_buff_len = 0;
  2782. maxp = usb_endpoint_maxp(&urb->ep->desc);
  2783. total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
  2784. /* Queueing functions don't count the current TRB into transferred */
  2785. return (total_packet_count - ((transferred + trb_buff_len) / maxp));
  2786. }
  2787. static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
  2788. u32 *trb_buff_len, struct xhci_segment *seg)
  2789. {
  2790. struct device *dev = xhci_to_hcd(xhci)->self.controller;
  2791. unsigned int unalign;
  2792. unsigned int max_pkt;
  2793. u32 new_buff_len;
  2794. size_t len;
  2795. max_pkt = usb_endpoint_maxp(&urb->ep->desc);
  2796. unalign = (enqd_len + *trb_buff_len) % max_pkt;
  2797. /* we got lucky, last normal TRB data on segment is packet aligned */
  2798. if (unalign == 0)
  2799. return 0;
  2800. xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n",
  2801. unalign, *trb_buff_len);
  2802. /* is the last nornal TRB alignable by splitting it */
  2803. if (*trb_buff_len > unalign) {
  2804. *trb_buff_len -= unalign;
  2805. xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len);
  2806. return 0;
  2807. }
  2808. /*
  2809. * We want enqd_len + trb_buff_len to sum up to a number aligned to
  2810. * number which is divisible by the endpoint's wMaxPacketSize. IOW:
  2811. * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0.
  2812. */
  2813. new_buff_len = max_pkt - (enqd_len % max_pkt);
  2814. if (new_buff_len > (urb->transfer_buffer_length - enqd_len))
  2815. new_buff_len = (urb->transfer_buffer_length - enqd_len);
  2816. /* create a max max_pkt sized bounce buffer pointed to by last trb */
  2817. if (usb_urb_dir_out(urb)) {
  2818. if (urb->num_sgs) {
  2819. len = sg_pcopy_to_buffer(urb->sg, urb->num_sgs,
  2820. seg->bounce_buf, new_buff_len, enqd_len);
  2821. if (len != new_buff_len)
  2822. xhci_warn(xhci, "WARN Wrong bounce buffer write length: %zu != %d\n",
  2823. len, new_buff_len);
  2824. } else {
  2825. memcpy(seg->bounce_buf, urb->transfer_buffer + enqd_len, new_buff_len);
  2826. }
  2827. seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
  2828. max_pkt, DMA_TO_DEVICE);
  2829. } else {
  2830. seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
  2831. max_pkt, DMA_FROM_DEVICE);
  2832. }
  2833. if (dma_mapping_error(dev, seg->bounce_dma)) {
  2834. /* try without aligning. Some host controllers survive */
  2835. xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n");
  2836. return 0;
  2837. }
  2838. *trb_buff_len = new_buff_len;
  2839. seg->bounce_len = new_buff_len;
  2840. seg->bounce_offs = enqd_len;
  2841. xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len);
  2842. return 1;
  2843. }
  2844. /* This is very similar to what ehci-q.c qtd_fill() does */
  2845. int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
  2846. struct urb *urb, int slot_id, unsigned int ep_index)
  2847. {
  2848. struct xhci_ring *ring;
  2849. struct urb_priv *urb_priv;
  2850. struct xhci_td *td;
  2851. struct xhci_generic_trb *start_trb;
  2852. struct scatterlist *sg = NULL;
  2853. bool more_trbs_coming = true;
  2854. bool need_zero_pkt = false;
  2855. bool first_trb = true;
  2856. unsigned int num_trbs;
  2857. unsigned int start_cycle, num_sgs = 0;
  2858. unsigned int enqd_len, block_len, trb_buff_len, full_len;
  2859. int sent_len, ret;
  2860. u32 field, length_field, remainder;
  2861. u64 addr, send_addr;
  2862. ring = xhci_urb_to_transfer_ring(xhci, urb);
  2863. if (!ring)
  2864. return -EINVAL;
  2865. full_len = urb->transfer_buffer_length;
  2866. /* If we have scatter/gather list, we use it. */
  2867. if (urb->num_sgs) {
  2868. num_sgs = urb->num_mapped_sgs;
  2869. sg = urb->sg;
  2870. addr = (u64) sg_dma_address(sg);
  2871. block_len = sg_dma_len(sg);
  2872. num_trbs = count_sg_trbs_needed(urb);
  2873. } else {
  2874. num_trbs = count_trbs_needed(urb);
  2875. addr = (u64) urb->transfer_dma;
  2876. block_len = full_len;
  2877. }
  2878. ret = prepare_transfer(xhci, xhci->devs[slot_id],
  2879. ep_index, urb->stream_id,
  2880. num_trbs, urb, 0, mem_flags);
  2881. if (unlikely(ret < 0))
  2882. return ret;
  2883. urb_priv = urb->hcpriv;
  2884. /* Deal with URB_ZERO_PACKET - need one more td/trb */
  2885. if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1)
  2886. need_zero_pkt = true;
  2887. td = &urb_priv->td[0];
  2888. /*
  2889. * Don't give the first TRB to the hardware (by toggling the cycle bit)
  2890. * until we've finished creating all the other TRBs. The ring's cycle
  2891. * state may change as we enqueue the other TRBs, so save it too.
  2892. */
  2893. start_trb = &ring->enqueue->generic;
  2894. start_cycle = ring->cycle_state;
  2895. send_addr = addr;
  2896. /* Queue the TRBs, even if they are zero-length */
  2897. for (enqd_len = 0; first_trb || enqd_len < full_len;
  2898. enqd_len += trb_buff_len) {
  2899. field = TRB_TYPE(TRB_NORMAL);
  2900. /* TRB buffer should not cross 64KB boundaries */
  2901. trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
  2902. trb_buff_len = min_t(unsigned int, trb_buff_len, block_len);
  2903. if (enqd_len + trb_buff_len > full_len)
  2904. trb_buff_len = full_len - enqd_len;
  2905. /* Don't change the cycle bit of the first TRB until later */
  2906. if (first_trb) {
  2907. first_trb = false;
  2908. if (start_cycle == 0)
  2909. field |= TRB_CYCLE;
  2910. } else
  2911. field |= ring->cycle_state;
  2912. /* Chain all the TRBs together; clear the chain bit in the last
  2913. * TRB to indicate it's the last TRB in the chain.
  2914. */
  2915. if (enqd_len + trb_buff_len < full_len) {
  2916. field |= TRB_CHAIN;
  2917. if (trb_is_link(ring->enqueue + 1)) {
  2918. if (xhci_align_td(xhci, urb, enqd_len,
  2919. &trb_buff_len,
  2920. ring->enq_seg)) {
  2921. send_addr = ring->enq_seg->bounce_dma;
  2922. /* assuming TD won't span 2 segs */
  2923. td->bounce_seg = ring->enq_seg;
  2924. }
  2925. }
  2926. }
  2927. if (enqd_len + trb_buff_len >= full_len) {
  2928. field &= ~TRB_CHAIN;
  2929. field |= TRB_IOC;
  2930. more_trbs_coming = false;
  2931. td->last_trb = ring->enqueue;
  2932. }
  2933. /* Only set interrupt on short packet for IN endpoints */
  2934. if (usb_urb_dir_in(urb))
  2935. field |= TRB_ISP;
  2936. /* Set the TRB length, TD size, and interrupter fields. */
  2937. remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len,
  2938. full_len, urb, more_trbs_coming);
  2939. length_field = TRB_LEN(trb_buff_len) |
  2940. TRB_TD_SIZE(remainder) |
  2941. TRB_INTR_TARGET(0);
  2942. queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt,
  2943. lower_32_bits(send_addr),
  2944. upper_32_bits(send_addr),
  2945. length_field,
  2946. field);
  2947. addr += trb_buff_len;
  2948. sent_len = trb_buff_len;
  2949. while (sg && sent_len >= block_len) {
  2950. /* New sg entry */
  2951. --num_sgs;
  2952. sent_len -= block_len;
  2953. sg = sg_next(sg);
  2954. if (num_sgs != 0 && sg) {
  2955. block_len = sg_dma_len(sg);
  2956. addr = (u64) sg_dma_address(sg);
  2957. addr += sent_len;
  2958. }
  2959. }
  2960. block_len -= sent_len;
  2961. send_addr = addr;
  2962. }
  2963. if (need_zero_pkt) {
  2964. ret = prepare_transfer(xhci, xhci->devs[slot_id],
  2965. ep_index, urb->stream_id,
  2966. 1, urb, 1, mem_flags);
  2967. urb_priv->td[1].last_trb = ring->enqueue;
  2968. field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC;
  2969. queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field);
  2970. }
  2971. check_trb_math(urb, enqd_len);
  2972. giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
  2973. start_cycle, start_trb);
  2974. return 0;
  2975. }
  2976. /* Caller must have locked xhci->lock */
  2977. int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
  2978. struct urb *urb, int slot_id, unsigned int ep_index)
  2979. {
  2980. struct xhci_ring *ep_ring;
  2981. int num_trbs;
  2982. int ret;
  2983. struct usb_ctrlrequest *setup;
  2984. struct xhci_generic_trb *start_trb;
  2985. int start_cycle;
  2986. u32 field;
  2987. struct urb_priv *urb_priv;
  2988. struct xhci_td *td;
  2989. ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
  2990. if (!ep_ring)
  2991. return -EINVAL;
  2992. /*
  2993. * Need to copy setup packet into setup TRB, so we can't use the setup
  2994. * DMA address.
  2995. */
  2996. if (!urb->setup_packet)
  2997. return -EINVAL;
  2998. /* 1 TRB for setup, 1 for status */
  2999. num_trbs = 2;
  3000. /*
  3001. * Don't need to check if we need additional event data and normal TRBs,
  3002. * since data in control transfers will never get bigger than 16MB
  3003. * XXX: can we get a buffer that crosses 64KB boundaries?
  3004. */
  3005. if (urb->transfer_buffer_length > 0)
  3006. num_trbs++;
  3007. ret = prepare_transfer(xhci, xhci->devs[slot_id],
  3008. ep_index, urb->stream_id,
  3009. num_trbs, urb, 0, mem_flags);
  3010. if (ret < 0)
  3011. return ret;
  3012. urb_priv = urb->hcpriv;
  3013. td = &urb_priv->td[0];
  3014. /*
  3015. * Don't give the first TRB to the hardware (by toggling the cycle bit)
  3016. * until we've finished creating all the other TRBs. The ring's cycle
  3017. * state may change as we enqueue the other TRBs, so save it too.
  3018. */
  3019. start_trb = &ep_ring->enqueue->generic;
  3020. start_cycle = ep_ring->cycle_state;
  3021. /* Queue setup TRB - see section 6.4.1.2.1 */
  3022. /* FIXME better way to translate setup_packet into two u32 fields? */
  3023. setup = (struct usb_ctrlrequest *) urb->setup_packet;
  3024. field = 0;
  3025. field |= TRB_IDT | TRB_TYPE(TRB_SETUP);
  3026. if (start_cycle == 0)
  3027. field |= 0x1;
  3028. /* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */
  3029. if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) {
  3030. if (urb->transfer_buffer_length > 0) {
  3031. if (setup->bRequestType & USB_DIR_IN)
  3032. field |= TRB_TX_TYPE(TRB_DATA_IN);
  3033. else
  3034. field |= TRB_TX_TYPE(TRB_DATA_OUT);
  3035. }
  3036. }
  3037. queue_trb(xhci, ep_ring, true,
  3038. setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16,
  3039. le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16,
  3040. TRB_LEN(8) | TRB_INTR_TARGET(0),
  3041. /* Immediate data in pointer */
  3042. field);
  3043. /* If there's data, queue data TRBs */
  3044. /* Only set interrupt on short packet for IN endpoints */
  3045. if (usb_urb_dir_in(urb))
  3046. field = TRB_ISP | TRB_TYPE(TRB_DATA);
  3047. else
  3048. field = TRB_TYPE(TRB_DATA);
  3049. if (urb->transfer_buffer_length > 0) {
  3050. u32 length_field, remainder;
  3051. remainder = xhci_td_remainder(xhci, 0,
  3052. urb->transfer_buffer_length,
  3053. urb->transfer_buffer_length,
  3054. urb, 1);
  3055. length_field = TRB_LEN(urb->transfer_buffer_length) |
  3056. TRB_TD_SIZE(remainder) |
  3057. TRB_INTR_TARGET(0);
  3058. if (setup->bRequestType & USB_DIR_IN)
  3059. field |= TRB_DIR_IN;
  3060. queue_trb(xhci, ep_ring, true,
  3061. lower_32_bits(urb->transfer_dma),
  3062. upper_32_bits(urb->transfer_dma),
  3063. length_field,
  3064. field | ep_ring->cycle_state);
  3065. }
  3066. /* Save the DMA address of the last TRB in the TD */
  3067. td->last_trb = ep_ring->enqueue;
  3068. /* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */
  3069. /* If the device sent data, the status stage is an OUT transfer */
  3070. if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN)
  3071. field = 0;
  3072. else
  3073. field = TRB_DIR_IN;
  3074. queue_trb(xhci, ep_ring, false,
  3075. 0,
  3076. 0,
  3077. TRB_INTR_TARGET(0),
  3078. /* Event on completion */
  3079. field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state);
  3080. giveback_first_trb(xhci, slot_id, ep_index, 0,
  3081. start_cycle, start_trb);
  3082. return 0;
  3083. }
  3084. /*
  3085. * The transfer burst count field of the isochronous TRB defines the number of
  3086. * bursts that are required to move all packets in this TD. Only SuperSpeed
  3087. * devices can burst up to bMaxBurst number of packets per service interval.
  3088. * This field is zero based, meaning a value of zero in the field means one
  3089. * burst. Basically, for everything but SuperSpeed devices, this field will be
  3090. * zero. Only xHCI 1.0 host controllers support this field.
  3091. */
  3092. static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci,
  3093. struct urb *urb, unsigned int total_packet_count)
  3094. {
  3095. unsigned int max_burst;
  3096. if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER)
  3097. return 0;
  3098. max_burst = urb->ep->ss_ep_comp.bMaxBurst;
  3099. return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1;
  3100. }
  3101. /*
  3102. * Returns the number of packets in the last "burst" of packets. This field is
  3103. * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so
  3104. * the last burst packet count is equal to the total number of packets in the
  3105. * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst
  3106. * must contain (bMaxBurst + 1) number of packets, but the last burst can
  3107. * contain 1 to (bMaxBurst + 1) packets.
  3108. */
  3109. static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci,
  3110. struct urb *urb, unsigned int total_packet_count)
  3111. {
  3112. unsigned int max_burst;
  3113. unsigned int residue;
  3114. if (xhci->hci_version < 0x100)
  3115. return 0;
  3116. if (urb->dev->speed >= USB_SPEED_SUPER) {
  3117. /* bMaxBurst is zero based: 0 means 1 packet per burst */
  3118. max_burst = urb->ep->ss_ep_comp.bMaxBurst;
  3119. residue = total_packet_count % (max_burst + 1);
  3120. /* If residue is zero, the last burst contains (max_burst + 1)
  3121. * number of packets, but the TLBPC field is zero-based.
  3122. */
  3123. if (residue == 0)
  3124. return max_burst;
  3125. return residue - 1;
  3126. }
  3127. if (total_packet_count == 0)
  3128. return 0;
  3129. return total_packet_count - 1;
  3130. }
  3131. /*
  3132. * Calculates Frame ID field of the isochronous TRB identifies the
  3133. * target frame that the Interval associated with this Isochronous
  3134. * Transfer Descriptor will start on. Refer to 4.11.2.5 in 1.1 spec.
  3135. *
  3136. * Returns actual frame id on success, negative value on error.
  3137. */
  3138. static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci,
  3139. struct urb *urb, int index)
  3140. {
  3141. int start_frame, ist, ret = 0;
  3142. int start_frame_id, end_frame_id, current_frame_id;
  3143. if (urb->dev->speed == USB_SPEED_LOW ||
  3144. urb->dev->speed == USB_SPEED_FULL)
  3145. start_frame = urb->start_frame + index * urb->interval;
  3146. else
  3147. start_frame = (urb->start_frame + index * urb->interval) >> 3;
  3148. /* Isochronous Scheduling Threshold (IST, bits 0~3 in HCSPARAMS2):
  3149. *
  3150. * If bit [3] of IST is cleared to '0', software can add a TRB no
  3151. * later than IST[2:0] Microframes before that TRB is scheduled to
  3152. * be executed.
  3153. * If bit [3] of IST is set to '1', software can add a TRB no later
  3154. * than IST[2:0] Frames before that TRB is scheduled to be executed.
  3155. */
  3156. ist = HCS_IST(xhci->hcs_params2) & 0x7;
  3157. if (HCS_IST(xhci->hcs_params2) & (1 << 3))
  3158. ist <<= 3;
  3159. /* Software shall not schedule an Isoch TD with a Frame ID value that
  3160. * is less than the Start Frame ID or greater than the End Frame ID,
  3161. * where:
  3162. *
  3163. * End Frame ID = (Current MFINDEX register value + 895 ms.) MOD 2048
  3164. * Start Frame ID = (Current MFINDEX register value + IST + 1) MOD 2048
  3165. *
  3166. * Both the End Frame ID and Start Frame ID values are calculated
  3167. * in microframes. When software determines the valid Frame ID value;
  3168. * The End Frame ID value should be rounded down to the nearest Frame
  3169. * boundary, and the Start Frame ID value should be rounded up to the
  3170. * nearest Frame boundary.
  3171. */
  3172. current_frame_id = readl(&xhci->run_regs->microframe_index);
  3173. start_frame_id = roundup(current_frame_id + ist + 1, 8);
  3174. end_frame_id = rounddown(current_frame_id + 895 * 8, 8);
  3175. start_frame &= 0x7ff;
  3176. start_frame_id = (start_frame_id >> 3) & 0x7ff;
  3177. end_frame_id = (end_frame_id >> 3) & 0x7ff;
  3178. xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n",
  3179. __func__, index, readl(&xhci->run_regs->microframe_index),
  3180. start_frame_id, end_frame_id, start_frame);
  3181. if (start_frame_id < end_frame_id) {
  3182. if (start_frame > end_frame_id ||
  3183. start_frame < start_frame_id)
  3184. ret = -EINVAL;
  3185. } else if (start_frame_id > end_frame_id) {
  3186. if ((start_frame > end_frame_id &&
  3187. start_frame < start_frame_id))
  3188. ret = -EINVAL;
  3189. } else {
  3190. ret = -EINVAL;
  3191. }
  3192. if (index == 0) {
  3193. if (ret == -EINVAL || start_frame == start_frame_id) {
  3194. start_frame = start_frame_id + 1;
  3195. if (urb->dev->speed == USB_SPEED_LOW ||
  3196. urb->dev->speed == USB_SPEED_FULL)
  3197. urb->start_frame = start_frame;
  3198. else
  3199. urb->start_frame = start_frame << 3;
  3200. ret = 0;
  3201. }
  3202. }
  3203. if (ret) {
  3204. xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n",
  3205. start_frame, current_frame_id, index,
  3206. start_frame_id, end_frame_id);
  3207. xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n");
  3208. return ret;
  3209. }
  3210. return start_frame;
  3211. }
  3212. /* This is for isoc transfer */
  3213. static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
  3214. struct urb *urb, int slot_id, unsigned int ep_index)
  3215. {
  3216. struct xhci_ring *ep_ring;
  3217. struct urb_priv *urb_priv;
  3218. struct xhci_td *td;
  3219. int num_tds, trbs_per_td;
  3220. struct xhci_generic_trb *start_trb;
  3221. bool first_trb;
  3222. int start_cycle;
  3223. u32 field, length_field;
  3224. int running_total, trb_buff_len, td_len, td_remain_len, ret;
  3225. u64 start_addr, addr;
  3226. int i, j;
  3227. bool more_trbs_coming;
  3228. struct xhci_virt_ep *xep;
  3229. int frame_id;
  3230. xep = &xhci->devs[slot_id]->eps[ep_index];
  3231. ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;
  3232. num_tds = urb->number_of_packets;
  3233. if (num_tds < 1) {
  3234. xhci_dbg(xhci, "Isoc URB with zero packets?\n");
  3235. return -EINVAL;
  3236. }
  3237. start_addr = (u64) urb->transfer_dma;
  3238. start_trb = &ep_ring->enqueue->generic;
  3239. start_cycle = ep_ring->cycle_state;
  3240. urb_priv = urb->hcpriv;
  3241. /* Queue the TRBs for each TD, even if they are zero-length */
  3242. for (i = 0; i < num_tds; i++) {
  3243. unsigned int total_pkt_count, max_pkt;
  3244. unsigned int burst_count, last_burst_pkt_count;
  3245. u32 sia_frame_id;
  3246. first_trb = true;
  3247. running_total = 0;
  3248. addr = start_addr + urb->iso_frame_desc[i].offset;
  3249. td_len = urb->iso_frame_desc[i].length;
  3250. td_remain_len = td_len;
  3251. max_pkt = usb_endpoint_maxp(&urb->ep->desc);
  3252. total_pkt_count = DIV_ROUND_UP(td_len, max_pkt);
  3253. /* A zero-length transfer still involves at least one packet. */
  3254. if (total_pkt_count == 0)
  3255. total_pkt_count++;
  3256. burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count);
  3257. last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci,
  3258. urb, total_pkt_count);
  3259. trbs_per_td = count_isoc_trbs_needed(urb, i);
  3260. ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index,
  3261. urb->stream_id, trbs_per_td, urb, i, mem_flags);
  3262. if (ret < 0) {
  3263. if (i == 0)
  3264. return ret;
  3265. goto cleanup;
  3266. }
  3267. td = &urb_priv->td[i];
  3268. /* use SIA as default, if frame id is used overwrite it */
  3269. sia_frame_id = TRB_SIA;
  3270. if (!(urb->transfer_flags & URB_ISO_ASAP) &&
  3271. HCC_CFC(xhci->hcc_params)) {
  3272. frame_id = xhci_get_isoc_frame_id(xhci, urb, i);
  3273. if (frame_id >= 0)
  3274. sia_frame_id = TRB_FRAME_ID(frame_id);
  3275. }
  3276. /*
  3277. * Set isoc specific data for the first TRB in a TD.
  3278. * Prevent HW from getting the TRBs by keeping the cycle state
  3279. * inverted in the first TDs isoc TRB.
  3280. */
  3281. field = TRB_TYPE(TRB_ISOC) |
  3282. TRB_TLBPC(last_burst_pkt_count) |
  3283. sia_frame_id |
  3284. (i ? ep_ring->cycle_state : !start_cycle);
  3285. /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */
  3286. if (!xep->use_extended_tbc)
  3287. field |= TRB_TBC(burst_count);
  3288. /* fill the rest of the TRB fields, and remaining normal TRBs */
  3289. for (j = 0; j < trbs_per_td; j++) {
  3290. u32 remainder = 0;
  3291. /* only first TRB is isoc, overwrite otherwise */
  3292. if (!first_trb)
  3293. field = TRB_TYPE(TRB_NORMAL) |
  3294. ep_ring->cycle_state;
  3295. /* Only set interrupt on short packet for IN EPs */
  3296. if (usb_urb_dir_in(urb))
  3297. field |= TRB_ISP;
  3298. /* Set the chain bit for all except the last TRB */
  3299. if (j < trbs_per_td - 1) {
  3300. more_trbs_coming = true;
  3301. field |= TRB_CHAIN;
  3302. } else {
  3303. more_trbs_coming = false;
  3304. td->last_trb = ep_ring->enqueue;
  3305. field |= TRB_IOC;
  3306. /* set BEI, except for the last TD */
  3307. if (xhci->hci_version >= 0x100 &&
  3308. !(xhci->quirks & XHCI_AVOID_BEI) &&
  3309. i < num_tds - 1)
  3310. field |= TRB_BEI;
  3311. }
  3312. /* Calculate TRB length */
  3313. trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
  3314. if (trb_buff_len > td_remain_len)
  3315. trb_buff_len = td_remain_len;
  3316. /* Set the TRB length, TD size, & interrupter fields. */
  3317. remainder = xhci_td_remainder(xhci, running_total,
  3318. trb_buff_len, td_len,
  3319. urb, more_trbs_coming);
  3320. length_field = TRB_LEN(trb_buff_len) |
  3321. TRB_INTR_TARGET(0);
  3322. /* xhci 1.1 with ETE uses TD Size field for TBC */
  3323. if (first_trb && xep->use_extended_tbc)
  3324. length_field |= TRB_TD_SIZE_TBC(burst_count);
  3325. else
  3326. length_field |= TRB_TD_SIZE(remainder);
  3327. first_trb = false;
  3328. queue_trb(xhci, ep_ring, more_trbs_coming,
  3329. lower_32_bits(addr),
  3330. upper_32_bits(addr),
  3331. length_field,
  3332. field);
  3333. running_total += trb_buff_len;
  3334. addr += trb_buff_len;
  3335. td_remain_len -= trb_buff_len;
  3336. }
  3337. /* Check TD length */
  3338. if (running_total != td_len) {
  3339. xhci_err(xhci, "ISOC TD length unmatch\n");
  3340. ret = -EINVAL;
  3341. goto cleanup;
  3342. }
  3343. }
  3344. /* store the next frame id */
  3345. if (HCC_CFC(xhci->hcc_params))
  3346. xep->next_frame_id = urb->start_frame + num_tds * urb->interval;
  3347. if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
  3348. if (xhci->quirks & XHCI_AMD_PLL_FIX)
  3349. usb_amd_quirk_pll_disable();
  3350. }
  3351. xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++;
  3352. giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
  3353. start_cycle, start_trb);
  3354. return 0;
  3355. cleanup:
  3356. /* Clean up a partially enqueued isoc transfer. */
  3357. for (i--; i >= 0; i--)
  3358. list_del_init(&urb_priv->td[i].td_list);
  3359. /* Use the first TD as a temporary variable to turn the TDs we've queued
  3360. * into No-ops with a software-owned cycle bit. That way the hardware
  3361. * won't accidentally start executing bogus TDs when we partially
  3362. * overwrite them. td->first_trb and td->start_seg are already set.
  3363. */
  3364. urb_priv->td[0].last_trb = ep_ring->enqueue;
  3365. /* Every TRB except the first & last will have its cycle bit flipped. */
  3366. td_to_noop(xhci, ep_ring, &urb_priv->td[0], true);
  3367. /* Reset the ring enqueue back to the first TRB and its cycle bit. */
  3368. ep_ring->enqueue = urb_priv->td[0].first_trb;
  3369. ep_ring->enq_seg = urb_priv->td[0].start_seg;
  3370. ep_ring->cycle_state = start_cycle;
  3371. ep_ring->num_trbs_free = ep_ring->num_trbs_free_temp;
  3372. usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);
  3373. return ret;
  3374. }
  3375. /*
  3376. * Check transfer ring to guarantee there is enough room for the urb.
  3377. * Update ISO URB start_frame and interval.
  3378. * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to
  3379. * update urb->start_frame if URB_ISO_ASAP is set in transfer_flags or
  3380. * Contiguous Frame ID is not supported by HC.
  3381. */
  3382. int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
  3383. struct urb *urb, int slot_id, unsigned int ep_index)
  3384. {
  3385. struct xhci_virt_device *xdev;
  3386. struct xhci_ring *ep_ring;
  3387. struct xhci_ep_ctx *ep_ctx;
  3388. int start_frame;
  3389. int num_tds, num_trbs, i;
  3390. int ret;
  3391. struct xhci_virt_ep *xep;
  3392. int ist;
  3393. xdev = xhci->devs[slot_id];
  3394. xep = &xhci->devs[slot_id]->eps[ep_index];
  3395. ep_ring = xdev->eps[ep_index].ring;
  3396. ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
  3397. num_trbs = 0;
  3398. num_tds = urb->number_of_packets;
  3399. for (i = 0; i < num_tds; i++)
  3400. num_trbs += count_isoc_trbs_needed(urb, i);
  3401. /* Check the ring to guarantee there is enough room for the whole urb.
  3402. * Do not insert any td of the urb to the ring if the check failed.
  3403. */
  3404. ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
  3405. num_trbs, mem_flags);
  3406. if (ret)
  3407. return ret;
  3408. /*
  3409. * Check interval value. This should be done before we start to
  3410. * calculate the start frame value.
  3411. */
  3412. check_interval(xhci, urb, ep_ctx);
  3413. /* Calculate the start frame and put it in urb->start_frame. */
  3414. if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {
  3415. if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_RUNNING) {
  3416. urb->start_frame = xep->next_frame_id;
  3417. goto skip_start_over;
  3418. }
  3419. }
  3420. start_frame = readl(&xhci->run_regs->microframe_index);
  3421. start_frame &= 0x3fff;
  3422. /*
  3423. * Round up to the next frame and consider the time before trb really
  3424. * gets scheduled by hardare.
  3425. */
  3426. ist = HCS_IST(xhci->hcs_params2) & 0x7;
  3427. if (HCS_IST(xhci->hcs_params2) & (1 << 3))
  3428. ist <<= 3;
  3429. start_frame += ist + XHCI_CFC_DELAY;
  3430. start_frame = roundup(start_frame, 8);
  3431. /*
  3432. * Round up to the next ESIT (Endpoint Service Interval Time) if ESIT
  3433. * is greate than 8 microframes.
  3434. */
  3435. if (urb->dev->speed == USB_SPEED_LOW ||
  3436. urb->dev->speed == USB_SPEED_FULL) {
  3437. start_frame = roundup(start_frame, urb->interval << 3);
  3438. urb->start_frame = start_frame >> 3;
  3439. } else {
  3440. start_frame = roundup(start_frame, urb->interval);
  3441. urb->start_frame = start_frame;
  3442. }
  3443. skip_start_over:
  3444. ep_ring->num_trbs_free_temp = ep_ring->num_trbs_free;
  3445. return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index);
  3446. }
  3447. /**** Command Ring Operations ****/
  3448. /* Generic function for queueing a command TRB on the command ring.
  3449. * Check to make sure there's room on the command ring for one command TRB.
  3450. * Also check that there's room reserved for commands that must not fail.
  3451. * If this is a command that must not fail, meaning command_must_succeed = TRUE,
  3452. * then only check for the number of reserved spots.
  3453. * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
  3454. * because the command event handler may want to resubmit a failed command.
  3455. */
  3456. static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
  3457. u32 field1, u32 field2,
  3458. u32 field3, u32 field4, bool command_must_succeed)
  3459. {
  3460. int reserved_trbs = xhci->cmd_ring_reserved_trbs;
  3461. int ret;
  3462. if ((xhci->xhc_state & XHCI_STATE_DYING) ||
  3463. (xhci->xhc_state & XHCI_STATE_HALTED)) {
  3464. xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n");
  3465. return -ESHUTDOWN;
  3466. }
  3467. if (!command_must_succeed)
  3468. reserved_trbs++;
  3469. ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING,
  3470. reserved_trbs, GFP_ATOMIC);
  3471. if (ret < 0) {
  3472. xhci_err(xhci, "ERR: No room for command on command ring\n");
  3473. if (command_must_succeed)
  3474. xhci_err(xhci, "ERR: Reserved TRB counting for "
  3475. "unfailable commands failed.\n");
  3476. return ret;
  3477. }
  3478. cmd->command_trb = xhci->cmd_ring->enqueue;
  3479. /* if there are no other commands queued we start the timeout timer */
  3480. if (list_empty(&xhci->cmd_list)) {
  3481. xhci->current_cmd = cmd;
  3482. xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
  3483. }
  3484. list_add_tail(&cmd->cmd_list, &xhci->cmd_list);
  3485. queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3,
  3486. field4 | xhci->cmd_ring->cycle_state);
  3487. return 0;
  3488. }
  3489. /* Queue a slot enable or disable request on the command ring */
  3490. int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd,
  3491. u32 trb_type, u32 slot_id)
  3492. {
  3493. return queue_command(xhci, cmd, 0, 0, 0,
  3494. TRB_TYPE(trb_type) | SLOT_ID_FOR_TRB(slot_id), false);
  3495. }
  3496. /* Queue an address device command TRB */
  3497. int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
  3498. dma_addr_t in_ctx_ptr, u32 slot_id, enum xhci_setup_dev setup)
  3499. {
  3500. return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
  3501. upper_32_bits(in_ctx_ptr), 0,
  3502. TRB_TYPE(TRB_ADDR_DEV) | SLOT_ID_FOR_TRB(slot_id)
  3503. | (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0), false);
  3504. }
  3505. int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
  3506. u32 field1, u32 field2, u32 field3, u32 field4)
  3507. {
  3508. return queue_command(xhci, cmd, field1, field2, field3, field4, false);
  3509. }
  3510. /* Queue a reset device command TRB */
  3511. int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
  3512. u32 slot_id)
  3513. {
  3514. return queue_command(xhci, cmd, 0, 0, 0,
  3515. TRB_TYPE(TRB_RESET_DEV) | SLOT_ID_FOR_TRB(slot_id),
  3516. false);
  3517. }
  3518. /* Queue a configure endpoint command TRB */
  3519. int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
  3520. struct xhci_command *cmd, dma_addr_t in_ctx_ptr,
  3521. u32 slot_id, bool command_must_succeed)
  3522. {
  3523. return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
  3524. upper_32_bits(in_ctx_ptr), 0,
  3525. TRB_TYPE(TRB_CONFIG_EP) | SLOT_ID_FOR_TRB(slot_id),
  3526. command_must_succeed);
  3527. }
  3528. /* Queue an evaluate context command TRB */
  3529. int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd,
  3530. dma_addr_t in_ctx_ptr, u32 slot_id, bool command_must_succeed)
  3531. {
  3532. return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
  3533. upper_32_bits(in_ctx_ptr), 0,
  3534. TRB_TYPE(TRB_EVAL_CONTEXT) | SLOT_ID_FOR_TRB(slot_id),
  3535. command_must_succeed);
  3536. }
  3537. /*
  3538. * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop
  3539. * activity on an endpoint that is about to be suspended.
  3540. */
  3541. int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd,
  3542. int slot_id, unsigned int ep_index, int suspend)
  3543. {
  3544. u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
  3545. u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
  3546. u32 type = TRB_TYPE(TRB_STOP_RING);
  3547. u32 trb_suspend = SUSPEND_PORT_FOR_TRB(suspend);
  3548. return queue_command(xhci, cmd, 0, 0, 0,
  3549. trb_slot_id | trb_ep_index | type | trb_suspend, false);
  3550. }
  3551. /* Set Transfer Ring Dequeue Pointer command */
  3552. void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
  3553. unsigned int slot_id, unsigned int ep_index,
  3554. struct xhci_dequeue_state *deq_state)
  3555. {
  3556. dma_addr_t addr;
  3557. u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
  3558. u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
  3559. u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id);
  3560. u32 trb_sct = 0;
  3561. u32 type = TRB_TYPE(TRB_SET_DEQ);
  3562. struct xhci_virt_ep *ep;
  3563. struct xhci_command *cmd;
  3564. int ret;
  3565. xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
  3566. "Set TR Deq Ptr cmd, new deq seg = %p (0x%llx dma), new deq ptr = %p (0x%llx dma), new cycle = %u",
  3567. deq_state->new_deq_seg,
  3568. (unsigned long long)deq_state->new_deq_seg->dma,
  3569. deq_state->new_deq_ptr,
  3570. (unsigned long long)xhci_trb_virt_to_dma(
  3571. deq_state->new_deq_seg, deq_state->new_deq_ptr),
  3572. deq_state->new_cycle_state);
  3573. addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
  3574. deq_state->new_deq_ptr);
  3575. if (addr == 0) {
  3576. xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
  3577. xhci_warn(xhci, "WARN deq seg = %p, deq pt = %p\n",
  3578. deq_state->new_deq_seg, deq_state->new_deq_ptr);
  3579. return;
  3580. }
  3581. ep = &xhci->devs[slot_id]->eps[ep_index];
  3582. if ((ep->ep_state & SET_DEQ_PENDING)) {
  3583. xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
  3584. xhci_warn(xhci, "A Set TR Deq Ptr command is pending.\n");
  3585. return;
  3586. }
  3587. /* This function gets called from contexts where it cannot sleep */
  3588. cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC);
  3589. if (!cmd)
  3590. return;
  3591. ep->queued_deq_seg = deq_state->new_deq_seg;
  3592. ep->queued_deq_ptr = deq_state->new_deq_ptr;
  3593. if (deq_state->stream_id)
  3594. trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
  3595. ret = queue_command(xhci, cmd,
  3596. lower_32_bits(addr) | trb_sct | deq_state->new_cycle_state,
  3597. upper_32_bits(addr), trb_stream_id,
  3598. trb_slot_id | trb_ep_index | type, false);
  3599. if (ret < 0) {
  3600. xhci_free_command(xhci, cmd);
  3601. return;
  3602. }
  3603. /* Stop the TD queueing code from ringing the doorbell until
  3604. * this command completes. The HC won't set the dequeue pointer
  3605. * if the ring is running, and ringing the doorbell starts the
  3606. * ring running.
  3607. */
  3608. ep->ep_state |= SET_DEQ_PENDING;
  3609. }
  3610. int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd,
  3611. int slot_id, unsigned int ep_index,
  3612. enum xhci_ep_reset_type reset_type)
  3613. {
  3614. u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
  3615. u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
  3616. u32 type = TRB_TYPE(TRB_RESET_EP);
  3617. if (reset_type == EP_SOFT_RESET)
  3618. type |= TRB_TSP;
  3619. return queue_command(xhci, cmd, 0, 0, 0,
  3620. trb_slot_id | trb_ep_index | type, false);
  3621. }