pci-ioda.c 109 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090
  1. /*
  2. * Support PCI/PCIe on PowerNV platforms
  3. *
  4. * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
  5. *
  6. * This program is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU General Public License
  8. * as published by the Free Software Foundation; either version
  9. * 2 of the License, or (at your option) any later version.
  10. */
  11. #undef DEBUG
  12. #include <linux/kernel.h>
  13. #include <linux/pci.h>
  14. #include <linux/crash_dump.h>
  15. #include <linux/delay.h>
  16. #include <linux/string.h>
  17. #include <linux/init.h>
  18. #include <linux/bootmem.h>
  19. #include <linux/irq.h>
  20. #include <linux/io.h>
  21. #include <linux/msi.h>
  22. #include <linux/memblock.h>
  23. #include <linux/iommu.h>
  24. #include <linux/rculist.h>
  25. #include <linux/sizes.h>
  26. #include <asm/sections.h>
  27. #include <asm/io.h>
  28. #include <asm/prom.h>
  29. #include <asm/pci-bridge.h>
  30. #include <asm/machdep.h>
  31. #include <asm/msi_bitmap.h>
  32. #include <asm/ppc-pci.h>
  33. #include <asm/opal.h>
  34. #include <asm/iommu.h>
  35. #include <asm/tce.h>
  36. #include <asm/xics.h>
  37. #include <asm/debugfs.h>
  38. #include <asm/firmware.h>
  39. #include <asm/pnv-pci.h>
  40. #include <asm/mmzone.h>
  41. #include <misc/cxl-base.h>
  42. #include "powernv.h"
  43. #include "pci.h"
  44. #include "../../../../drivers/pci/pci.h"
  45. #define PNV_IODA1_M64_NUM 16 /* Number of M64 BARs */
  46. #define PNV_IODA1_M64_SEGS 8 /* Segments per M64 BAR */
  47. #define PNV_IODA1_DMA32_SEGSIZE 0x10000000
  48. static const char * const pnv_phb_names[] = { "IODA1", "IODA2", "NPU_NVLINK",
  49. "NPU_OCAPI" };
  50. void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
  51. const char *fmt, ...)
  52. {
  53. struct va_format vaf;
  54. va_list args;
  55. char pfix[32];
  56. va_start(args, fmt);
  57. vaf.fmt = fmt;
  58. vaf.va = &args;
  59. if (pe->flags & PNV_IODA_PE_DEV)
  60. strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix));
  61. else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  62. sprintf(pfix, "%04x:%02x ",
  63. pci_domain_nr(pe->pbus), pe->pbus->number);
  64. #ifdef CONFIG_PCI_IOV
  65. else if (pe->flags & PNV_IODA_PE_VF)
  66. sprintf(pfix, "%04x:%02x:%2x.%d",
  67. pci_domain_nr(pe->parent_dev->bus),
  68. (pe->rid & 0xff00) >> 8,
  69. PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
  70. #endif /* CONFIG_PCI_IOV*/
  71. printk("%spci %s: [PE# %.2x] %pV",
  72. level, pfix, pe->pe_number, &vaf);
  73. va_end(args);
  74. }
  75. static bool pnv_iommu_bypass_disabled __read_mostly;
  76. static bool pci_reset_phbs __read_mostly;
  77. static int __init iommu_setup(char *str)
  78. {
  79. if (!str)
  80. return -EINVAL;
  81. while (*str) {
  82. if (!strncmp(str, "nobypass", 8)) {
  83. pnv_iommu_bypass_disabled = true;
  84. pr_info("PowerNV: IOMMU bypass window disabled.\n");
  85. break;
  86. }
  87. str += strcspn(str, ",");
  88. if (*str == ',')
  89. str++;
  90. }
  91. return 0;
  92. }
  93. early_param("iommu", iommu_setup);
  94. static int __init pci_reset_phbs_setup(char *str)
  95. {
  96. pci_reset_phbs = true;
  97. return 0;
  98. }
  99. early_param("ppc_pci_reset_phbs", pci_reset_phbs_setup);
  100. static inline bool pnv_pci_is_m64(struct pnv_phb *phb, struct resource *r)
  101. {
  102. /*
  103. * WARNING: We cannot rely on the resource flags. The Linux PCI
  104. * allocation code sometimes decides to put a 64-bit prefetchable
  105. * BAR in the 32-bit window, so we have to compare the addresses.
  106. *
  107. * For simplicity we only test resource start.
  108. */
  109. return (r->start >= phb->ioda.m64_base &&
  110. r->start < (phb->ioda.m64_base + phb->ioda.m64_size));
  111. }
  112. static inline bool pnv_pci_is_m64_flags(unsigned long resource_flags)
  113. {
  114. unsigned long flags = (IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
  115. return (resource_flags & flags) == flags;
  116. }
  117. static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
  118. {
  119. s64 rc;
  120. phb->ioda.pe_array[pe_no].phb = phb;
  121. phb->ioda.pe_array[pe_no].pe_number = pe_no;
  122. /*
  123. * Clear the PE frozen state as it might be put into frozen state
  124. * in the last PCI remove path. It's not harmful to do so when the
  125. * PE is already in unfrozen state.
  126. */
  127. rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
  128. OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
  129. if (rc != OPAL_SUCCESS && rc != OPAL_UNSUPPORTED)
  130. pr_warn("%s: Error %lld unfreezing PHB#%x-PE#%x\n",
  131. __func__, rc, phb->hose->global_number, pe_no);
  132. return &phb->ioda.pe_array[pe_no];
  133. }
  134. static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no)
  135. {
  136. if (!(pe_no >= 0 && pe_no < phb->ioda.total_pe_num)) {
  137. pr_warn("%s: Invalid PE %x on PHB#%x\n",
  138. __func__, pe_no, phb->hose->global_number);
  139. return;
  140. }
  141. if (test_and_set_bit(pe_no, phb->ioda.pe_alloc))
  142. pr_debug("%s: PE %x was reserved on PHB#%x\n",
  143. __func__, pe_no, phb->hose->global_number);
  144. pnv_ioda_init_pe(phb, pe_no);
  145. }
  146. static struct pnv_ioda_pe *pnv_ioda_alloc_pe(struct pnv_phb *phb)
  147. {
  148. long pe;
  149. for (pe = phb->ioda.total_pe_num - 1; pe >= 0; pe--) {
  150. if (!test_and_set_bit(pe, phb->ioda.pe_alloc))
  151. return pnv_ioda_init_pe(phb, pe);
  152. }
  153. return NULL;
  154. }
  155. static void pnv_ioda_free_pe(struct pnv_ioda_pe *pe)
  156. {
  157. struct pnv_phb *phb = pe->phb;
  158. unsigned int pe_num = pe->pe_number;
  159. WARN_ON(pe->pdev);
  160. memset(pe, 0, sizeof(struct pnv_ioda_pe));
  161. clear_bit(pe_num, phb->ioda.pe_alloc);
  162. }
  163. /* The default M64 BAR is shared by all PEs */
  164. static int pnv_ioda2_init_m64(struct pnv_phb *phb)
  165. {
  166. const char *desc;
  167. struct resource *r;
  168. s64 rc;
  169. /* Configure the default M64 BAR */
  170. rc = opal_pci_set_phb_mem_window(phb->opal_id,
  171. OPAL_M64_WINDOW_TYPE,
  172. phb->ioda.m64_bar_idx,
  173. phb->ioda.m64_base,
  174. 0, /* unused */
  175. phb->ioda.m64_size);
  176. if (rc != OPAL_SUCCESS) {
  177. desc = "configuring";
  178. goto fail;
  179. }
  180. /* Enable the default M64 BAR */
  181. rc = opal_pci_phb_mmio_enable(phb->opal_id,
  182. OPAL_M64_WINDOW_TYPE,
  183. phb->ioda.m64_bar_idx,
  184. OPAL_ENABLE_M64_SPLIT);
  185. if (rc != OPAL_SUCCESS) {
  186. desc = "enabling";
  187. goto fail;
  188. }
  189. /*
  190. * Exclude the segments for reserved and root bus PE, which
  191. * are first or last two PEs.
  192. */
  193. r = &phb->hose->mem_resources[1];
  194. if (phb->ioda.reserved_pe_idx == 0)
  195. r->start += (2 * phb->ioda.m64_segsize);
  196. else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1))
  197. r->end -= (2 * phb->ioda.m64_segsize);
  198. else
  199. pr_warn(" Cannot strip M64 segment for reserved PE#%x\n",
  200. phb->ioda.reserved_pe_idx);
  201. return 0;
  202. fail:
  203. pr_warn(" Failure %lld %s M64 BAR#%d\n",
  204. rc, desc, phb->ioda.m64_bar_idx);
  205. opal_pci_phb_mmio_enable(phb->opal_id,
  206. OPAL_M64_WINDOW_TYPE,
  207. phb->ioda.m64_bar_idx,
  208. OPAL_DISABLE_M64);
  209. return -EIO;
  210. }
  211. static void pnv_ioda_reserve_dev_m64_pe(struct pci_dev *pdev,
  212. unsigned long *pe_bitmap)
  213. {
  214. struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  215. struct pnv_phb *phb = hose->private_data;
  216. struct resource *r;
  217. resource_size_t base, sgsz, start, end;
  218. int segno, i;
  219. base = phb->ioda.m64_base;
  220. sgsz = phb->ioda.m64_segsize;
  221. for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
  222. r = &pdev->resource[i];
  223. if (!r->parent || !pnv_pci_is_m64(phb, r))
  224. continue;
  225. start = _ALIGN_DOWN(r->start - base, sgsz);
  226. end = _ALIGN_UP(r->end - base, sgsz);
  227. for (segno = start / sgsz; segno < end / sgsz; segno++) {
  228. if (pe_bitmap)
  229. set_bit(segno, pe_bitmap);
  230. else
  231. pnv_ioda_reserve_pe(phb, segno);
  232. }
  233. }
  234. }
  235. static int pnv_ioda1_init_m64(struct pnv_phb *phb)
  236. {
  237. struct resource *r;
  238. int index;
  239. /*
  240. * There are 16 M64 BARs, each of which has 8 segments. So
  241. * there are as many M64 segments as the maximum number of
  242. * PEs, which is 128.
  243. */
  244. for (index = 0; index < PNV_IODA1_M64_NUM; index++) {
  245. unsigned long base, segsz = phb->ioda.m64_segsize;
  246. int64_t rc;
  247. base = phb->ioda.m64_base +
  248. index * PNV_IODA1_M64_SEGS * segsz;
  249. rc = opal_pci_set_phb_mem_window(phb->opal_id,
  250. OPAL_M64_WINDOW_TYPE, index, base, 0,
  251. PNV_IODA1_M64_SEGS * segsz);
  252. if (rc != OPAL_SUCCESS) {
  253. pr_warn(" Error %lld setting M64 PHB#%x-BAR#%d\n",
  254. rc, phb->hose->global_number, index);
  255. goto fail;
  256. }
  257. rc = opal_pci_phb_mmio_enable(phb->opal_id,
  258. OPAL_M64_WINDOW_TYPE, index,
  259. OPAL_ENABLE_M64_SPLIT);
  260. if (rc != OPAL_SUCCESS) {
  261. pr_warn(" Error %lld enabling M64 PHB#%x-BAR#%d\n",
  262. rc, phb->hose->global_number, index);
  263. goto fail;
  264. }
  265. }
  266. /*
  267. * Exclude the segments for reserved and root bus PE, which
  268. * are first or last two PEs.
  269. */
  270. r = &phb->hose->mem_resources[1];
  271. if (phb->ioda.reserved_pe_idx == 0)
  272. r->start += (2 * phb->ioda.m64_segsize);
  273. else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1))
  274. r->end -= (2 * phb->ioda.m64_segsize);
  275. else
  276. WARN(1, "Wrong reserved PE#%x on PHB#%x\n",
  277. phb->ioda.reserved_pe_idx, phb->hose->global_number);
  278. return 0;
  279. fail:
  280. for ( ; index >= 0; index--)
  281. opal_pci_phb_mmio_enable(phb->opal_id,
  282. OPAL_M64_WINDOW_TYPE, index, OPAL_DISABLE_M64);
  283. return -EIO;
  284. }
  285. static void pnv_ioda_reserve_m64_pe(struct pci_bus *bus,
  286. unsigned long *pe_bitmap,
  287. bool all)
  288. {
  289. struct pci_dev *pdev;
  290. list_for_each_entry(pdev, &bus->devices, bus_list) {
  291. pnv_ioda_reserve_dev_m64_pe(pdev, pe_bitmap);
  292. if (all && pdev->subordinate)
  293. pnv_ioda_reserve_m64_pe(pdev->subordinate,
  294. pe_bitmap, all);
  295. }
  296. }
  297. static struct pnv_ioda_pe *pnv_ioda_pick_m64_pe(struct pci_bus *bus, bool all)
  298. {
  299. struct pci_controller *hose = pci_bus_to_host(bus);
  300. struct pnv_phb *phb = hose->private_data;
  301. struct pnv_ioda_pe *master_pe, *pe;
  302. unsigned long size, *pe_alloc;
  303. int i;
  304. /* Root bus shouldn't use M64 */
  305. if (pci_is_root_bus(bus))
  306. return NULL;
  307. /* Allocate bitmap */
  308. size = _ALIGN_UP(phb->ioda.total_pe_num / 8, sizeof(unsigned long));
  309. pe_alloc = kzalloc(size, GFP_KERNEL);
  310. if (!pe_alloc) {
  311. pr_warn("%s: Out of memory !\n",
  312. __func__);
  313. return NULL;
  314. }
  315. /* Figure out reserved PE numbers by the PE */
  316. pnv_ioda_reserve_m64_pe(bus, pe_alloc, all);
  317. /*
  318. * the current bus might not own M64 window and that's all
  319. * contributed by its child buses. For the case, we needn't
  320. * pick M64 dependent PE#.
  321. */
  322. if (bitmap_empty(pe_alloc, phb->ioda.total_pe_num)) {
  323. kfree(pe_alloc);
  324. return NULL;
  325. }
  326. /*
  327. * Figure out the master PE and put all slave PEs to master
  328. * PE's list to form compound PE.
  329. */
  330. master_pe = NULL;
  331. i = -1;
  332. while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe_num, i + 1)) <
  333. phb->ioda.total_pe_num) {
  334. pe = &phb->ioda.pe_array[i];
  335. phb->ioda.m64_segmap[pe->pe_number] = pe->pe_number;
  336. if (!master_pe) {
  337. pe->flags |= PNV_IODA_PE_MASTER;
  338. INIT_LIST_HEAD(&pe->slaves);
  339. master_pe = pe;
  340. } else {
  341. pe->flags |= PNV_IODA_PE_SLAVE;
  342. pe->master = master_pe;
  343. list_add_tail(&pe->list, &master_pe->slaves);
  344. }
  345. /*
  346. * P7IOC supports M64DT, which helps mapping M64 segment
  347. * to one particular PE#. However, PHB3 has fixed mapping
  348. * between M64 segment and PE#. In order to have same logic
  349. * for P7IOC and PHB3, we enforce fixed mapping between M64
  350. * segment and PE# on P7IOC.
  351. */
  352. if (phb->type == PNV_PHB_IODA1) {
  353. int64_t rc;
  354. rc = opal_pci_map_pe_mmio_window(phb->opal_id,
  355. pe->pe_number, OPAL_M64_WINDOW_TYPE,
  356. pe->pe_number / PNV_IODA1_M64_SEGS,
  357. pe->pe_number % PNV_IODA1_M64_SEGS);
  358. if (rc != OPAL_SUCCESS)
  359. pr_warn("%s: Error %lld mapping M64 for PHB#%x-PE#%x\n",
  360. __func__, rc, phb->hose->global_number,
  361. pe->pe_number);
  362. }
  363. }
  364. kfree(pe_alloc);
  365. return master_pe;
  366. }
  367. static void __init pnv_ioda_parse_m64_window(struct pnv_phb *phb)
  368. {
  369. struct pci_controller *hose = phb->hose;
  370. struct device_node *dn = hose->dn;
  371. struct resource *res;
  372. u32 m64_range[2], i;
  373. const __be32 *r;
  374. u64 pci_addr;
  375. if (phb->type != PNV_PHB_IODA1 && phb->type != PNV_PHB_IODA2) {
  376. pr_info(" Not support M64 window\n");
  377. return;
  378. }
  379. if (!firmware_has_feature(FW_FEATURE_OPAL)) {
  380. pr_info(" Firmware too old to support M64 window\n");
  381. return;
  382. }
  383. r = of_get_property(dn, "ibm,opal-m64-window", NULL);
  384. if (!r) {
  385. pr_info(" No <ibm,opal-m64-window> on %pOF\n",
  386. dn);
  387. return;
  388. }
  389. /*
  390. * Find the available M64 BAR range and pickup the last one for
  391. * covering the whole 64-bits space. We support only one range.
  392. */
  393. if (of_property_read_u32_array(dn, "ibm,opal-available-m64-ranges",
  394. m64_range, 2)) {
  395. /* In absence of the property, assume 0..15 */
  396. m64_range[0] = 0;
  397. m64_range[1] = 16;
  398. }
  399. /* We only support 64 bits in our allocator */
  400. if (m64_range[1] > 63) {
  401. pr_warn("%s: Limiting M64 range to 63 (from %d) on PHB#%x\n",
  402. __func__, m64_range[1], phb->hose->global_number);
  403. m64_range[1] = 63;
  404. }
  405. /* Empty range, no m64 */
  406. if (m64_range[1] <= m64_range[0]) {
  407. pr_warn("%s: M64 empty, disabling M64 usage on PHB#%x\n",
  408. __func__, phb->hose->global_number);
  409. return;
  410. }
  411. /* Configure M64 informations */
  412. res = &hose->mem_resources[1];
  413. res->name = dn->full_name;
  414. res->start = of_translate_address(dn, r + 2);
  415. res->end = res->start + of_read_number(r + 4, 2) - 1;
  416. res->flags = (IORESOURCE_MEM | IORESOURCE_MEM_64 | IORESOURCE_PREFETCH);
  417. pci_addr = of_read_number(r, 2);
  418. hose->mem_offset[1] = res->start - pci_addr;
  419. phb->ioda.m64_size = resource_size(res);
  420. phb->ioda.m64_segsize = phb->ioda.m64_size / phb->ioda.total_pe_num;
  421. phb->ioda.m64_base = pci_addr;
  422. /* This lines up nicely with the display from processing OF ranges */
  423. pr_info(" MEM 0x%016llx..0x%016llx -> 0x%016llx (M64 #%d..%d)\n",
  424. res->start, res->end, pci_addr, m64_range[0],
  425. m64_range[0] + m64_range[1] - 1);
  426. /* Mark all M64 used up by default */
  427. phb->ioda.m64_bar_alloc = (unsigned long)-1;
  428. /* Use last M64 BAR to cover M64 window */
  429. m64_range[1]--;
  430. phb->ioda.m64_bar_idx = m64_range[0] + m64_range[1];
  431. pr_info(" Using M64 #%d as default window\n", phb->ioda.m64_bar_idx);
  432. /* Mark remaining ones free */
  433. for (i = m64_range[0]; i < m64_range[1]; i++)
  434. clear_bit(i, &phb->ioda.m64_bar_alloc);
  435. /*
  436. * Setup init functions for M64 based on IODA version, IODA3 uses
  437. * the IODA2 code.
  438. */
  439. if (phb->type == PNV_PHB_IODA1)
  440. phb->init_m64 = pnv_ioda1_init_m64;
  441. else
  442. phb->init_m64 = pnv_ioda2_init_m64;
  443. phb->reserve_m64_pe = pnv_ioda_reserve_m64_pe;
  444. phb->pick_m64_pe = pnv_ioda_pick_m64_pe;
  445. }
  446. static void pnv_ioda_freeze_pe(struct pnv_phb *phb, int pe_no)
  447. {
  448. struct pnv_ioda_pe *pe = &phb->ioda.pe_array[pe_no];
  449. struct pnv_ioda_pe *slave;
  450. s64 rc;
  451. /* Fetch master PE */
  452. if (pe->flags & PNV_IODA_PE_SLAVE) {
  453. pe = pe->master;
  454. if (WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER)))
  455. return;
  456. pe_no = pe->pe_number;
  457. }
  458. /* Freeze master PE */
  459. rc = opal_pci_eeh_freeze_set(phb->opal_id,
  460. pe_no,
  461. OPAL_EEH_ACTION_SET_FREEZE_ALL);
  462. if (rc != OPAL_SUCCESS) {
  463. pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
  464. __func__, rc, phb->hose->global_number, pe_no);
  465. return;
  466. }
  467. /* Freeze slave PEs */
  468. if (!(pe->flags & PNV_IODA_PE_MASTER))
  469. return;
  470. list_for_each_entry(slave, &pe->slaves, list) {
  471. rc = opal_pci_eeh_freeze_set(phb->opal_id,
  472. slave->pe_number,
  473. OPAL_EEH_ACTION_SET_FREEZE_ALL);
  474. if (rc != OPAL_SUCCESS)
  475. pr_warn("%s: Failure %lld freezing PHB#%x-PE#%x\n",
  476. __func__, rc, phb->hose->global_number,
  477. slave->pe_number);
  478. }
  479. }
  480. static int pnv_ioda_unfreeze_pe(struct pnv_phb *phb, int pe_no, int opt)
  481. {
  482. struct pnv_ioda_pe *pe, *slave;
  483. s64 rc;
  484. /* Find master PE */
  485. pe = &phb->ioda.pe_array[pe_no];
  486. if (pe->flags & PNV_IODA_PE_SLAVE) {
  487. pe = pe->master;
  488. WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
  489. pe_no = pe->pe_number;
  490. }
  491. /* Clear frozen state for master PE */
  492. rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no, opt);
  493. if (rc != OPAL_SUCCESS) {
  494. pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
  495. __func__, rc, opt, phb->hose->global_number, pe_no);
  496. return -EIO;
  497. }
  498. if (!(pe->flags & PNV_IODA_PE_MASTER))
  499. return 0;
  500. /* Clear frozen state for slave PEs */
  501. list_for_each_entry(slave, &pe->slaves, list) {
  502. rc = opal_pci_eeh_freeze_clear(phb->opal_id,
  503. slave->pe_number,
  504. opt);
  505. if (rc != OPAL_SUCCESS) {
  506. pr_warn("%s: Failure %lld clear %d on PHB#%x-PE#%x\n",
  507. __func__, rc, opt, phb->hose->global_number,
  508. slave->pe_number);
  509. return -EIO;
  510. }
  511. }
  512. return 0;
  513. }
  514. static int pnv_ioda_get_pe_state(struct pnv_phb *phb, int pe_no)
  515. {
  516. struct pnv_ioda_pe *slave, *pe;
  517. u8 fstate = 0, state;
  518. __be16 pcierr = 0;
  519. s64 rc;
  520. /* Sanity check on PE number */
  521. if (pe_no < 0 || pe_no >= phb->ioda.total_pe_num)
  522. return OPAL_EEH_STOPPED_PERM_UNAVAIL;
  523. /*
  524. * Fetch the master PE and the PE instance might be
  525. * not initialized yet.
  526. */
  527. pe = &phb->ioda.pe_array[pe_no];
  528. if (pe->flags & PNV_IODA_PE_SLAVE) {
  529. pe = pe->master;
  530. WARN_ON(!pe || !(pe->flags & PNV_IODA_PE_MASTER));
  531. pe_no = pe->pe_number;
  532. }
  533. /* Check the master PE */
  534. rc = opal_pci_eeh_freeze_status(phb->opal_id, pe_no,
  535. &state, &pcierr, NULL);
  536. if (rc != OPAL_SUCCESS) {
  537. pr_warn("%s: Failure %lld getting "
  538. "PHB#%x-PE#%x state\n",
  539. __func__, rc,
  540. phb->hose->global_number, pe_no);
  541. return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
  542. }
  543. /* Check the slave PE */
  544. if (!(pe->flags & PNV_IODA_PE_MASTER))
  545. return state;
  546. list_for_each_entry(slave, &pe->slaves, list) {
  547. rc = opal_pci_eeh_freeze_status(phb->opal_id,
  548. slave->pe_number,
  549. &fstate,
  550. &pcierr,
  551. NULL);
  552. if (rc != OPAL_SUCCESS) {
  553. pr_warn("%s: Failure %lld getting "
  554. "PHB#%x-PE#%x state\n",
  555. __func__, rc,
  556. phb->hose->global_number, slave->pe_number);
  557. return OPAL_EEH_STOPPED_TEMP_UNAVAIL;
  558. }
  559. /*
  560. * Override the result based on the ascending
  561. * priority.
  562. */
  563. if (fstate > state)
  564. state = fstate;
  565. }
  566. return state;
  567. }
  568. /* Currently those 2 are only used when MSIs are enabled, this will change
  569. * but in the meantime, we need to protect them to avoid warnings
  570. */
  571. #ifdef CONFIG_PCI_MSI
  572. struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
  573. {
  574. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  575. struct pnv_phb *phb = hose->private_data;
  576. struct pci_dn *pdn = pci_get_pdn(dev);
  577. if (!pdn)
  578. return NULL;
  579. if (pdn->pe_number == IODA_INVALID_PE)
  580. return NULL;
  581. return &phb->ioda.pe_array[pdn->pe_number];
  582. }
  583. #endif /* CONFIG_PCI_MSI */
  584. static int pnv_ioda_set_one_peltv(struct pnv_phb *phb,
  585. struct pnv_ioda_pe *parent,
  586. struct pnv_ioda_pe *child,
  587. bool is_add)
  588. {
  589. const char *desc = is_add ? "adding" : "removing";
  590. uint8_t op = is_add ? OPAL_ADD_PE_TO_DOMAIN :
  591. OPAL_REMOVE_PE_FROM_DOMAIN;
  592. struct pnv_ioda_pe *slave;
  593. long rc;
  594. /* Parent PE affects child PE */
  595. rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
  596. child->pe_number, op);
  597. if (rc != OPAL_SUCCESS) {
  598. pe_warn(child, "OPAL error %ld %s to parent PELTV\n",
  599. rc, desc);
  600. return -ENXIO;
  601. }
  602. if (!(child->flags & PNV_IODA_PE_MASTER))
  603. return 0;
  604. /* Compound case: parent PE affects slave PEs */
  605. list_for_each_entry(slave, &child->slaves, list) {
  606. rc = opal_pci_set_peltv(phb->opal_id, parent->pe_number,
  607. slave->pe_number, op);
  608. if (rc != OPAL_SUCCESS) {
  609. pe_warn(slave, "OPAL error %ld %s to parent PELTV\n",
  610. rc, desc);
  611. return -ENXIO;
  612. }
  613. }
  614. return 0;
  615. }
  616. static int pnv_ioda_set_peltv(struct pnv_phb *phb,
  617. struct pnv_ioda_pe *pe,
  618. bool is_add)
  619. {
  620. struct pnv_ioda_pe *slave;
  621. struct pci_dev *pdev = NULL;
  622. int ret;
  623. /*
  624. * Clear PE frozen state. If it's master PE, we need
  625. * clear slave PE frozen state as well.
  626. */
  627. if (is_add) {
  628. opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
  629. OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
  630. if (pe->flags & PNV_IODA_PE_MASTER) {
  631. list_for_each_entry(slave, &pe->slaves, list)
  632. opal_pci_eeh_freeze_clear(phb->opal_id,
  633. slave->pe_number,
  634. OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
  635. }
  636. }
  637. /*
  638. * Associate PE in PELT. We need add the PE into the
  639. * corresponding PELT-V as well. Otherwise, the error
  640. * originated from the PE might contribute to other
  641. * PEs.
  642. */
  643. ret = pnv_ioda_set_one_peltv(phb, pe, pe, is_add);
  644. if (ret)
  645. return ret;
  646. /* For compound PEs, any one affects all of them */
  647. if (pe->flags & PNV_IODA_PE_MASTER) {
  648. list_for_each_entry(slave, &pe->slaves, list) {
  649. ret = pnv_ioda_set_one_peltv(phb, slave, pe, is_add);
  650. if (ret)
  651. return ret;
  652. }
  653. }
  654. if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS))
  655. pdev = pe->pbus->self;
  656. else if (pe->flags & PNV_IODA_PE_DEV)
  657. pdev = pe->pdev->bus->self;
  658. #ifdef CONFIG_PCI_IOV
  659. else if (pe->flags & PNV_IODA_PE_VF)
  660. pdev = pe->parent_dev;
  661. #endif /* CONFIG_PCI_IOV */
  662. while (pdev) {
  663. struct pci_dn *pdn = pci_get_pdn(pdev);
  664. struct pnv_ioda_pe *parent;
  665. if (pdn && pdn->pe_number != IODA_INVALID_PE) {
  666. parent = &phb->ioda.pe_array[pdn->pe_number];
  667. ret = pnv_ioda_set_one_peltv(phb, parent, pe, is_add);
  668. if (ret)
  669. return ret;
  670. }
  671. pdev = pdev->bus->self;
  672. }
  673. return 0;
  674. }
  675. static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
  676. {
  677. struct pci_dev *parent;
  678. uint8_t bcomp, dcomp, fcomp;
  679. int64_t rc;
  680. long rid_end, rid;
  681. /* Currently, we just deconfigure VF PE. Bus PE will always there.*/
  682. if (pe->pbus) {
  683. int count;
  684. dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
  685. fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
  686. parent = pe->pbus->self;
  687. if (pe->flags & PNV_IODA_PE_BUS_ALL)
  688. count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
  689. else
  690. count = 1;
  691. switch(count) {
  692. case 1: bcomp = OpalPciBusAll; break;
  693. case 2: bcomp = OpalPciBus7Bits; break;
  694. case 4: bcomp = OpalPciBus6Bits; break;
  695. case 8: bcomp = OpalPciBus5Bits; break;
  696. case 16: bcomp = OpalPciBus4Bits; break;
  697. case 32: bcomp = OpalPciBus3Bits; break;
  698. default:
  699. dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
  700. count);
  701. /* Do an exact match only */
  702. bcomp = OpalPciBusAll;
  703. }
  704. rid_end = pe->rid + (count << 8);
  705. } else {
  706. #ifdef CONFIG_PCI_IOV
  707. if (pe->flags & PNV_IODA_PE_VF)
  708. parent = pe->parent_dev;
  709. else
  710. #endif
  711. parent = pe->pdev->bus->self;
  712. bcomp = OpalPciBusAll;
  713. dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
  714. fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
  715. rid_end = pe->rid + 1;
  716. }
  717. /* Clear the reverse map */
  718. for (rid = pe->rid; rid < rid_end; rid++)
  719. phb->ioda.pe_rmap[rid] = IODA_INVALID_PE;
  720. /* Release from all parents PELT-V */
  721. while (parent) {
  722. struct pci_dn *pdn = pci_get_pdn(parent);
  723. if (pdn && pdn->pe_number != IODA_INVALID_PE) {
  724. rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
  725. pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
  726. /* XXX What to do in case of error ? */
  727. }
  728. parent = parent->bus->self;
  729. }
  730. opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number,
  731. OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
  732. /* Disassociate PE in PELT */
  733. rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number,
  734. pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
  735. if (rc)
  736. pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc);
  737. rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
  738. bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
  739. if (rc)
  740. pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
  741. pe->pbus = NULL;
  742. pe->pdev = NULL;
  743. #ifdef CONFIG_PCI_IOV
  744. pe->parent_dev = NULL;
  745. #endif
  746. return 0;
  747. }
  748. static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
  749. {
  750. struct pci_dev *parent;
  751. uint8_t bcomp, dcomp, fcomp;
  752. long rc, rid_end, rid;
  753. /* Bus validation ? */
  754. if (pe->pbus) {
  755. int count;
  756. dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
  757. fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
  758. parent = pe->pbus->self;
  759. if (pe->flags & PNV_IODA_PE_BUS_ALL)
  760. count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
  761. else
  762. count = 1;
  763. switch(count) {
  764. case 1: bcomp = OpalPciBusAll; break;
  765. case 2: bcomp = OpalPciBus7Bits; break;
  766. case 4: bcomp = OpalPciBus6Bits; break;
  767. case 8: bcomp = OpalPciBus5Bits; break;
  768. case 16: bcomp = OpalPciBus4Bits; break;
  769. case 32: bcomp = OpalPciBus3Bits; break;
  770. default:
  771. dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n",
  772. count);
  773. /* Do an exact match only */
  774. bcomp = OpalPciBusAll;
  775. }
  776. rid_end = pe->rid + (count << 8);
  777. } else {
  778. #ifdef CONFIG_PCI_IOV
  779. if (pe->flags & PNV_IODA_PE_VF)
  780. parent = pe->parent_dev;
  781. else
  782. #endif /* CONFIG_PCI_IOV */
  783. parent = pe->pdev->bus->self;
  784. bcomp = OpalPciBusAll;
  785. dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
  786. fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
  787. rid_end = pe->rid + 1;
  788. }
  789. /*
  790. * Associate PE in PELT. We need add the PE into the
  791. * corresponding PELT-V as well. Otherwise, the error
  792. * originated from the PE might contribute to other
  793. * PEs.
  794. */
  795. rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
  796. bcomp, dcomp, fcomp, OPAL_MAP_PE);
  797. if (rc) {
  798. pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
  799. return -ENXIO;
  800. }
  801. /*
  802. * Configure PELTV. NPUs don't have a PELTV table so skip
  803. * configuration on them.
  804. */
  805. if (phb->type != PNV_PHB_NPU_NVLINK && phb->type != PNV_PHB_NPU_OCAPI)
  806. pnv_ioda_set_peltv(phb, pe, true);
  807. /* Setup reverse map */
  808. for (rid = pe->rid; rid < rid_end; rid++)
  809. phb->ioda.pe_rmap[rid] = pe->pe_number;
  810. /* Setup one MVTs on IODA1 */
  811. if (phb->type != PNV_PHB_IODA1) {
  812. pe->mve_number = 0;
  813. goto out;
  814. }
  815. pe->mve_number = pe->pe_number;
  816. rc = opal_pci_set_mve(phb->opal_id, pe->mve_number, pe->pe_number);
  817. if (rc != OPAL_SUCCESS) {
  818. pe_err(pe, "OPAL error %ld setting up MVE %x\n",
  819. rc, pe->mve_number);
  820. pe->mve_number = -1;
  821. } else {
  822. rc = opal_pci_set_mve_enable(phb->opal_id,
  823. pe->mve_number, OPAL_ENABLE_MVE);
  824. if (rc) {
  825. pe_err(pe, "OPAL error %ld enabling MVE %x\n",
  826. rc, pe->mve_number);
  827. pe->mve_number = -1;
  828. }
  829. }
  830. out:
  831. return 0;
  832. }
  833. #ifdef CONFIG_PCI_IOV
  834. static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
  835. {
  836. struct pci_dn *pdn = pci_get_pdn(dev);
  837. int i;
  838. struct resource *res, res2;
  839. resource_size_t size;
  840. u16 num_vfs;
  841. if (!dev->is_physfn)
  842. return -EINVAL;
  843. /*
  844. * "offset" is in VFs. The M64 windows are sized so that when they
  845. * are segmented, each segment is the same size as the IOV BAR.
  846. * Each segment is in a separate PE, and the high order bits of the
  847. * address are the PE number. Therefore, each VF's BAR is in a
  848. * separate PE, and changing the IOV BAR start address changes the
  849. * range of PEs the VFs are in.
  850. */
  851. num_vfs = pdn->num_vfs;
  852. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  853. res = &dev->resource[i + PCI_IOV_RESOURCES];
  854. if (!res->flags || !res->parent)
  855. continue;
  856. /*
  857. * The actual IOV BAR range is determined by the start address
  858. * and the actual size for num_vfs VFs BAR. This check is to
  859. * make sure that after shifting, the range will not overlap
  860. * with another device.
  861. */
  862. size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
  863. res2.flags = res->flags;
  864. res2.start = res->start + (size * offset);
  865. res2.end = res2.start + (size * num_vfs) - 1;
  866. if (res2.end > res->end) {
  867. dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
  868. i, &res2, res, num_vfs, offset);
  869. return -EBUSY;
  870. }
  871. }
  872. /*
  873. * Since M64 BAR shares segments among all possible 256 PEs,
  874. * we have to shift the beginning of PF IOV BAR to make it start from
  875. * the segment which belongs to the PE number assigned to the first VF.
  876. * This creates a "hole" in the /proc/iomem which could be used for
  877. * allocating other resources so we reserve this area below and
  878. * release when IOV is released.
  879. */
  880. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  881. res = &dev->resource[i + PCI_IOV_RESOURCES];
  882. if (!res->flags || !res->parent)
  883. continue;
  884. size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
  885. res2 = *res;
  886. res->start += size * offset;
  887. dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (%sabling %d VFs shifted by %d)\n",
  888. i, &res2, res, (offset > 0) ? "En" : "Dis",
  889. num_vfs, offset);
  890. if (offset < 0) {
  891. devm_release_resource(&dev->dev, &pdn->holes[i]);
  892. memset(&pdn->holes[i], 0, sizeof(pdn->holes[i]));
  893. }
  894. pci_update_resource(dev, i + PCI_IOV_RESOURCES);
  895. if (offset > 0) {
  896. pdn->holes[i].start = res2.start;
  897. pdn->holes[i].end = res2.start + size * offset - 1;
  898. pdn->holes[i].flags = IORESOURCE_BUS;
  899. pdn->holes[i].name = "pnv_iov_reserved";
  900. devm_request_resource(&dev->dev, res->parent,
  901. &pdn->holes[i]);
  902. }
  903. }
  904. return 0;
  905. }
  906. #endif /* CONFIG_PCI_IOV */
  907. static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
  908. {
  909. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  910. struct pnv_phb *phb = hose->private_data;
  911. struct pci_dn *pdn = pci_get_pdn(dev);
  912. struct pnv_ioda_pe *pe;
  913. if (!pdn) {
  914. pr_err("%s: Device tree node not associated properly\n",
  915. pci_name(dev));
  916. return NULL;
  917. }
  918. if (pdn->pe_number != IODA_INVALID_PE)
  919. return NULL;
  920. pe = pnv_ioda_alloc_pe(phb);
  921. if (!pe) {
  922. pr_warn("%s: Not enough PE# available, disabling device\n",
  923. pci_name(dev));
  924. return NULL;
  925. }
  926. /* NOTE: We get only one ref to the pci_dev for the pdn, not for the
  927. * pointer in the PE data structure, both should be destroyed at the
  928. * same time. However, this needs to be looked at more closely again
  929. * once we actually start removing things (Hotplug, SR-IOV, ...)
  930. *
  931. * At some point we want to remove the PDN completely anyways
  932. */
  933. pci_dev_get(dev);
  934. pdn->pe_number = pe->pe_number;
  935. pe->flags = PNV_IODA_PE_DEV;
  936. pe->pdev = dev;
  937. pe->pbus = NULL;
  938. pe->mve_number = -1;
  939. pe->rid = dev->bus->number << 8 | pdn->devfn;
  940. pe_info(pe, "Associated device to PE\n");
  941. if (pnv_ioda_configure_pe(phb, pe)) {
  942. /* XXX What do we do here ? */
  943. pnv_ioda_free_pe(pe);
  944. pdn->pe_number = IODA_INVALID_PE;
  945. pe->pdev = NULL;
  946. pci_dev_put(dev);
  947. return NULL;
  948. }
  949. /* Put PE to the list */
  950. list_add_tail(&pe->list, &phb->ioda.pe_list);
  951. return pe;
  952. }
  953. static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
  954. {
  955. struct pci_dev *dev;
  956. list_for_each_entry(dev, &bus->devices, bus_list) {
  957. struct pci_dn *pdn = pci_get_pdn(dev);
  958. if (pdn == NULL) {
  959. pr_warn("%s: No device node associated with device !\n",
  960. pci_name(dev));
  961. continue;
  962. }
  963. /*
  964. * In partial hotplug case, the PCI device might be still
  965. * associated with the PE and needn't attach it to the PE
  966. * again.
  967. */
  968. if (pdn->pe_number != IODA_INVALID_PE)
  969. continue;
  970. pe->device_count++;
  971. pdn->pe_number = pe->pe_number;
  972. if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
  973. pnv_ioda_setup_same_PE(dev->subordinate, pe);
  974. }
  975. }
  976. /*
  977. * There're 2 types of PCI bus sensitive PEs: One that is compromised of
  978. * single PCI bus. Another one that contains the primary PCI bus and its
  979. * subordinate PCI devices and buses. The second type of PE is normally
  980. * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports.
  981. */
  982. static struct pnv_ioda_pe *pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all)
  983. {
  984. struct pci_controller *hose = pci_bus_to_host(bus);
  985. struct pnv_phb *phb = hose->private_data;
  986. struct pnv_ioda_pe *pe = NULL;
  987. unsigned int pe_num;
  988. /*
  989. * In partial hotplug case, the PE instance might be still alive.
  990. * We should reuse it instead of allocating a new one.
  991. */
  992. pe_num = phb->ioda.pe_rmap[bus->number << 8];
  993. if (pe_num != IODA_INVALID_PE) {
  994. pe = &phb->ioda.pe_array[pe_num];
  995. pnv_ioda_setup_same_PE(bus, pe);
  996. return NULL;
  997. }
  998. /* PE number for root bus should have been reserved */
  999. if (pci_is_root_bus(bus) &&
  1000. phb->ioda.root_pe_idx != IODA_INVALID_PE)
  1001. pe = &phb->ioda.pe_array[phb->ioda.root_pe_idx];
  1002. /* Check if PE is determined by M64 */
  1003. if (!pe && phb->pick_m64_pe)
  1004. pe = phb->pick_m64_pe(bus, all);
  1005. /* The PE number isn't pinned by M64 */
  1006. if (!pe)
  1007. pe = pnv_ioda_alloc_pe(phb);
  1008. if (!pe) {
  1009. pr_warn("%s: Not enough PE# available for PCI bus %04x:%02x\n",
  1010. __func__, pci_domain_nr(bus), bus->number);
  1011. return NULL;
  1012. }
  1013. pe->flags |= (all ? PNV_IODA_PE_BUS_ALL : PNV_IODA_PE_BUS);
  1014. pe->pbus = bus;
  1015. pe->pdev = NULL;
  1016. pe->mve_number = -1;
  1017. pe->rid = bus->busn_res.start << 8;
  1018. if (all)
  1019. pe_info(pe, "Secondary bus %d..%d associated with PE#%x\n",
  1020. bus->busn_res.start, bus->busn_res.end, pe->pe_number);
  1021. else
  1022. pe_info(pe, "Secondary bus %d associated with PE#%x\n",
  1023. bus->busn_res.start, pe->pe_number);
  1024. if (pnv_ioda_configure_pe(phb, pe)) {
  1025. /* XXX What do we do here ? */
  1026. pnv_ioda_free_pe(pe);
  1027. pe->pbus = NULL;
  1028. return NULL;
  1029. }
  1030. /* Associate it with all child devices */
  1031. pnv_ioda_setup_same_PE(bus, pe);
  1032. /* Put PE to the list */
  1033. list_add_tail(&pe->list, &phb->ioda.pe_list);
  1034. return pe;
  1035. }
  1036. static struct pnv_ioda_pe *pnv_ioda_setup_npu_PE(struct pci_dev *npu_pdev)
  1037. {
  1038. int pe_num, found_pe = false, rc;
  1039. long rid;
  1040. struct pnv_ioda_pe *pe;
  1041. struct pci_dev *gpu_pdev;
  1042. struct pci_dn *npu_pdn;
  1043. struct pci_controller *hose = pci_bus_to_host(npu_pdev->bus);
  1044. struct pnv_phb *phb = hose->private_data;
  1045. /*
  1046. * Due to a hardware errata PE#0 on the NPU is reserved for
  1047. * error handling. This means we only have three PEs remaining
  1048. * which need to be assigned to four links, implying some
  1049. * links must share PEs.
  1050. *
  1051. * To achieve this we assign PEs such that NPUs linking the
  1052. * same GPU get assigned the same PE.
  1053. */
  1054. gpu_pdev = pnv_pci_get_gpu_dev(npu_pdev);
  1055. for (pe_num = 0; pe_num < phb->ioda.total_pe_num; pe_num++) {
  1056. pe = &phb->ioda.pe_array[pe_num];
  1057. if (!pe->pdev)
  1058. continue;
  1059. if (pnv_pci_get_gpu_dev(pe->pdev) == gpu_pdev) {
  1060. /*
  1061. * This device has the same peer GPU so should
  1062. * be assigned the same PE as the existing
  1063. * peer NPU.
  1064. */
  1065. dev_info(&npu_pdev->dev,
  1066. "Associating to existing PE %x\n", pe_num);
  1067. pci_dev_get(npu_pdev);
  1068. npu_pdn = pci_get_pdn(npu_pdev);
  1069. rid = npu_pdev->bus->number << 8 | npu_pdn->devfn;
  1070. npu_pdn->pe_number = pe_num;
  1071. phb->ioda.pe_rmap[rid] = pe->pe_number;
  1072. /* Map the PE to this link */
  1073. rc = opal_pci_set_pe(phb->opal_id, pe_num, rid,
  1074. OpalPciBusAll,
  1075. OPAL_COMPARE_RID_DEVICE_NUMBER,
  1076. OPAL_COMPARE_RID_FUNCTION_NUMBER,
  1077. OPAL_MAP_PE);
  1078. WARN_ON(rc != OPAL_SUCCESS);
  1079. found_pe = true;
  1080. break;
  1081. }
  1082. }
  1083. if (!found_pe)
  1084. /*
  1085. * Could not find an existing PE so allocate a new
  1086. * one.
  1087. */
  1088. return pnv_ioda_setup_dev_PE(npu_pdev);
  1089. else
  1090. return pe;
  1091. }
  1092. static void pnv_ioda_setup_npu_PEs(struct pci_bus *bus)
  1093. {
  1094. struct pci_dev *pdev;
  1095. list_for_each_entry(pdev, &bus->devices, bus_list)
  1096. pnv_ioda_setup_npu_PE(pdev);
  1097. }
  1098. static void pnv_pci_ioda_setup_PEs(void)
  1099. {
  1100. struct pci_controller *hose, *tmp;
  1101. struct pnv_phb *phb;
  1102. struct pci_bus *bus;
  1103. struct pci_dev *pdev;
  1104. list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
  1105. phb = hose->private_data;
  1106. if (phb->type == PNV_PHB_NPU_NVLINK) {
  1107. /* PE#0 is needed for error reporting */
  1108. pnv_ioda_reserve_pe(phb, 0);
  1109. pnv_ioda_setup_npu_PEs(hose->bus);
  1110. if (phb->model == PNV_PHB_MODEL_NPU2)
  1111. pnv_npu2_init(phb);
  1112. }
  1113. if (phb->type == PNV_PHB_NPU_OCAPI) {
  1114. bus = hose->bus;
  1115. list_for_each_entry(pdev, &bus->devices, bus_list)
  1116. pnv_ioda_setup_dev_PE(pdev);
  1117. }
  1118. }
  1119. }
  1120. #ifdef CONFIG_PCI_IOV
  1121. static int pnv_pci_vf_release_m64(struct pci_dev *pdev, u16 num_vfs)
  1122. {
  1123. struct pci_bus *bus;
  1124. struct pci_controller *hose;
  1125. struct pnv_phb *phb;
  1126. struct pci_dn *pdn;
  1127. int i, j;
  1128. int m64_bars;
  1129. bus = pdev->bus;
  1130. hose = pci_bus_to_host(bus);
  1131. phb = hose->private_data;
  1132. pdn = pci_get_pdn(pdev);
  1133. if (pdn->m64_single_mode)
  1134. m64_bars = num_vfs;
  1135. else
  1136. m64_bars = 1;
  1137. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
  1138. for (j = 0; j < m64_bars; j++) {
  1139. if (pdn->m64_map[j][i] == IODA_INVALID_M64)
  1140. continue;
  1141. opal_pci_phb_mmio_enable(phb->opal_id,
  1142. OPAL_M64_WINDOW_TYPE, pdn->m64_map[j][i], 0);
  1143. clear_bit(pdn->m64_map[j][i], &phb->ioda.m64_bar_alloc);
  1144. pdn->m64_map[j][i] = IODA_INVALID_M64;
  1145. }
  1146. kfree(pdn->m64_map);
  1147. return 0;
  1148. }
  1149. static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
  1150. {
  1151. struct pci_bus *bus;
  1152. struct pci_controller *hose;
  1153. struct pnv_phb *phb;
  1154. struct pci_dn *pdn;
  1155. unsigned int win;
  1156. struct resource *res;
  1157. int i, j;
  1158. int64_t rc;
  1159. int total_vfs;
  1160. resource_size_t size, start;
  1161. int pe_num;
  1162. int m64_bars;
  1163. bus = pdev->bus;
  1164. hose = pci_bus_to_host(bus);
  1165. phb = hose->private_data;
  1166. pdn = pci_get_pdn(pdev);
  1167. total_vfs = pci_sriov_get_totalvfs(pdev);
  1168. if (pdn->m64_single_mode)
  1169. m64_bars = num_vfs;
  1170. else
  1171. m64_bars = 1;
  1172. pdn->m64_map = kmalloc_array(m64_bars,
  1173. sizeof(*pdn->m64_map),
  1174. GFP_KERNEL);
  1175. if (!pdn->m64_map)
  1176. return -ENOMEM;
  1177. /* Initialize the m64_map to IODA_INVALID_M64 */
  1178. for (i = 0; i < m64_bars ; i++)
  1179. for (j = 0; j < PCI_SRIOV_NUM_BARS; j++)
  1180. pdn->m64_map[i][j] = IODA_INVALID_M64;
  1181. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  1182. res = &pdev->resource[i + PCI_IOV_RESOURCES];
  1183. if (!res->flags || !res->parent)
  1184. continue;
  1185. for (j = 0; j < m64_bars; j++) {
  1186. do {
  1187. win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
  1188. phb->ioda.m64_bar_idx + 1, 0);
  1189. if (win >= phb->ioda.m64_bar_idx + 1)
  1190. goto m64_failed;
  1191. } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
  1192. pdn->m64_map[j][i] = win;
  1193. if (pdn->m64_single_mode) {
  1194. size = pci_iov_resource_size(pdev,
  1195. PCI_IOV_RESOURCES + i);
  1196. start = res->start + size * j;
  1197. } else {
  1198. size = resource_size(res);
  1199. start = res->start;
  1200. }
  1201. /* Map the M64 here */
  1202. if (pdn->m64_single_mode) {
  1203. pe_num = pdn->pe_num_map[j];
  1204. rc = opal_pci_map_pe_mmio_window(phb->opal_id,
  1205. pe_num, OPAL_M64_WINDOW_TYPE,
  1206. pdn->m64_map[j][i], 0);
  1207. }
  1208. rc = opal_pci_set_phb_mem_window(phb->opal_id,
  1209. OPAL_M64_WINDOW_TYPE,
  1210. pdn->m64_map[j][i],
  1211. start,
  1212. 0, /* unused */
  1213. size);
  1214. if (rc != OPAL_SUCCESS) {
  1215. dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n",
  1216. win, rc);
  1217. goto m64_failed;
  1218. }
  1219. if (pdn->m64_single_mode)
  1220. rc = opal_pci_phb_mmio_enable(phb->opal_id,
  1221. OPAL_M64_WINDOW_TYPE, pdn->m64_map[j][i], 2);
  1222. else
  1223. rc = opal_pci_phb_mmio_enable(phb->opal_id,
  1224. OPAL_M64_WINDOW_TYPE, pdn->m64_map[j][i], 1);
  1225. if (rc != OPAL_SUCCESS) {
  1226. dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n",
  1227. win, rc);
  1228. goto m64_failed;
  1229. }
  1230. }
  1231. }
  1232. return 0;
  1233. m64_failed:
  1234. pnv_pci_vf_release_m64(pdev, num_vfs);
  1235. return -EBUSY;
  1236. }
  1237. static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
  1238. int num);
  1239. static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe)
  1240. {
  1241. struct iommu_table *tbl;
  1242. int64_t rc;
  1243. tbl = pe->table_group.tables[0];
  1244. rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
  1245. if (rc)
  1246. pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
  1247. pnv_pci_ioda2_set_bypass(pe, false);
  1248. if (pe->table_group.group) {
  1249. iommu_group_put(pe->table_group.group);
  1250. BUG_ON(pe->table_group.group);
  1251. }
  1252. iommu_tce_table_put(tbl);
  1253. }
  1254. static void pnv_ioda_release_vf_PE(struct pci_dev *pdev)
  1255. {
  1256. struct pci_bus *bus;
  1257. struct pci_controller *hose;
  1258. struct pnv_phb *phb;
  1259. struct pnv_ioda_pe *pe, *pe_n;
  1260. struct pci_dn *pdn;
  1261. bus = pdev->bus;
  1262. hose = pci_bus_to_host(bus);
  1263. phb = hose->private_data;
  1264. pdn = pci_get_pdn(pdev);
  1265. if (!pdev->is_physfn)
  1266. return;
  1267. list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
  1268. if (pe->parent_dev != pdev)
  1269. continue;
  1270. pnv_pci_ioda2_release_dma_pe(pdev, pe);
  1271. /* Remove from list */
  1272. mutex_lock(&phb->ioda.pe_list_mutex);
  1273. list_del(&pe->list);
  1274. mutex_unlock(&phb->ioda.pe_list_mutex);
  1275. pnv_ioda_deconfigure_pe(phb, pe);
  1276. pnv_ioda_free_pe(pe);
  1277. }
  1278. }
  1279. void pnv_pci_sriov_disable(struct pci_dev *pdev)
  1280. {
  1281. struct pci_bus *bus;
  1282. struct pci_controller *hose;
  1283. struct pnv_phb *phb;
  1284. struct pnv_ioda_pe *pe;
  1285. struct pci_dn *pdn;
  1286. u16 num_vfs, i;
  1287. bus = pdev->bus;
  1288. hose = pci_bus_to_host(bus);
  1289. phb = hose->private_data;
  1290. pdn = pci_get_pdn(pdev);
  1291. num_vfs = pdn->num_vfs;
  1292. /* Release VF PEs */
  1293. pnv_ioda_release_vf_PE(pdev);
  1294. if (phb->type == PNV_PHB_IODA2) {
  1295. if (!pdn->m64_single_mode)
  1296. pnv_pci_vf_resource_shift(pdev, -*pdn->pe_num_map);
  1297. /* Release M64 windows */
  1298. pnv_pci_vf_release_m64(pdev, num_vfs);
  1299. /* Release PE numbers */
  1300. if (pdn->m64_single_mode) {
  1301. for (i = 0; i < num_vfs; i++) {
  1302. if (pdn->pe_num_map[i] == IODA_INVALID_PE)
  1303. continue;
  1304. pe = &phb->ioda.pe_array[pdn->pe_num_map[i]];
  1305. pnv_ioda_free_pe(pe);
  1306. }
  1307. } else
  1308. bitmap_clear(phb->ioda.pe_alloc, *pdn->pe_num_map, num_vfs);
  1309. /* Releasing pe_num_map */
  1310. kfree(pdn->pe_num_map);
  1311. }
  1312. }
  1313. static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
  1314. struct pnv_ioda_pe *pe);
  1315. static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
  1316. {
  1317. struct pci_bus *bus;
  1318. struct pci_controller *hose;
  1319. struct pnv_phb *phb;
  1320. struct pnv_ioda_pe *pe;
  1321. int pe_num;
  1322. u16 vf_index;
  1323. struct pci_dn *pdn;
  1324. bus = pdev->bus;
  1325. hose = pci_bus_to_host(bus);
  1326. phb = hose->private_data;
  1327. pdn = pci_get_pdn(pdev);
  1328. if (!pdev->is_physfn)
  1329. return;
  1330. /* Reserve PE for each VF */
  1331. for (vf_index = 0; vf_index < num_vfs; vf_index++) {
  1332. int vf_devfn = pci_iov_virtfn_devfn(pdev, vf_index);
  1333. int vf_bus = pci_iov_virtfn_bus(pdev, vf_index);
  1334. struct pci_dn *vf_pdn;
  1335. if (pdn->m64_single_mode)
  1336. pe_num = pdn->pe_num_map[vf_index];
  1337. else
  1338. pe_num = *pdn->pe_num_map + vf_index;
  1339. pe = &phb->ioda.pe_array[pe_num];
  1340. pe->pe_number = pe_num;
  1341. pe->phb = phb;
  1342. pe->flags = PNV_IODA_PE_VF;
  1343. pe->pbus = NULL;
  1344. pe->parent_dev = pdev;
  1345. pe->mve_number = -1;
  1346. pe->rid = (vf_bus << 8) | vf_devfn;
  1347. pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%x\n",
  1348. hose->global_number, pdev->bus->number,
  1349. PCI_SLOT(vf_devfn), PCI_FUNC(vf_devfn), pe_num);
  1350. if (pnv_ioda_configure_pe(phb, pe)) {
  1351. /* XXX What do we do here ? */
  1352. pnv_ioda_free_pe(pe);
  1353. pe->pdev = NULL;
  1354. continue;
  1355. }
  1356. /* Put PE to the list */
  1357. mutex_lock(&phb->ioda.pe_list_mutex);
  1358. list_add_tail(&pe->list, &phb->ioda.pe_list);
  1359. mutex_unlock(&phb->ioda.pe_list_mutex);
  1360. /* associate this pe to it's pdn */
  1361. list_for_each_entry(vf_pdn, &pdn->parent->child_list, list) {
  1362. if (vf_pdn->busno == vf_bus &&
  1363. vf_pdn->devfn == vf_devfn) {
  1364. vf_pdn->pe_number = pe_num;
  1365. break;
  1366. }
  1367. }
  1368. pnv_pci_ioda2_setup_dma_pe(phb, pe);
  1369. }
  1370. }
  1371. int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
  1372. {
  1373. struct pci_bus *bus;
  1374. struct pci_controller *hose;
  1375. struct pnv_phb *phb;
  1376. struct pnv_ioda_pe *pe;
  1377. struct pci_dn *pdn;
  1378. int ret;
  1379. u16 i;
  1380. bus = pdev->bus;
  1381. hose = pci_bus_to_host(bus);
  1382. phb = hose->private_data;
  1383. pdn = pci_get_pdn(pdev);
  1384. if (phb->type == PNV_PHB_IODA2) {
  1385. if (!pdn->vfs_expanded) {
  1386. dev_info(&pdev->dev, "don't support this SRIOV device"
  1387. " with non 64bit-prefetchable IOV BAR\n");
  1388. return -ENOSPC;
  1389. }
  1390. /*
  1391. * When M64 BARs functions in Single PE mode, the number of VFs
  1392. * could be enabled must be less than the number of M64 BARs.
  1393. */
  1394. if (pdn->m64_single_mode && num_vfs > phb->ioda.m64_bar_idx) {
  1395. dev_info(&pdev->dev, "Not enough M64 BAR for VFs\n");
  1396. return -EBUSY;
  1397. }
  1398. /* Allocating pe_num_map */
  1399. if (pdn->m64_single_mode)
  1400. pdn->pe_num_map = kmalloc_array(num_vfs,
  1401. sizeof(*pdn->pe_num_map),
  1402. GFP_KERNEL);
  1403. else
  1404. pdn->pe_num_map = kmalloc(sizeof(*pdn->pe_num_map), GFP_KERNEL);
  1405. if (!pdn->pe_num_map)
  1406. return -ENOMEM;
  1407. if (pdn->m64_single_mode)
  1408. for (i = 0; i < num_vfs; i++)
  1409. pdn->pe_num_map[i] = IODA_INVALID_PE;
  1410. /* Calculate available PE for required VFs */
  1411. if (pdn->m64_single_mode) {
  1412. for (i = 0; i < num_vfs; i++) {
  1413. pe = pnv_ioda_alloc_pe(phb);
  1414. if (!pe) {
  1415. ret = -EBUSY;
  1416. goto m64_failed;
  1417. }
  1418. pdn->pe_num_map[i] = pe->pe_number;
  1419. }
  1420. } else {
  1421. mutex_lock(&phb->ioda.pe_alloc_mutex);
  1422. *pdn->pe_num_map = bitmap_find_next_zero_area(
  1423. phb->ioda.pe_alloc, phb->ioda.total_pe_num,
  1424. 0, num_vfs, 0);
  1425. if (*pdn->pe_num_map >= phb->ioda.total_pe_num) {
  1426. mutex_unlock(&phb->ioda.pe_alloc_mutex);
  1427. dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs);
  1428. kfree(pdn->pe_num_map);
  1429. return -EBUSY;
  1430. }
  1431. bitmap_set(phb->ioda.pe_alloc, *pdn->pe_num_map, num_vfs);
  1432. mutex_unlock(&phb->ioda.pe_alloc_mutex);
  1433. }
  1434. pdn->num_vfs = num_vfs;
  1435. /* Assign M64 window accordingly */
  1436. ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
  1437. if (ret) {
  1438. dev_info(&pdev->dev, "Not enough M64 window resources\n");
  1439. goto m64_failed;
  1440. }
  1441. /*
  1442. * When using one M64 BAR to map one IOV BAR, we need to shift
  1443. * the IOV BAR according to the PE# allocated to the VFs.
  1444. * Otherwise, the PE# for the VF will conflict with others.
  1445. */
  1446. if (!pdn->m64_single_mode) {
  1447. ret = pnv_pci_vf_resource_shift(pdev, *pdn->pe_num_map);
  1448. if (ret)
  1449. goto m64_failed;
  1450. }
  1451. }
  1452. /* Setup VF PEs */
  1453. pnv_ioda_setup_vf_PE(pdev, num_vfs);
  1454. return 0;
  1455. m64_failed:
  1456. if (pdn->m64_single_mode) {
  1457. for (i = 0; i < num_vfs; i++) {
  1458. if (pdn->pe_num_map[i] == IODA_INVALID_PE)
  1459. continue;
  1460. pe = &phb->ioda.pe_array[pdn->pe_num_map[i]];
  1461. pnv_ioda_free_pe(pe);
  1462. }
  1463. } else
  1464. bitmap_clear(phb->ioda.pe_alloc, *pdn->pe_num_map, num_vfs);
  1465. /* Releasing pe_num_map */
  1466. kfree(pdn->pe_num_map);
  1467. return ret;
  1468. }
  1469. int pnv_pcibios_sriov_disable(struct pci_dev *pdev)
  1470. {
  1471. pnv_pci_sriov_disable(pdev);
  1472. /* Release PCI data */
  1473. remove_dev_pci_data(pdev);
  1474. return 0;
  1475. }
  1476. int pnv_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
  1477. {
  1478. /* Allocate PCI data */
  1479. add_dev_pci_data(pdev);
  1480. return pnv_pci_sriov_enable(pdev, num_vfs);
  1481. }
  1482. #endif /* CONFIG_PCI_IOV */
  1483. static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
  1484. {
  1485. struct pci_dn *pdn = pci_get_pdn(pdev);
  1486. struct pnv_ioda_pe *pe;
  1487. /*
  1488. * The function can be called while the PE#
  1489. * hasn't been assigned. Do nothing for the
  1490. * case.
  1491. */
  1492. if (!pdn || pdn->pe_number == IODA_INVALID_PE)
  1493. return;
  1494. pe = &phb->ioda.pe_array[pdn->pe_number];
  1495. WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
  1496. set_dma_offset(&pdev->dev, pe->tce_bypass_base);
  1497. set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]);
  1498. /*
  1499. * Note: iommu_add_device() will fail here as
  1500. * for physical PE: the device is already added by now;
  1501. * for virtual PE: sysfs entries are not ready yet and
  1502. * tce_iommu_bus_notifier will add the device to a group later.
  1503. */
  1504. }
  1505. static bool pnv_pci_ioda_pe_single_vendor(struct pnv_ioda_pe *pe)
  1506. {
  1507. unsigned short vendor = 0;
  1508. struct pci_dev *pdev;
  1509. if (pe->device_count == 1)
  1510. return true;
  1511. /* pe->pdev should be set if it's a single device, pe->pbus if not */
  1512. if (!pe->pbus)
  1513. return true;
  1514. list_for_each_entry(pdev, &pe->pbus->devices, bus_list) {
  1515. if (!vendor) {
  1516. vendor = pdev->vendor;
  1517. continue;
  1518. }
  1519. if (pdev->vendor != vendor)
  1520. return false;
  1521. }
  1522. return true;
  1523. }
  1524. /*
  1525. * Reconfigure TVE#0 to be usable as 64-bit DMA space.
  1526. *
  1527. * The first 4GB of virtual memory for a PE is reserved for 32-bit accesses.
  1528. * Devices can only access more than that if bit 59 of the PCI address is set
  1529. * by hardware, which indicates TVE#1 should be used instead of TVE#0.
  1530. * Many PCI devices are not capable of addressing that many bits, and as a
  1531. * result are limited to the 4GB of virtual memory made available to 32-bit
  1532. * devices in TVE#0.
  1533. *
  1534. * In order to work around this, reconfigure TVE#0 to be suitable for 64-bit
  1535. * devices by configuring the virtual memory past the first 4GB inaccessible
  1536. * by 64-bit DMAs. This should only be used by devices that want more than
  1537. * 4GB, and only on PEs that have no 32-bit devices.
  1538. *
  1539. * Currently this will only work on PHB3 (POWER8).
  1540. */
  1541. static int pnv_pci_ioda_dma_64bit_bypass(struct pnv_ioda_pe *pe)
  1542. {
  1543. u64 window_size, table_size, tce_count, addr;
  1544. struct page *table_pages;
  1545. u64 tce_order = 28; /* 256MB TCEs */
  1546. __be64 *tces;
  1547. s64 rc;
  1548. /*
  1549. * Window size needs to be a power of two, but needs to account for
  1550. * shifting memory by the 4GB offset required to skip 32bit space.
  1551. */
  1552. window_size = roundup_pow_of_two(memory_hotplug_max() + (1ULL << 32));
  1553. tce_count = window_size >> tce_order;
  1554. table_size = tce_count << 3;
  1555. if (table_size < PAGE_SIZE)
  1556. table_size = PAGE_SIZE;
  1557. table_pages = alloc_pages_node(pe->phb->hose->node, GFP_KERNEL,
  1558. get_order(table_size));
  1559. if (!table_pages)
  1560. goto err;
  1561. tces = page_address(table_pages);
  1562. if (!tces)
  1563. goto err;
  1564. memset(tces, 0, table_size);
  1565. for (addr = 0; addr < memory_hotplug_max(); addr += (1 << tce_order)) {
  1566. tces[(addr + (1ULL << 32)) >> tce_order] =
  1567. cpu_to_be64(addr | TCE_PCI_READ | TCE_PCI_WRITE);
  1568. }
  1569. rc = opal_pci_map_pe_dma_window(pe->phb->opal_id,
  1570. pe->pe_number,
  1571. /* reconfigure window 0 */
  1572. (pe->pe_number << 1) + 0,
  1573. 1,
  1574. __pa(tces),
  1575. table_size,
  1576. 1 << tce_order);
  1577. if (rc == OPAL_SUCCESS) {
  1578. pe_info(pe, "Using 64-bit DMA iommu bypass (through TVE#0)\n");
  1579. return 0;
  1580. }
  1581. err:
  1582. pe_err(pe, "Error configuring 64-bit DMA bypass\n");
  1583. return -EIO;
  1584. }
  1585. static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
  1586. {
  1587. struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  1588. struct pnv_phb *phb = hose->private_data;
  1589. struct pci_dn *pdn = pci_get_pdn(pdev);
  1590. struct pnv_ioda_pe *pe;
  1591. uint64_t top;
  1592. bool bypass = false;
  1593. s64 rc;
  1594. if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
  1595. return -ENODEV;
  1596. pe = &phb->ioda.pe_array[pdn->pe_number];
  1597. if (pe->tce_bypass_enabled) {
  1598. top = pe->tce_bypass_base + memblock_end_of_DRAM() - 1;
  1599. bypass = (dma_mask >= top);
  1600. }
  1601. if (bypass) {
  1602. dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
  1603. set_dma_ops(&pdev->dev, &dma_nommu_ops);
  1604. } else {
  1605. /*
  1606. * If the device can't set the TCE bypass bit but still wants
  1607. * to access 4GB or more, on PHB3 we can reconfigure TVE#0 to
  1608. * bypass the 32-bit region and be usable for 64-bit DMAs.
  1609. * The device needs to be able to address all of this space.
  1610. */
  1611. if (dma_mask >> 32 &&
  1612. dma_mask > (memory_hotplug_max() + (1ULL << 32)) &&
  1613. pnv_pci_ioda_pe_single_vendor(pe) &&
  1614. phb->model == PNV_PHB_MODEL_PHB3) {
  1615. /* Configure the bypass mode */
  1616. rc = pnv_pci_ioda_dma_64bit_bypass(pe);
  1617. if (rc)
  1618. return rc;
  1619. /* 4GB offset bypasses 32-bit space */
  1620. set_dma_offset(&pdev->dev, (1ULL << 32));
  1621. set_dma_ops(&pdev->dev, &dma_nommu_ops);
  1622. } else if (dma_mask >> 32 && dma_mask != DMA_BIT_MASK(64)) {
  1623. /*
  1624. * Fail the request if a DMA mask between 32 and 64 bits
  1625. * was requested but couldn't be fulfilled. Ideally we
  1626. * would do this for 64-bits but historically we have
  1627. * always fallen back to 32-bits.
  1628. */
  1629. return -ENOMEM;
  1630. } else {
  1631. dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
  1632. set_dma_ops(&pdev->dev, &dma_iommu_ops);
  1633. }
  1634. }
  1635. *pdev->dev.dma_mask = dma_mask;
  1636. /* Update peer npu devices */
  1637. pnv_npu_try_dma_set_bypass(pdev, bypass);
  1638. return 0;
  1639. }
  1640. static u64 pnv_pci_ioda_dma_get_required_mask(struct pci_dev *pdev)
  1641. {
  1642. struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  1643. struct pnv_phb *phb = hose->private_data;
  1644. struct pci_dn *pdn = pci_get_pdn(pdev);
  1645. struct pnv_ioda_pe *pe;
  1646. u64 end, mask;
  1647. if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
  1648. return 0;
  1649. pe = &phb->ioda.pe_array[pdn->pe_number];
  1650. if (!pe->tce_bypass_enabled)
  1651. return __dma_get_required_mask(&pdev->dev);
  1652. end = pe->tce_bypass_base + memblock_end_of_DRAM();
  1653. mask = 1ULL << (fls64(end) - 1);
  1654. mask += mask - 1;
  1655. return mask;
  1656. }
  1657. static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe,
  1658. struct pci_bus *bus,
  1659. bool add_to_group)
  1660. {
  1661. struct pci_dev *dev;
  1662. list_for_each_entry(dev, &bus->devices, bus_list) {
  1663. set_iommu_table_base(&dev->dev, pe->table_group.tables[0]);
  1664. set_dma_offset(&dev->dev, pe->tce_bypass_base);
  1665. if (add_to_group)
  1666. iommu_add_device(&dev->dev);
  1667. if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
  1668. pnv_ioda_setup_bus_dma(pe, dev->subordinate,
  1669. add_to_group);
  1670. }
  1671. }
  1672. static inline __be64 __iomem *pnv_ioda_get_inval_reg(struct pnv_phb *phb,
  1673. bool real_mode)
  1674. {
  1675. return real_mode ? (__be64 __iomem *)(phb->regs_phys + 0x210) :
  1676. (phb->regs + 0x210);
  1677. }
  1678. static void pnv_pci_p7ioc_tce_invalidate(struct iommu_table *tbl,
  1679. unsigned long index, unsigned long npages, bool rm)
  1680. {
  1681. struct iommu_table_group_link *tgl = list_first_entry_or_null(
  1682. &tbl->it_group_list, struct iommu_table_group_link,
  1683. next);
  1684. struct pnv_ioda_pe *pe = container_of(tgl->table_group,
  1685. struct pnv_ioda_pe, table_group);
  1686. __be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb, rm);
  1687. unsigned long start, end, inc;
  1688. start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);
  1689. end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +
  1690. npages - 1);
  1691. /* p7ioc-style invalidation, 2 TCEs per write */
  1692. start |= (1ull << 63);
  1693. end |= (1ull << 63);
  1694. inc = 16;
  1695. end |= inc - 1; /* round up end to be different than start */
  1696. mb(); /* Ensure above stores are visible */
  1697. while (start <= end) {
  1698. if (rm)
  1699. __raw_rm_writeq_be(start, invalidate);
  1700. else
  1701. __raw_writeq_be(start, invalidate);
  1702. start += inc;
  1703. }
  1704. /*
  1705. * The iommu layer will do another mb() for us on build()
  1706. * and we don't care on free()
  1707. */
  1708. }
  1709. static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,
  1710. long npages, unsigned long uaddr,
  1711. enum dma_data_direction direction,
  1712. unsigned long attrs)
  1713. {
  1714. int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
  1715. attrs);
  1716. if (!ret)
  1717. pnv_pci_p7ioc_tce_invalidate(tbl, index, npages, false);
  1718. return ret;
  1719. }
  1720. #ifdef CONFIG_IOMMU_API
  1721. static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,
  1722. unsigned long *hpa, enum dma_data_direction *direction)
  1723. {
  1724. long ret = pnv_tce_xchg(tbl, index, hpa, direction, true);
  1725. if (!ret)
  1726. pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, false);
  1727. return ret;
  1728. }
  1729. static int pnv_ioda1_tce_xchg_rm(struct iommu_table *tbl, long index,
  1730. unsigned long *hpa, enum dma_data_direction *direction)
  1731. {
  1732. long ret = pnv_tce_xchg(tbl, index, hpa, direction, false);
  1733. if (!ret)
  1734. pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, true);
  1735. return ret;
  1736. }
  1737. #endif
  1738. static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,
  1739. long npages)
  1740. {
  1741. pnv_tce_free(tbl, index, npages);
  1742. pnv_pci_p7ioc_tce_invalidate(tbl, index, npages, false);
  1743. }
  1744. static struct iommu_table_ops pnv_ioda1_iommu_ops = {
  1745. .set = pnv_ioda1_tce_build,
  1746. #ifdef CONFIG_IOMMU_API
  1747. .exchange = pnv_ioda1_tce_xchg,
  1748. .exchange_rm = pnv_ioda1_tce_xchg_rm,
  1749. .useraddrptr = pnv_tce_useraddrptr,
  1750. #endif
  1751. .clear = pnv_ioda1_tce_free,
  1752. .get = pnv_tce_get,
  1753. };
  1754. #define PHB3_TCE_KILL_INVAL_ALL PPC_BIT(0)
  1755. #define PHB3_TCE_KILL_INVAL_PE PPC_BIT(1)
  1756. #define PHB3_TCE_KILL_INVAL_ONE PPC_BIT(2)
  1757. static void pnv_pci_phb3_tce_invalidate_entire(struct pnv_phb *phb, bool rm)
  1758. {
  1759. __be64 __iomem *invalidate = pnv_ioda_get_inval_reg(phb, rm);
  1760. const unsigned long val = PHB3_TCE_KILL_INVAL_ALL;
  1761. mb(); /* Ensure previous TCE table stores are visible */
  1762. if (rm)
  1763. __raw_rm_writeq_be(val, invalidate);
  1764. else
  1765. __raw_writeq_be(val, invalidate);
  1766. }
  1767. static inline void pnv_pci_phb3_tce_invalidate_pe(struct pnv_ioda_pe *pe)
  1768. {
  1769. /* 01xb - invalidate TCEs that match the specified PE# */
  1770. __be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb, false);
  1771. unsigned long val = PHB3_TCE_KILL_INVAL_PE | (pe->pe_number & 0xFF);
  1772. mb(); /* Ensure above stores are visible */
  1773. __raw_writeq_be(val, invalidate);
  1774. }
  1775. static void pnv_pci_phb3_tce_invalidate(struct pnv_ioda_pe *pe, bool rm,
  1776. unsigned shift, unsigned long index,
  1777. unsigned long npages)
  1778. {
  1779. __be64 __iomem *invalidate = pnv_ioda_get_inval_reg(pe->phb, rm);
  1780. unsigned long start, end, inc;
  1781. /* We'll invalidate DMA address in PE scope */
  1782. start = PHB3_TCE_KILL_INVAL_ONE;
  1783. start |= (pe->pe_number & 0xFF);
  1784. end = start;
  1785. /* Figure out the start, end and step */
  1786. start |= (index << shift);
  1787. end |= ((index + npages - 1) << shift);
  1788. inc = (0x1ull << shift);
  1789. mb();
  1790. while (start <= end) {
  1791. if (rm)
  1792. __raw_rm_writeq_be(start, invalidate);
  1793. else
  1794. __raw_writeq_be(start, invalidate);
  1795. start += inc;
  1796. }
  1797. }
  1798. static inline void pnv_pci_ioda2_tce_invalidate_pe(struct pnv_ioda_pe *pe)
  1799. {
  1800. struct pnv_phb *phb = pe->phb;
  1801. if (phb->model == PNV_PHB_MODEL_PHB3 && phb->regs)
  1802. pnv_pci_phb3_tce_invalidate_pe(pe);
  1803. else
  1804. opal_pci_tce_kill(phb->opal_id, OPAL_PCI_TCE_KILL_PE,
  1805. pe->pe_number, 0, 0, 0);
  1806. }
  1807. static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,
  1808. unsigned long index, unsigned long npages, bool rm)
  1809. {
  1810. struct iommu_table_group_link *tgl;
  1811. list_for_each_entry_lockless(tgl, &tbl->it_group_list, next) {
  1812. struct pnv_ioda_pe *pe = container_of(tgl->table_group,
  1813. struct pnv_ioda_pe, table_group);
  1814. struct pnv_phb *phb = pe->phb;
  1815. unsigned int shift = tbl->it_page_shift;
  1816. /*
  1817. * NVLink1 can use the TCE kill register directly as
  1818. * it's the same as PHB3. NVLink2 is different and
  1819. * should go via the OPAL call.
  1820. */
  1821. if (phb->model == PNV_PHB_MODEL_NPU) {
  1822. /*
  1823. * The NVLink hardware does not support TCE kill
  1824. * per TCE entry so we have to invalidate
  1825. * the entire cache for it.
  1826. */
  1827. pnv_pci_phb3_tce_invalidate_entire(phb, rm);
  1828. continue;
  1829. }
  1830. if (phb->model == PNV_PHB_MODEL_PHB3 && phb->regs)
  1831. pnv_pci_phb3_tce_invalidate(pe, rm, shift,
  1832. index, npages);
  1833. else
  1834. opal_pci_tce_kill(phb->opal_id,
  1835. OPAL_PCI_TCE_KILL_PAGES,
  1836. pe->pe_number, 1u << shift,
  1837. index << shift, npages);
  1838. }
  1839. }
  1840. void pnv_pci_ioda2_tce_invalidate_entire(struct pnv_phb *phb, bool rm)
  1841. {
  1842. if (phb->model == PNV_PHB_MODEL_NPU || phb->model == PNV_PHB_MODEL_PHB3)
  1843. pnv_pci_phb3_tce_invalidate_entire(phb, rm);
  1844. else
  1845. opal_pci_tce_kill(phb->opal_id, OPAL_PCI_TCE_KILL, 0, 0, 0, 0);
  1846. }
  1847. static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,
  1848. long npages, unsigned long uaddr,
  1849. enum dma_data_direction direction,
  1850. unsigned long attrs)
  1851. {
  1852. int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,
  1853. attrs);
  1854. if (!ret)
  1855. pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
  1856. return ret;
  1857. }
  1858. #ifdef CONFIG_IOMMU_API
  1859. static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,
  1860. unsigned long *hpa, enum dma_data_direction *direction)
  1861. {
  1862. long ret = pnv_tce_xchg(tbl, index, hpa, direction, true);
  1863. if (!ret)
  1864. pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);
  1865. return ret;
  1866. }
  1867. static int pnv_ioda2_tce_xchg_rm(struct iommu_table *tbl, long index,
  1868. unsigned long *hpa, enum dma_data_direction *direction)
  1869. {
  1870. long ret = pnv_tce_xchg(tbl, index, hpa, direction, false);
  1871. if (!ret)
  1872. pnv_pci_ioda2_tce_invalidate(tbl, index, 1, true);
  1873. return ret;
  1874. }
  1875. #endif
  1876. static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,
  1877. long npages)
  1878. {
  1879. pnv_tce_free(tbl, index, npages);
  1880. pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);
  1881. }
  1882. static struct iommu_table_ops pnv_ioda2_iommu_ops = {
  1883. .set = pnv_ioda2_tce_build,
  1884. #ifdef CONFIG_IOMMU_API
  1885. .exchange = pnv_ioda2_tce_xchg,
  1886. .exchange_rm = pnv_ioda2_tce_xchg_rm,
  1887. .useraddrptr = pnv_tce_useraddrptr,
  1888. #endif
  1889. .clear = pnv_ioda2_tce_free,
  1890. .get = pnv_tce_get,
  1891. .free = pnv_pci_ioda2_table_free_pages,
  1892. };
  1893. static int pnv_pci_ioda_dev_dma_weight(struct pci_dev *dev, void *data)
  1894. {
  1895. unsigned int *weight = (unsigned int *)data;
  1896. /* This is quite simplistic. The "base" weight of a device
  1897. * is 10. 0 means no DMA is to be accounted for it.
  1898. */
  1899. if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)
  1900. return 0;
  1901. if (dev->class == PCI_CLASS_SERIAL_USB_UHCI ||
  1902. dev->class == PCI_CLASS_SERIAL_USB_OHCI ||
  1903. dev->class == PCI_CLASS_SERIAL_USB_EHCI)
  1904. *weight += 3;
  1905. else if ((dev->class >> 8) == PCI_CLASS_STORAGE_RAID)
  1906. *weight += 15;
  1907. else
  1908. *weight += 10;
  1909. return 0;
  1910. }
  1911. static unsigned int pnv_pci_ioda_pe_dma_weight(struct pnv_ioda_pe *pe)
  1912. {
  1913. unsigned int weight = 0;
  1914. /* SRIOV VF has same DMA32 weight as its PF */
  1915. #ifdef CONFIG_PCI_IOV
  1916. if ((pe->flags & PNV_IODA_PE_VF) && pe->parent_dev) {
  1917. pnv_pci_ioda_dev_dma_weight(pe->parent_dev, &weight);
  1918. return weight;
  1919. }
  1920. #endif
  1921. if ((pe->flags & PNV_IODA_PE_DEV) && pe->pdev) {
  1922. pnv_pci_ioda_dev_dma_weight(pe->pdev, &weight);
  1923. } else if ((pe->flags & PNV_IODA_PE_BUS) && pe->pbus) {
  1924. struct pci_dev *pdev;
  1925. list_for_each_entry(pdev, &pe->pbus->devices, bus_list)
  1926. pnv_pci_ioda_dev_dma_weight(pdev, &weight);
  1927. } else if ((pe->flags & PNV_IODA_PE_BUS_ALL) && pe->pbus) {
  1928. pci_walk_bus(pe->pbus, pnv_pci_ioda_dev_dma_weight, &weight);
  1929. }
  1930. return weight;
  1931. }
  1932. static void pnv_pci_ioda1_setup_dma_pe(struct pnv_phb *phb,
  1933. struct pnv_ioda_pe *pe)
  1934. {
  1935. struct page *tce_mem = NULL;
  1936. struct iommu_table *tbl;
  1937. unsigned int weight, total_weight = 0;
  1938. unsigned int tce32_segsz, base, segs, avail, i;
  1939. int64_t rc;
  1940. void *addr;
  1941. /* XXX FIXME: Handle 64-bit only DMA devices */
  1942. /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
  1943. /* XXX FIXME: Allocate multi-level tables on PHB3 */
  1944. weight = pnv_pci_ioda_pe_dma_weight(pe);
  1945. if (!weight)
  1946. return;
  1947. pci_walk_bus(phb->hose->bus, pnv_pci_ioda_dev_dma_weight,
  1948. &total_weight);
  1949. segs = (weight * phb->ioda.dma32_count) / total_weight;
  1950. if (!segs)
  1951. segs = 1;
  1952. /*
  1953. * Allocate contiguous DMA32 segments. We begin with the expected
  1954. * number of segments. With one more attempt, the number of DMA32
  1955. * segments to be allocated is decreased by one until one segment
  1956. * is allocated successfully.
  1957. */
  1958. do {
  1959. for (base = 0; base <= phb->ioda.dma32_count - segs; base++) {
  1960. for (avail = 0, i = base; i < base + segs; i++) {
  1961. if (phb->ioda.dma32_segmap[i] ==
  1962. IODA_INVALID_PE)
  1963. avail++;
  1964. }
  1965. if (avail == segs)
  1966. goto found;
  1967. }
  1968. } while (--segs);
  1969. if (!segs) {
  1970. pe_warn(pe, "No available DMA32 segments\n");
  1971. return;
  1972. }
  1973. found:
  1974. tbl = pnv_pci_table_alloc(phb->hose->node);
  1975. if (WARN_ON(!tbl))
  1976. return;
  1977. iommu_register_group(&pe->table_group, phb->hose->global_number,
  1978. pe->pe_number);
  1979. pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);
  1980. /* Grab a 32-bit TCE table */
  1981. pe_info(pe, "DMA weight %d (%d), assigned (%d) %d DMA32 segments\n",
  1982. weight, total_weight, base, segs);
  1983. pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
  1984. base * PNV_IODA1_DMA32_SEGSIZE,
  1985. (base + segs) * PNV_IODA1_DMA32_SEGSIZE - 1);
  1986. /* XXX Currently, we allocate one big contiguous table for the
  1987. * TCEs. We only really need one chunk per 256M of TCE space
  1988. * (ie per segment) but that's an optimization for later, it
  1989. * requires some added smarts with our get/put_tce implementation
  1990. *
  1991. * Each TCE page is 4KB in size and each TCE entry occupies 8
  1992. * bytes
  1993. */
  1994. tce32_segsz = PNV_IODA1_DMA32_SEGSIZE >> (IOMMU_PAGE_SHIFT_4K - 3);
  1995. tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
  1996. get_order(tce32_segsz * segs));
  1997. if (!tce_mem) {
  1998. pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
  1999. goto fail;
  2000. }
  2001. addr = page_address(tce_mem);
  2002. memset(addr, 0, tce32_segsz * segs);
  2003. /* Configure HW */
  2004. for (i = 0; i < segs; i++) {
  2005. rc = opal_pci_map_pe_dma_window(phb->opal_id,
  2006. pe->pe_number,
  2007. base + i, 1,
  2008. __pa(addr) + tce32_segsz * i,
  2009. tce32_segsz, IOMMU_PAGE_SIZE_4K);
  2010. if (rc) {
  2011. pe_err(pe, " Failed to configure 32-bit TCE table,"
  2012. " err %ld\n", rc);
  2013. goto fail;
  2014. }
  2015. }
  2016. /* Setup DMA32 segment mapping */
  2017. for (i = base; i < base + segs; i++)
  2018. phb->ioda.dma32_segmap[i] = pe->pe_number;
  2019. /* Setup linux iommu table */
  2020. pnv_pci_setup_iommu_table(tbl, addr, tce32_segsz * segs,
  2021. base * PNV_IODA1_DMA32_SEGSIZE,
  2022. IOMMU_PAGE_SHIFT_4K);
  2023. tbl->it_ops = &pnv_ioda1_iommu_ops;
  2024. pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;
  2025. pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;
  2026. iommu_init_table(tbl, phb->hose->node);
  2027. if (pe->flags & PNV_IODA_PE_DEV) {
  2028. /*
  2029. * Setting table base here only for carrying iommu_group
  2030. * further down to let iommu_add_device() do the job.
  2031. * pnv_pci_ioda_dma_dev_setup will override it later anyway.
  2032. */
  2033. set_iommu_table_base(&pe->pdev->dev, tbl);
  2034. iommu_add_device(&pe->pdev->dev);
  2035. } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  2036. pnv_ioda_setup_bus_dma(pe, pe->pbus, true);
  2037. return;
  2038. fail:
  2039. /* XXX Failure: Try to fallback to 64-bit only ? */
  2040. if (tce_mem)
  2041. __free_pages(tce_mem, get_order(tce32_segsz * segs));
  2042. if (tbl) {
  2043. pnv_pci_unlink_table_and_group(tbl, &pe->table_group);
  2044. iommu_tce_table_put(tbl);
  2045. }
  2046. }
  2047. static long pnv_pci_ioda2_set_window(struct iommu_table_group *table_group,
  2048. int num, struct iommu_table *tbl)
  2049. {
  2050. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  2051. table_group);
  2052. struct pnv_phb *phb = pe->phb;
  2053. int64_t rc;
  2054. const unsigned long size = tbl->it_indirect_levels ?
  2055. tbl->it_level_size : tbl->it_size;
  2056. const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
  2057. const __u64 win_size = tbl->it_size << tbl->it_page_shift;
  2058. pe_info(pe, "Setting up window#%d %llx..%llx pg=%x\n", num,
  2059. start_addr, start_addr + win_size - 1,
  2060. IOMMU_PAGE_SIZE(tbl));
  2061. /*
  2062. * Map TCE table through TVT. The TVE index is the PE number
  2063. * shifted by 1 bit for 32-bits DMA space.
  2064. */
  2065. rc = opal_pci_map_pe_dma_window(phb->opal_id,
  2066. pe->pe_number,
  2067. (pe->pe_number << 1) + num,
  2068. tbl->it_indirect_levels + 1,
  2069. __pa(tbl->it_base),
  2070. size << 3,
  2071. IOMMU_PAGE_SIZE(tbl));
  2072. if (rc) {
  2073. pe_err(pe, "Failed to configure TCE table, err %ld\n", rc);
  2074. return rc;
  2075. }
  2076. pnv_pci_link_table_and_group(phb->hose->node, num,
  2077. tbl, &pe->table_group);
  2078. pnv_pci_ioda2_tce_invalidate_pe(pe);
  2079. return 0;
  2080. }
  2081. void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable)
  2082. {
  2083. uint16_t window_id = (pe->pe_number << 1 ) + 1;
  2084. int64_t rc;
  2085. pe_info(pe, "%sabling 64-bit DMA bypass\n", enable ? "En" : "Dis");
  2086. if (enable) {
  2087. phys_addr_t top = memblock_end_of_DRAM();
  2088. top = roundup_pow_of_two(top);
  2089. rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
  2090. pe->pe_number,
  2091. window_id,
  2092. pe->tce_bypass_base,
  2093. top);
  2094. } else {
  2095. rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id,
  2096. pe->pe_number,
  2097. window_id,
  2098. pe->tce_bypass_base,
  2099. 0);
  2100. }
  2101. if (rc)
  2102. pe_err(pe, "OPAL error %lld configuring bypass window\n", rc);
  2103. else
  2104. pe->tce_bypass_enabled = enable;
  2105. }
  2106. static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group,
  2107. int num, __u32 page_shift, __u64 window_size, __u32 levels,
  2108. bool alloc_userspace_copy, struct iommu_table **ptbl)
  2109. {
  2110. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  2111. table_group);
  2112. int nid = pe->phb->hose->node;
  2113. __u64 bus_offset = num ? pe->tce_bypass_base : table_group->tce32_start;
  2114. long ret;
  2115. struct iommu_table *tbl;
  2116. tbl = pnv_pci_table_alloc(nid);
  2117. if (!tbl)
  2118. return -ENOMEM;
  2119. tbl->it_ops = &pnv_ioda2_iommu_ops;
  2120. ret = pnv_pci_ioda2_table_alloc_pages(nid,
  2121. bus_offset, page_shift, window_size,
  2122. levels, alloc_userspace_copy, tbl);
  2123. if (ret) {
  2124. iommu_tce_table_put(tbl);
  2125. return ret;
  2126. }
  2127. *ptbl = tbl;
  2128. return 0;
  2129. }
  2130. static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe)
  2131. {
  2132. struct iommu_table *tbl = NULL;
  2133. long rc;
  2134. /*
  2135. * crashkernel= specifies the kdump kernel's maximum memory at
  2136. * some offset and there is no guaranteed the result is a power
  2137. * of 2, which will cause errors later.
  2138. */
  2139. const u64 max_memory = __rounddown_pow_of_two(memory_hotplug_max());
  2140. /*
  2141. * In memory constrained environments, e.g. kdump kernel, the
  2142. * DMA window can be larger than available memory, which will
  2143. * cause errors later.
  2144. */
  2145. const u64 window_size = min((u64)pe->table_group.tce32_size, max_memory);
  2146. rc = pnv_pci_ioda2_create_table(&pe->table_group, 0,
  2147. IOMMU_PAGE_SHIFT_4K,
  2148. window_size,
  2149. POWERNV_IOMMU_DEFAULT_LEVELS, false, &tbl);
  2150. if (rc) {
  2151. pe_err(pe, "Failed to create 32-bit TCE table, err %ld",
  2152. rc);
  2153. return rc;
  2154. }
  2155. iommu_init_table(tbl, pe->phb->hose->node);
  2156. rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);
  2157. if (rc) {
  2158. pe_err(pe, "Failed to configure 32-bit TCE table, err %ld\n",
  2159. rc);
  2160. iommu_tce_table_put(tbl);
  2161. return rc;
  2162. }
  2163. if (!pnv_iommu_bypass_disabled)
  2164. pnv_pci_ioda2_set_bypass(pe, true);
  2165. /*
  2166. * Setting table base here only for carrying iommu_group
  2167. * further down to let iommu_add_device() do the job.
  2168. * pnv_pci_ioda_dma_dev_setup will override it later anyway.
  2169. */
  2170. if (pe->flags & PNV_IODA_PE_DEV)
  2171. set_iommu_table_base(&pe->pdev->dev, tbl);
  2172. return 0;
  2173. }
  2174. #if defined(CONFIG_IOMMU_API) || defined(CONFIG_PCI_IOV)
  2175. static long pnv_pci_ioda2_unset_window(struct iommu_table_group *table_group,
  2176. int num)
  2177. {
  2178. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  2179. table_group);
  2180. struct pnv_phb *phb = pe->phb;
  2181. long ret;
  2182. pe_info(pe, "Removing DMA window #%d\n", num);
  2183. ret = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
  2184. (pe->pe_number << 1) + num,
  2185. 0/* levels */, 0/* table address */,
  2186. 0/* table size */, 0/* page size */);
  2187. if (ret)
  2188. pe_warn(pe, "Unmapping failed, ret = %ld\n", ret);
  2189. else
  2190. pnv_pci_ioda2_tce_invalidate_pe(pe);
  2191. pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
  2192. return ret;
  2193. }
  2194. #endif
  2195. #ifdef CONFIG_IOMMU_API
  2196. static unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
  2197. __u64 window_size, __u32 levels)
  2198. {
  2199. unsigned long bytes = 0;
  2200. const unsigned window_shift = ilog2(window_size);
  2201. unsigned entries_shift = window_shift - page_shift;
  2202. unsigned table_shift = entries_shift + 3;
  2203. unsigned long tce_table_size = max(0x1000UL, 1UL << table_shift);
  2204. unsigned long direct_table_size;
  2205. if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
  2206. !is_power_of_2(window_size))
  2207. return 0;
  2208. /* Calculate a direct table size from window_size and levels */
  2209. entries_shift = (entries_shift + levels - 1) / levels;
  2210. table_shift = entries_shift + 3;
  2211. table_shift = max_t(unsigned, table_shift, PAGE_SHIFT);
  2212. direct_table_size = 1UL << table_shift;
  2213. for ( ; levels; --levels) {
  2214. bytes += _ALIGN_UP(tce_table_size, direct_table_size);
  2215. tce_table_size /= direct_table_size;
  2216. tce_table_size <<= 3;
  2217. tce_table_size = max_t(unsigned long,
  2218. tce_table_size, direct_table_size);
  2219. }
  2220. return bytes + bytes; /* one for HW table, one for userspace copy */
  2221. }
  2222. static long pnv_pci_ioda2_create_table_userspace(
  2223. struct iommu_table_group *table_group,
  2224. int num, __u32 page_shift, __u64 window_size, __u32 levels,
  2225. struct iommu_table **ptbl)
  2226. {
  2227. long ret = pnv_pci_ioda2_create_table(table_group,
  2228. num, page_shift, window_size, levels, true, ptbl);
  2229. if (!ret)
  2230. (*ptbl)->it_allocated_size = pnv_pci_ioda2_get_table_size(
  2231. page_shift, window_size, levels);
  2232. return ret;
  2233. }
  2234. static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group)
  2235. {
  2236. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  2237. table_group);
  2238. /* Store @tbl as pnv_pci_ioda2_unset_window() resets it */
  2239. struct iommu_table *tbl = pe->table_group.tables[0];
  2240. pnv_pci_ioda2_set_bypass(pe, false);
  2241. pnv_pci_ioda2_unset_window(&pe->table_group, 0);
  2242. if (pe->pbus)
  2243. pnv_ioda_setup_bus_dma(pe, pe->pbus, false);
  2244. iommu_tce_table_put(tbl);
  2245. }
  2246. static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group)
  2247. {
  2248. struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe,
  2249. table_group);
  2250. pnv_pci_ioda2_setup_default_config(pe);
  2251. if (pe->pbus)
  2252. pnv_ioda_setup_bus_dma(pe, pe->pbus, false);
  2253. }
  2254. static struct iommu_table_group_ops pnv_pci_ioda2_ops = {
  2255. .get_table_size = pnv_pci_ioda2_get_table_size,
  2256. .create_table = pnv_pci_ioda2_create_table_userspace,
  2257. .set_window = pnv_pci_ioda2_set_window,
  2258. .unset_window = pnv_pci_ioda2_unset_window,
  2259. .take_ownership = pnv_ioda2_take_ownership,
  2260. .release_ownership = pnv_ioda2_release_ownership,
  2261. };
  2262. static int gpe_table_group_to_npe_cb(struct device *dev, void *opaque)
  2263. {
  2264. struct pci_controller *hose;
  2265. struct pnv_phb *phb;
  2266. struct pnv_ioda_pe **ptmppe = opaque;
  2267. struct pci_dev *pdev = container_of(dev, struct pci_dev, dev);
  2268. struct pci_dn *pdn = pci_get_pdn(pdev);
  2269. if (!pdn || pdn->pe_number == IODA_INVALID_PE)
  2270. return 0;
  2271. hose = pci_bus_to_host(pdev->bus);
  2272. phb = hose->private_data;
  2273. if (phb->type != PNV_PHB_NPU_NVLINK)
  2274. return 0;
  2275. *ptmppe = &phb->ioda.pe_array[pdn->pe_number];
  2276. return 1;
  2277. }
  2278. /*
  2279. * This returns PE of associated NPU.
  2280. * This assumes that NPU is in the same IOMMU group with GPU and there is
  2281. * no other PEs.
  2282. */
  2283. static struct pnv_ioda_pe *gpe_table_group_to_npe(
  2284. struct iommu_table_group *table_group)
  2285. {
  2286. struct pnv_ioda_pe *npe = NULL;
  2287. int ret = iommu_group_for_each_dev(table_group->group, &npe,
  2288. gpe_table_group_to_npe_cb);
  2289. BUG_ON(!ret || !npe);
  2290. return npe;
  2291. }
  2292. static long pnv_pci_ioda2_npu_set_window(struct iommu_table_group *table_group,
  2293. int num, struct iommu_table *tbl)
  2294. {
  2295. struct pnv_ioda_pe *npe = gpe_table_group_to_npe(table_group);
  2296. int num2 = (num == 0) ? 1 : 0;
  2297. long ret = pnv_pci_ioda2_set_window(table_group, num, tbl);
  2298. if (ret)
  2299. return ret;
  2300. if (table_group->tables[num2])
  2301. pnv_npu_unset_window(npe, num2);
  2302. ret = pnv_npu_set_window(npe, num, tbl);
  2303. if (ret) {
  2304. pnv_pci_ioda2_unset_window(table_group, num);
  2305. if (table_group->tables[num2])
  2306. pnv_npu_set_window(npe, num2,
  2307. table_group->tables[num2]);
  2308. }
  2309. return ret;
  2310. }
  2311. static long pnv_pci_ioda2_npu_unset_window(
  2312. struct iommu_table_group *table_group,
  2313. int num)
  2314. {
  2315. struct pnv_ioda_pe *npe = gpe_table_group_to_npe(table_group);
  2316. int num2 = (num == 0) ? 1 : 0;
  2317. long ret = pnv_pci_ioda2_unset_window(table_group, num);
  2318. if (ret)
  2319. return ret;
  2320. if (!npe->table_group.tables[num])
  2321. return 0;
  2322. ret = pnv_npu_unset_window(npe, num);
  2323. if (ret)
  2324. return ret;
  2325. if (table_group->tables[num2])
  2326. ret = pnv_npu_set_window(npe, num2, table_group->tables[num2]);
  2327. return ret;
  2328. }
  2329. static void pnv_ioda2_npu_take_ownership(struct iommu_table_group *table_group)
  2330. {
  2331. /*
  2332. * Detach NPU first as pnv_ioda2_take_ownership() will destroy
  2333. * the iommu_table if 32bit DMA is enabled.
  2334. */
  2335. pnv_npu_take_ownership(gpe_table_group_to_npe(table_group));
  2336. pnv_ioda2_take_ownership(table_group);
  2337. }
  2338. static struct iommu_table_group_ops pnv_pci_ioda2_npu_ops = {
  2339. .get_table_size = pnv_pci_ioda2_get_table_size,
  2340. .create_table = pnv_pci_ioda2_create_table_userspace,
  2341. .set_window = pnv_pci_ioda2_npu_set_window,
  2342. .unset_window = pnv_pci_ioda2_npu_unset_window,
  2343. .take_ownership = pnv_ioda2_npu_take_ownership,
  2344. .release_ownership = pnv_ioda2_release_ownership,
  2345. };
  2346. static void pnv_pci_ioda_setup_iommu_api(void)
  2347. {
  2348. struct pci_controller *hose, *tmp;
  2349. struct pnv_phb *phb;
  2350. struct pnv_ioda_pe *pe, *gpe;
  2351. /*
  2352. * Now we have all PHBs discovered, time to add NPU devices to
  2353. * the corresponding IOMMU groups.
  2354. */
  2355. list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
  2356. phb = hose->private_data;
  2357. if (phb->type != PNV_PHB_NPU_NVLINK)
  2358. continue;
  2359. list_for_each_entry(pe, &phb->ioda.pe_list, list) {
  2360. gpe = pnv_pci_npu_setup_iommu(pe);
  2361. if (gpe)
  2362. gpe->table_group.ops = &pnv_pci_ioda2_npu_ops;
  2363. }
  2364. }
  2365. }
  2366. #else /* !CONFIG_IOMMU_API */
  2367. static void pnv_pci_ioda_setup_iommu_api(void) { };
  2368. #endif
  2369. static unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb)
  2370. {
  2371. struct pci_controller *hose = phb->hose;
  2372. struct device_node *dn = hose->dn;
  2373. unsigned long mask = 0;
  2374. int i, rc, count;
  2375. u32 val;
  2376. count = of_property_count_u32_elems(dn, "ibm,supported-tce-sizes");
  2377. if (count <= 0) {
  2378. mask = SZ_4K | SZ_64K;
  2379. /* Add 16M for POWER8 by default */
  2380. if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
  2381. !cpu_has_feature(CPU_FTR_ARCH_300))
  2382. mask |= SZ_16M | SZ_256M;
  2383. return mask;
  2384. }
  2385. for (i = 0; i < count; i++) {
  2386. rc = of_property_read_u32_index(dn, "ibm,supported-tce-sizes",
  2387. i, &val);
  2388. if (rc == 0)
  2389. mask |= 1ULL << val;
  2390. }
  2391. return mask;
  2392. }
  2393. static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
  2394. struct pnv_ioda_pe *pe)
  2395. {
  2396. int64_t rc;
  2397. if (!pnv_pci_ioda_pe_dma_weight(pe))
  2398. return;
  2399. /* TVE #1 is selected by PCI address bit 59 */
  2400. pe->tce_bypass_base = 1ull << 59;
  2401. iommu_register_group(&pe->table_group, phb->hose->global_number,
  2402. pe->pe_number);
  2403. /* The PE will reserve all possible 32-bits space */
  2404. pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
  2405. phb->ioda.m32_pci_base);
  2406. /* Setup linux iommu table */
  2407. pe->table_group.tce32_start = 0;
  2408. pe->table_group.tce32_size = phb->ioda.m32_pci_base;
  2409. pe->table_group.max_dynamic_windows_supported =
  2410. IOMMU_TABLE_GROUP_MAX_TABLES;
  2411. pe->table_group.max_levels = POWERNV_IOMMU_MAX_LEVELS;
  2412. pe->table_group.pgsizes = pnv_ioda_parse_tce_sizes(phb);
  2413. #ifdef CONFIG_IOMMU_API
  2414. pe->table_group.ops = &pnv_pci_ioda2_ops;
  2415. #endif
  2416. rc = pnv_pci_ioda2_setup_default_config(pe);
  2417. if (rc)
  2418. return;
  2419. if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL))
  2420. pnv_ioda_setup_bus_dma(pe, pe->pbus, true);
  2421. }
  2422. #ifdef CONFIG_PCI_MSI
  2423. int64_t pnv_opal_pci_msi_eoi(struct irq_chip *chip, unsigned int hw_irq)
  2424. {
  2425. struct pnv_phb *phb = container_of(chip, struct pnv_phb,
  2426. ioda.irq_chip);
  2427. return opal_pci_msi_eoi(phb->opal_id, hw_irq);
  2428. }
  2429. static void pnv_ioda2_msi_eoi(struct irq_data *d)
  2430. {
  2431. int64_t rc;
  2432. unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d);
  2433. struct irq_chip *chip = irq_data_get_irq_chip(d);
  2434. rc = pnv_opal_pci_msi_eoi(chip, hw_irq);
  2435. WARN_ON_ONCE(rc);
  2436. icp_native_eoi(d);
  2437. }
  2438. void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq)
  2439. {
  2440. struct irq_data *idata;
  2441. struct irq_chip *ichip;
  2442. /* The MSI EOI OPAL call is only needed on PHB3 */
  2443. if (phb->model != PNV_PHB_MODEL_PHB3)
  2444. return;
  2445. if (!phb->ioda.irq_chip_init) {
  2446. /*
  2447. * First time we setup an MSI IRQ, we need to setup the
  2448. * corresponding IRQ chip to route correctly.
  2449. */
  2450. idata = irq_get_irq_data(virq);
  2451. ichip = irq_data_get_irq_chip(idata);
  2452. phb->ioda.irq_chip_init = 1;
  2453. phb->ioda.irq_chip = *ichip;
  2454. phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi;
  2455. }
  2456. irq_set_chip(virq, &phb->ioda.irq_chip);
  2457. }
  2458. /*
  2459. * Returns true iff chip is something that we could call
  2460. * pnv_opal_pci_msi_eoi for.
  2461. */
  2462. bool is_pnv_opal_msi(struct irq_chip *chip)
  2463. {
  2464. return chip->irq_eoi == pnv_ioda2_msi_eoi;
  2465. }
  2466. EXPORT_SYMBOL_GPL(is_pnv_opal_msi);
  2467. static int pnv_pci_ioda_msi_setup(struct pnv_phb *phb, struct pci_dev *dev,
  2468. unsigned int hwirq, unsigned int virq,
  2469. unsigned int is_64, struct msi_msg *msg)
  2470. {
  2471. struct pnv_ioda_pe *pe = pnv_ioda_get_pe(dev);
  2472. unsigned int xive_num = hwirq - phb->msi_base;
  2473. __be32 data;
  2474. int rc;
  2475. /* No PE assigned ? bail out ... no MSI for you ! */
  2476. if (pe == NULL)
  2477. return -ENXIO;
  2478. /* Check if we have an MVE */
  2479. if (pe->mve_number < 0)
  2480. return -ENXIO;
  2481. /* Force 32-bit MSI on some broken devices */
  2482. if (dev->no_64bit_msi)
  2483. is_64 = 0;
  2484. /* Assign XIVE to PE */
  2485. rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num);
  2486. if (rc) {
  2487. pr_warn("%s: OPAL error %d setting XIVE %d PE\n",
  2488. pci_name(dev), rc, xive_num);
  2489. return -EIO;
  2490. }
  2491. if (is_64) {
  2492. __be64 addr64;
  2493. rc = opal_get_msi_64(phb->opal_id, pe->mve_number, xive_num, 1,
  2494. &addr64, &data);
  2495. if (rc) {
  2496. pr_warn("%s: OPAL error %d getting 64-bit MSI data\n",
  2497. pci_name(dev), rc);
  2498. return -EIO;
  2499. }
  2500. msg->address_hi = be64_to_cpu(addr64) >> 32;
  2501. msg->address_lo = be64_to_cpu(addr64) & 0xfffffffful;
  2502. } else {
  2503. __be32 addr32;
  2504. rc = opal_get_msi_32(phb->opal_id, pe->mve_number, xive_num, 1,
  2505. &addr32, &data);
  2506. if (rc) {
  2507. pr_warn("%s: OPAL error %d getting 32-bit MSI data\n",
  2508. pci_name(dev), rc);
  2509. return -EIO;
  2510. }
  2511. msg->address_hi = 0;
  2512. msg->address_lo = be32_to_cpu(addr32);
  2513. }
  2514. msg->data = be32_to_cpu(data);
  2515. pnv_set_msi_irq_chip(phb, virq);
  2516. pr_devel("%s: %s-bit MSI on hwirq %x (xive #%d),"
  2517. " address=%x_%08x data=%x PE# %x\n",
  2518. pci_name(dev), is_64 ? "64" : "32", hwirq, xive_num,
  2519. msg->address_hi, msg->address_lo, data, pe->pe_number);
  2520. return 0;
  2521. }
  2522. static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
  2523. {
  2524. unsigned int count;
  2525. const __be32 *prop = of_get_property(phb->hose->dn,
  2526. "ibm,opal-msi-ranges", NULL);
  2527. if (!prop) {
  2528. /* BML Fallback */
  2529. prop = of_get_property(phb->hose->dn, "msi-ranges", NULL);
  2530. }
  2531. if (!prop)
  2532. return;
  2533. phb->msi_base = be32_to_cpup(prop);
  2534. count = be32_to_cpup(prop + 1);
  2535. if (msi_bitmap_alloc(&phb->msi_bmp, count, phb->hose->dn)) {
  2536. pr_err("PCI %d: Failed to allocate MSI bitmap !\n",
  2537. phb->hose->global_number);
  2538. return;
  2539. }
  2540. phb->msi_setup = pnv_pci_ioda_msi_setup;
  2541. phb->msi32_support = 1;
  2542. pr_info(" Allocated bitmap for %d MSIs (base IRQ 0x%x)\n",
  2543. count, phb->msi_base);
  2544. }
  2545. #else
  2546. static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
  2547. #endif /* CONFIG_PCI_MSI */
  2548. #ifdef CONFIG_PCI_IOV
  2549. static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
  2550. {
  2551. struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  2552. struct pnv_phb *phb = hose->private_data;
  2553. const resource_size_t gate = phb->ioda.m64_segsize >> 2;
  2554. struct resource *res;
  2555. int i;
  2556. resource_size_t size, total_vf_bar_sz;
  2557. struct pci_dn *pdn;
  2558. int mul, total_vfs;
  2559. pdn = pci_get_pdn(pdev);
  2560. pdn->vfs_expanded = 0;
  2561. pdn->m64_single_mode = false;
  2562. total_vfs = pci_sriov_get_totalvfs(pdev);
  2563. mul = phb->ioda.total_pe_num;
  2564. total_vf_bar_sz = 0;
  2565. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  2566. res = &pdev->resource[i + PCI_IOV_RESOURCES];
  2567. if (!res->flags || res->parent)
  2568. continue;
  2569. if (!pnv_pci_is_m64_flags(res->flags)) {
  2570. dev_warn(&pdev->dev, "Don't support SR-IOV with"
  2571. " non M64 VF BAR%d: %pR. \n",
  2572. i, res);
  2573. goto truncate_iov;
  2574. }
  2575. total_vf_bar_sz += pci_iov_resource_size(pdev,
  2576. i + PCI_IOV_RESOURCES);
  2577. /*
  2578. * If bigger than quarter of M64 segment size, just round up
  2579. * power of two.
  2580. *
  2581. * Generally, one M64 BAR maps one IOV BAR. To avoid conflict
  2582. * with other devices, IOV BAR size is expanded to be
  2583. * (total_pe * VF_BAR_size). When VF_BAR_size is half of M64
  2584. * segment size , the expanded size would equal to half of the
  2585. * whole M64 space size, which will exhaust the M64 Space and
  2586. * limit the system flexibility. This is a design decision to
  2587. * set the boundary to quarter of the M64 segment size.
  2588. */
  2589. if (total_vf_bar_sz > gate) {
  2590. mul = roundup_pow_of_two(total_vfs);
  2591. dev_info(&pdev->dev,
  2592. "VF BAR Total IOV size %llx > %llx, roundup to %d VFs\n",
  2593. total_vf_bar_sz, gate, mul);
  2594. pdn->m64_single_mode = true;
  2595. break;
  2596. }
  2597. }
  2598. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  2599. res = &pdev->resource[i + PCI_IOV_RESOURCES];
  2600. if (!res->flags || res->parent)
  2601. continue;
  2602. size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
  2603. /*
  2604. * On PHB3, the minimum size alignment of M64 BAR in single
  2605. * mode is 32MB.
  2606. */
  2607. if (pdn->m64_single_mode && (size < SZ_32M))
  2608. goto truncate_iov;
  2609. dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res);
  2610. res->end = res->start + size * mul - 1;
  2611. dev_dbg(&pdev->dev, " %pR\n", res);
  2612. dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
  2613. i, res, mul);
  2614. }
  2615. pdn->vfs_expanded = mul;
  2616. return;
  2617. truncate_iov:
  2618. /* To save MMIO space, IOV BAR is truncated. */
  2619. for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
  2620. res = &pdev->resource[i + PCI_IOV_RESOURCES];
  2621. res->flags = 0;
  2622. res->end = res->start - 1;
  2623. }
  2624. }
  2625. static void pnv_pci_ioda_fixup_iov(struct pci_dev *pdev)
  2626. {
  2627. if (WARN_ON(pci_dev_is_added(pdev)))
  2628. return;
  2629. if (pdev->is_virtfn) {
  2630. struct pnv_ioda_pe *pe = pnv_ioda_get_pe(pdev);
  2631. /*
  2632. * VF PEs are single-device PEs so their pdev pointer needs to
  2633. * be set. The pdev doesn't exist when the PE is allocated (in
  2634. * (pcibios_sriov_enable()) so we fix it up here.
  2635. */
  2636. pe->pdev = pdev;
  2637. WARN_ON(!(pe->flags & PNV_IODA_PE_VF));
  2638. } else if (pdev->is_physfn) {
  2639. /*
  2640. * For PFs adjust their allocated IOV resources to match what
  2641. * the PHB can support using it's M64 BAR table.
  2642. */
  2643. pnv_pci_ioda_fixup_iov_resources(pdev);
  2644. }
  2645. }
  2646. #endif /* CONFIG_PCI_IOV */
  2647. static void pnv_ioda_setup_pe_res(struct pnv_ioda_pe *pe,
  2648. struct resource *res)
  2649. {
  2650. struct pnv_phb *phb = pe->phb;
  2651. struct pci_bus_region region;
  2652. int index;
  2653. int64_t rc;
  2654. if (!res || !res->flags || res->start > res->end)
  2655. return;
  2656. if (res->flags & IORESOURCE_IO) {
  2657. region.start = res->start - phb->ioda.io_pci_base;
  2658. region.end = res->end - phb->ioda.io_pci_base;
  2659. index = region.start / phb->ioda.io_segsize;
  2660. while (index < phb->ioda.total_pe_num &&
  2661. region.start <= region.end) {
  2662. phb->ioda.io_segmap[index] = pe->pe_number;
  2663. rc = opal_pci_map_pe_mmio_window(phb->opal_id,
  2664. pe->pe_number, OPAL_IO_WINDOW_TYPE, 0, index);
  2665. if (rc != OPAL_SUCCESS) {
  2666. pr_err("%s: Error %lld mapping IO segment#%d to PE#%x\n",
  2667. __func__, rc, index, pe->pe_number);
  2668. break;
  2669. }
  2670. region.start += phb->ioda.io_segsize;
  2671. index++;
  2672. }
  2673. } else if ((res->flags & IORESOURCE_MEM) &&
  2674. !pnv_pci_is_m64(phb, res)) {
  2675. region.start = res->start -
  2676. phb->hose->mem_offset[0] -
  2677. phb->ioda.m32_pci_base;
  2678. region.end = res->end -
  2679. phb->hose->mem_offset[0] -
  2680. phb->ioda.m32_pci_base;
  2681. index = region.start / phb->ioda.m32_segsize;
  2682. while (index < phb->ioda.total_pe_num &&
  2683. region.start <= region.end) {
  2684. phb->ioda.m32_segmap[index] = pe->pe_number;
  2685. rc = opal_pci_map_pe_mmio_window(phb->opal_id,
  2686. pe->pe_number, OPAL_M32_WINDOW_TYPE, 0, index);
  2687. if (rc != OPAL_SUCCESS) {
  2688. pr_err("%s: Error %lld mapping M32 segment#%d to PE#%x",
  2689. __func__, rc, index, pe->pe_number);
  2690. break;
  2691. }
  2692. region.start += phb->ioda.m32_segsize;
  2693. index++;
  2694. }
  2695. }
  2696. }
  2697. /*
  2698. * This function is supposed to be called on basis of PE from top
  2699. * to bottom style. So the the I/O or MMIO segment assigned to
  2700. * parent PE could be overridden by its child PEs if necessary.
  2701. */
  2702. static void pnv_ioda_setup_pe_seg(struct pnv_ioda_pe *pe)
  2703. {
  2704. struct pci_dev *pdev;
  2705. int i;
  2706. /*
  2707. * NOTE: We only care PCI bus based PE for now. For PCI
  2708. * device based PE, for example SRIOV sensitive VF should
  2709. * be figured out later.
  2710. */
  2711. BUG_ON(!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)));
  2712. list_for_each_entry(pdev, &pe->pbus->devices, bus_list) {
  2713. for (i = 0; i <= PCI_ROM_RESOURCE; i++)
  2714. pnv_ioda_setup_pe_res(pe, &pdev->resource[i]);
  2715. /*
  2716. * If the PE contains all subordinate PCI buses, the
  2717. * windows of the child bridges should be mapped to
  2718. * the PE as well.
  2719. */
  2720. if (!(pe->flags & PNV_IODA_PE_BUS_ALL) || !pci_is_bridge(pdev))
  2721. continue;
  2722. for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
  2723. pnv_ioda_setup_pe_res(pe,
  2724. &pdev->resource[PCI_BRIDGE_RESOURCES + i]);
  2725. }
  2726. }
  2727. #ifdef CONFIG_DEBUG_FS
  2728. static int pnv_pci_diag_data_set(void *data, u64 val)
  2729. {
  2730. struct pci_controller *hose;
  2731. struct pnv_phb *phb;
  2732. s64 ret;
  2733. if (val != 1ULL)
  2734. return -EINVAL;
  2735. hose = (struct pci_controller *)data;
  2736. if (!hose || !hose->private_data)
  2737. return -ENODEV;
  2738. phb = hose->private_data;
  2739. /* Retrieve the diag data from firmware */
  2740. ret = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag_data,
  2741. phb->diag_data_size);
  2742. if (ret != OPAL_SUCCESS)
  2743. return -EIO;
  2744. /* Print the diag data to the kernel log */
  2745. pnv_pci_dump_phb_diag_data(phb->hose, phb->diag_data);
  2746. return 0;
  2747. }
  2748. DEFINE_SIMPLE_ATTRIBUTE(pnv_pci_diag_data_fops, NULL,
  2749. pnv_pci_diag_data_set, "%llu\n");
  2750. #endif /* CONFIG_DEBUG_FS */
  2751. static void pnv_pci_ioda_create_dbgfs(void)
  2752. {
  2753. #ifdef CONFIG_DEBUG_FS
  2754. struct pci_controller *hose, *tmp;
  2755. struct pnv_phb *phb;
  2756. char name[16];
  2757. list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
  2758. phb = hose->private_data;
  2759. /* Notify initialization of PHB done */
  2760. phb->initialized = 1;
  2761. sprintf(name, "PCI%04x", hose->global_number);
  2762. phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
  2763. if (!phb->dbgfs) {
  2764. pr_warn("%s: Error on creating debugfs on PHB#%x\n",
  2765. __func__, hose->global_number);
  2766. continue;
  2767. }
  2768. debugfs_create_file("dump_diag_regs", 0200, phb->dbgfs, hose,
  2769. &pnv_pci_diag_data_fops);
  2770. }
  2771. #endif /* CONFIG_DEBUG_FS */
  2772. }
  2773. static void pnv_pci_enable_bridge(struct pci_bus *bus)
  2774. {
  2775. struct pci_dev *dev = bus->self;
  2776. struct pci_bus *child;
  2777. /* Empty bus ? bail */
  2778. if (list_empty(&bus->devices))
  2779. return;
  2780. /*
  2781. * If there's a bridge associated with that bus enable it. This works
  2782. * around races in the generic code if the enabling is done during
  2783. * parallel probing. This can be removed once those races have been
  2784. * fixed.
  2785. */
  2786. if (dev) {
  2787. int rc = pci_enable_device(dev);
  2788. if (rc)
  2789. pci_err(dev, "Error enabling bridge (%d)\n", rc);
  2790. pci_set_master(dev);
  2791. }
  2792. /* Perform the same to child busses */
  2793. list_for_each_entry(child, &bus->children, node)
  2794. pnv_pci_enable_bridge(child);
  2795. }
  2796. static void pnv_pci_enable_bridges(void)
  2797. {
  2798. struct pci_controller *hose;
  2799. list_for_each_entry(hose, &hose_list, list_node)
  2800. pnv_pci_enable_bridge(hose->bus);
  2801. }
  2802. static void pnv_pci_ioda_fixup(void)
  2803. {
  2804. pnv_pci_ioda_setup_PEs();
  2805. pnv_pci_ioda_setup_iommu_api();
  2806. pnv_pci_ioda_create_dbgfs();
  2807. pnv_pci_enable_bridges();
  2808. #ifdef CONFIG_EEH
  2809. pnv_eeh_post_init();
  2810. #endif
  2811. }
  2812. /*
  2813. * Returns the alignment for I/O or memory windows for P2P
  2814. * bridges. That actually depends on how PEs are segmented.
  2815. * For now, we return I/O or M32 segment size for PE sensitive
  2816. * P2P bridges. Otherwise, the default values (4KiB for I/O,
  2817. * 1MiB for memory) will be returned.
  2818. *
  2819. * The current PCI bus might be put into one PE, which was
  2820. * create against the parent PCI bridge. For that case, we
  2821. * needn't enlarge the alignment so that we can save some
  2822. * resources.
  2823. */
  2824. static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
  2825. unsigned long type)
  2826. {
  2827. struct pci_dev *bridge;
  2828. struct pci_controller *hose = pci_bus_to_host(bus);
  2829. struct pnv_phb *phb = hose->private_data;
  2830. int num_pci_bridges = 0;
  2831. bridge = bus->self;
  2832. while (bridge) {
  2833. if (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE) {
  2834. num_pci_bridges++;
  2835. if (num_pci_bridges >= 2)
  2836. return 1;
  2837. }
  2838. bridge = bridge->bus->self;
  2839. }
  2840. /*
  2841. * We fall back to M32 if M64 isn't supported. We enforce the M64
  2842. * alignment for any 64-bit resource, PCIe doesn't care and
  2843. * bridges only do 64-bit prefetchable anyway.
  2844. */
  2845. if (phb->ioda.m64_segsize && pnv_pci_is_m64_flags(type))
  2846. return phb->ioda.m64_segsize;
  2847. if (type & IORESOURCE_MEM)
  2848. return phb->ioda.m32_segsize;
  2849. return phb->ioda.io_segsize;
  2850. }
  2851. /*
  2852. * We are updating root port or the upstream port of the
  2853. * bridge behind the root port with PHB's windows in order
  2854. * to accommodate the changes on required resources during
  2855. * PCI (slot) hotplug, which is connected to either root
  2856. * port or the downstream ports of PCIe switch behind the
  2857. * root port.
  2858. */
  2859. static void pnv_pci_fixup_bridge_resources(struct pci_bus *bus,
  2860. unsigned long type)
  2861. {
  2862. struct pci_controller *hose = pci_bus_to_host(bus);
  2863. struct pnv_phb *phb = hose->private_data;
  2864. struct pci_dev *bridge = bus->self;
  2865. struct resource *r, *w;
  2866. bool msi_region = false;
  2867. int i;
  2868. /* Check if we need apply fixup to the bridge's windows */
  2869. if (!pci_is_root_bus(bridge->bus) &&
  2870. !pci_is_root_bus(bridge->bus->self->bus))
  2871. return;
  2872. /* Fixup the resources */
  2873. for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
  2874. r = &bridge->resource[PCI_BRIDGE_RESOURCES + i];
  2875. if (!r->flags || !r->parent)
  2876. continue;
  2877. w = NULL;
  2878. if (r->flags & type & IORESOURCE_IO)
  2879. w = &hose->io_resource;
  2880. else if (pnv_pci_is_m64(phb, r) &&
  2881. (type & IORESOURCE_PREFETCH) &&
  2882. phb->ioda.m64_segsize)
  2883. w = &hose->mem_resources[1];
  2884. else if (r->flags & type & IORESOURCE_MEM) {
  2885. w = &hose->mem_resources[0];
  2886. msi_region = true;
  2887. }
  2888. r->start = w->start;
  2889. r->end = w->end;
  2890. /* The 64KB 32-bits MSI region shouldn't be included in
  2891. * the 32-bits bridge window. Otherwise, we can see strange
  2892. * issues. One of them is EEH error observed on Garrison.
  2893. *
  2894. * Exclude top 1MB region which is the minimal alignment of
  2895. * 32-bits bridge window.
  2896. */
  2897. if (msi_region) {
  2898. r->end += 0x10000;
  2899. r->end -= 0x100000;
  2900. }
  2901. }
  2902. }
  2903. static void pnv_pci_setup_bridge(struct pci_bus *bus, unsigned long type)
  2904. {
  2905. struct pci_controller *hose = pci_bus_to_host(bus);
  2906. struct pnv_phb *phb = hose->private_data;
  2907. struct pci_dev *bridge = bus->self;
  2908. struct pnv_ioda_pe *pe;
  2909. bool all = (pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE);
  2910. /* Extend bridge's windows if necessary */
  2911. pnv_pci_fixup_bridge_resources(bus, type);
  2912. /* The PE for root bus should be realized before any one else */
  2913. if (!phb->ioda.root_pe_populated) {
  2914. pe = pnv_ioda_setup_bus_PE(phb->hose->bus, false);
  2915. if (pe) {
  2916. phb->ioda.root_pe_idx = pe->pe_number;
  2917. phb->ioda.root_pe_populated = true;
  2918. }
  2919. }
  2920. /* Don't assign PE to PCI bus, which doesn't have subordinate devices */
  2921. if (list_empty(&bus->devices))
  2922. return;
  2923. /* Reserve PEs according to used M64 resources */
  2924. if (phb->reserve_m64_pe)
  2925. phb->reserve_m64_pe(bus, NULL, all);
  2926. /*
  2927. * Assign PE. We might run here because of partial hotplug.
  2928. * For the case, we just pick up the existing PE and should
  2929. * not allocate resources again.
  2930. */
  2931. pe = pnv_ioda_setup_bus_PE(bus, all);
  2932. if (!pe)
  2933. return;
  2934. pnv_ioda_setup_pe_seg(pe);
  2935. switch (phb->type) {
  2936. case PNV_PHB_IODA1:
  2937. pnv_pci_ioda1_setup_dma_pe(phb, pe);
  2938. break;
  2939. case PNV_PHB_IODA2:
  2940. pnv_pci_ioda2_setup_dma_pe(phb, pe);
  2941. break;
  2942. default:
  2943. pr_warn("%s: No DMA for PHB#%x (type %d)\n",
  2944. __func__, phb->hose->global_number, phb->type);
  2945. }
  2946. }
  2947. static resource_size_t pnv_pci_default_alignment(void)
  2948. {
  2949. return PAGE_SIZE;
  2950. }
  2951. #ifdef CONFIG_PCI_IOV
  2952. static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
  2953. int resno)
  2954. {
  2955. struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  2956. struct pnv_phb *phb = hose->private_data;
  2957. struct pci_dn *pdn = pci_get_pdn(pdev);
  2958. resource_size_t align;
  2959. /*
  2960. * On PowerNV platform, IOV BAR is mapped by M64 BAR to enable the
  2961. * SR-IOV. While from hardware perspective, the range mapped by M64
  2962. * BAR should be size aligned.
  2963. *
  2964. * When IOV BAR is mapped with M64 BAR in Single PE mode, the extra
  2965. * powernv-specific hardware restriction is gone. But if just use the
  2966. * VF BAR size as the alignment, PF BAR / VF BAR may be allocated with
  2967. * in one segment of M64 #15, which introduces the PE conflict between
  2968. * PF and VF. Based on this, the minimum alignment of an IOV BAR is
  2969. * m64_segsize.
  2970. *
  2971. * This function returns the total IOV BAR size if M64 BAR is in
  2972. * Shared PE mode or just VF BAR size if not.
  2973. * If the M64 BAR is in Single PE mode, return the VF BAR size or
  2974. * M64 segment size if IOV BAR size is less.
  2975. */
  2976. align = pci_iov_resource_size(pdev, resno);
  2977. if (!pdn->vfs_expanded)
  2978. return align;
  2979. if (pdn->m64_single_mode)
  2980. return max(align, (resource_size_t)phb->ioda.m64_segsize);
  2981. return pdn->vfs_expanded * align;
  2982. }
  2983. #endif /* CONFIG_PCI_IOV */
  2984. /* Prevent enabling devices for which we couldn't properly
  2985. * assign a PE
  2986. */
  2987. static bool pnv_pci_enable_device_hook(struct pci_dev *dev)
  2988. {
  2989. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  2990. struct pnv_phb *phb = hose->private_data;
  2991. struct pci_dn *pdn;
  2992. /* The function is probably called while the PEs have
  2993. * not be created yet. For example, resource reassignment
  2994. * during PCI probe period. We just skip the check if
  2995. * PEs isn't ready.
  2996. */
  2997. if (!phb->initialized)
  2998. return true;
  2999. pdn = pci_get_pdn(dev);
  3000. if (!pdn || pdn->pe_number == IODA_INVALID_PE)
  3001. return false;
  3002. return true;
  3003. }
  3004. static long pnv_pci_ioda1_unset_window(struct iommu_table_group *table_group,
  3005. int num)
  3006. {
  3007. struct pnv_ioda_pe *pe = container_of(table_group,
  3008. struct pnv_ioda_pe, table_group);
  3009. struct pnv_phb *phb = pe->phb;
  3010. unsigned int idx;
  3011. long rc;
  3012. pe_info(pe, "Removing DMA window #%d\n", num);
  3013. for (idx = 0; idx < phb->ioda.dma32_count; idx++) {
  3014. if (phb->ioda.dma32_segmap[idx] != pe->pe_number)
  3015. continue;
  3016. rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
  3017. idx, 0, 0ul, 0ul, 0ul);
  3018. if (rc != OPAL_SUCCESS) {
  3019. pe_warn(pe, "Failure %ld unmapping DMA32 segment#%d\n",
  3020. rc, idx);
  3021. return rc;
  3022. }
  3023. phb->ioda.dma32_segmap[idx] = IODA_INVALID_PE;
  3024. }
  3025. pnv_pci_unlink_table_and_group(table_group->tables[num], table_group);
  3026. return OPAL_SUCCESS;
  3027. }
  3028. static void pnv_pci_ioda1_release_pe_dma(struct pnv_ioda_pe *pe)
  3029. {
  3030. unsigned int weight = pnv_pci_ioda_pe_dma_weight(pe);
  3031. struct iommu_table *tbl = pe->table_group.tables[0];
  3032. int64_t rc;
  3033. if (!weight)
  3034. return;
  3035. rc = pnv_pci_ioda1_unset_window(&pe->table_group, 0);
  3036. if (rc != OPAL_SUCCESS)
  3037. return;
  3038. pnv_pci_p7ioc_tce_invalidate(tbl, tbl->it_offset, tbl->it_size, false);
  3039. if (pe->table_group.group) {
  3040. iommu_group_put(pe->table_group.group);
  3041. WARN_ON(pe->table_group.group);
  3042. }
  3043. free_pages(tbl->it_base, get_order(tbl->it_size << 3));
  3044. iommu_tce_table_put(tbl);
  3045. }
  3046. static void pnv_pci_ioda2_release_pe_dma(struct pnv_ioda_pe *pe)
  3047. {
  3048. struct iommu_table *tbl = pe->table_group.tables[0];
  3049. unsigned int weight = pnv_pci_ioda_pe_dma_weight(pe);
  3050. #ifdef CONFIG_IOMMU_API
  3051. int64_t rc;
  3052. #endif
  3053. if (!weight)
  3054. return;
  3055. #ifdef CONFIG_IOMMU_API
  3056. rc = pnv_pci_ioda2_unset_window(&pe->table_group, 0);
  3057. if (rc)
  3058. pe_warn(pe, "OPAL error %ld release DMA window\n", rc);
  3059. #endif
  3060. pnv_pci_ioda2_set_bypass(pe, false);
  3061. if (pe->table_group.group) {
  3062. iommu_group_put(pe->table_group.group);
  3063. WARN_ON(pe->table_group.group);
  3064. }
  3065. iommu_tce_table_put(tbl);
  3066. }
  3067. static void pnv_ioda_free_pe_seg(struct pnv_ioda_pe *pe,
  3068. unsigned short win,
  3069. unsigned int *map)
  3070. {
  3071. struct pnv_phb *phb = pe->phb;
  3072. int idx;
  3073. int64_t rc;
  3074. for (idx = 0; idx < phb->ioda.total_pe_num; idx++) {
  3075. if (map[idx] != pe->pe_number)
  3076. continue;
  3077. if (win == OPAL_M64_WINDOW_TYPE)
  3078. rc = opal_pci_map_pe_mmio_window(phb->opal_id,
  3079. phb->ioda.reserved_pe_idx, win,
  3080. idx / PNV_IODA1_M64_SEGS,
  3081. idx % PNV_IODA1_M64_SEGS);
  3082. else
  3083. rc = opal_pci_map_pe_mmio_window(phb->opal_id,
  3084. phb->ioda.reserved_pe_idx, win, 0, idx);
  3085. if (rc != OPAL_SUCCESS)
  3086. pe_warn(pe, "Error %ld unmapping (%d) segment#%d\n",
  3087. rc, win, idx);
  3088. map[idx] = IODA_INVALID_PE;
  3089. }
  3090. }
  3091. static void pnv_ioda_release_pe_seg(struct pnv_ioda_pe *pe)
  3092. {
  3093. struct pnv_phb *phb = pe->phb;
  3094. if (phb->type == PNV_PHB_IODA1) {
  3095. pnv_ioda_free_pe_seg(pe, OPAL_IO_WINDOW_TYPE,
  3096. phb->ioda.io_segmap);
  3097. pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE,
  3098. phb->ioda.m32_segmap);
  3099. pnv_ioda_free_pe_seg(pe, OPAL_M64_WINDOW_TYPE,
  3100. phb->ioda.m64_segmap);
  3101. } else if (phb->type == PNV_PHB_IODA2) {
  3102. pnv_ioda_free_pe_seg(pe, OPAL_M32_WINDOW_TYPE,
  3103. phb->ioda.m32_segmap);
  3104. }
  3105. }
  3106. static void pnv_ioda_release_pe(struct pnv_ioda_pe *pe)
  3107. {
  3108. struct pnv_phb *phb = pe->phb;
  3109. struct pnv_ioda_pe *slave, *tmp;
  3110. list_del(&pe->list);
  3111. switch (phb->type) {
  3112. case PNV_PHB_IODA1:
  3113. pnv_pci_ioda1_release_pe_dma(pe);
  3114. break;
  3115. case PNV_PHB_IODA2:
  3116. pnv_pci_ioda2_release_pe_dma(pe);
  3117. break;
  3118. default:
  3119. WARN_ON(1);
  3120. }
  3121. pnv_ioda_release_pe_seg(pe);
  3122. pnv_ioda_deconfigure_pe(pe->phb, pe);
  3123. /* Release slave PEs in the compound PE */
  3124. if (pe->flags & PNV_IODA_PE_MASTER) {
  3125. list_for_each_entry_safe(slave, tmp, &pe->slaves, list) {
  3126. list_del(&slave->list);
  3127. pnv_ioda_free_pe(slave);
  3128. }
  3129. }
  3130. /*
  3131. * The PE for root bus can be removed because of hotplug in EEH
  3132. * recovery for fenced PHB error. We need to mark the PE dead so
  3133. * that it can be populated again in PCI hot add path. The PE
  3134. * shouldn't be destroyed as it's the global reserved resource.
  3135. */
  3136. if (phb->ioda.root_pe_populated &&
  3137. phb->ioda.root_pe_idx == pe->pe_number)
  3138. phb->ioda.root_pe_populated = false;
  3139. else
  3140. pnv_ioda_free_pe(pe);
  3141. }
  3142. static void pnv_pci_release_device(struct pci_dev *pdev)
  3143. {
  3144. struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  3145. struct pnv_phb *phb = hose->private_data;
  3146. struct pci_dn *pdn = pci_get_pdn(pdev);
  3147. struct pnv_ioda_pe *pe;
  3148. if (pdev->is_virtfn)
  3149. return;
  3150. if (!pdn || pdn->pe_number == IODA_INVALID_PE)
  3151. return;
  3152. /*
  3153. * PCI hotplug can happen as part of EEH error recovery. The @pdn
  3154. * isn't removed and added afterwards in this scenario. We should
  3155. * set the PE number in @pdn to an invalid one. Otherwise, the PE's
  3156. * device count is decreased on removing devices while failing to
  3157. * be increased on adding devices. It leads to unbalanced PE's device
  3158. * count and eventually make normal PCI hotplug path broken.
  3159. */
  3160. pe = &phb->ioda.pe_array[pdn->pe_number];
  3161. pdn->pe_number = IODA_INVALID_PE;
  3162. WARN_ON(--pe->device_count < 0);
  3163. if (pe->device_count == 0)
  3164. pnv_ioda_release_pe(pe);
  3165. }
  3166. static void pnv_pci_ioda_shutdown(struct pci_controller *hose)
  3167. {
  3168. struct pnv_phb *phb = hose->private_data;
  3169. opal_pci_reset(phb->opal_id, OPAL_RESET_PCI_IODA_TABLE,
  3170. OPAL_ASSERT_RESET);
  3171. }
  3172. static const struct pci_controller_ops pnv_pci_ioda_controller_ops = {
  3173. .dma_dev_setup = pnv_pci_dma_dev_setup,
  3174. .dma_bus_setup = pnv_pci_dma_bus_setup,
  3175. #ifdef CONFIG_PCI_MSI
  3176. .setup_msi_irqs = pnv_setup_msi_irqs,
  3177. .teardown_msi_irqs = pnv_teardown_msi_irqs,
  3178. #endif
  3179. .enable_device_hook = pnv_pci_enable_device_hook,
  3180. .release_device = pnv_pci_release_device,
  3181. .window_alignment = pnv_pci_window_alignment,
  3182. .setup_bridge = pnv_pci_setup_bridge,
  3183. .reset_secondary_bus = pnv_pci_reset_secondary_bus,
  3184. .dma_set_mask = pnv_pci_ioda_dma_set_mask,
  3185. .dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask,
  3186. .shutdown = pnv_pci_ioda_shutdown,
  3187. };
  3188. static int pnv_npu_dma_set_mask(struct pci_dev *npdev, u64 dma_mask)
  3189. {
  3190. dev_err_once(&npdev->dev,
  3191. "%s operation unsupported for NVLink devices\n",
  3192. __func__);
  3193. return -EPERM;
  3194. }
  3195. static const struct pci_controller_ops pnv_npu_ioda_controller_ops = {
  3196. .dma_dev_setup = pnv_pci_dma_dev_setup,
  3197. #ifdef CONFIG_PCI_MSI
  3198. .setup_msi_irqs = pnv_setup_msi_irqs,
  3199. .teardown_msi_irqs = pnv_teardown_msi_irqs,
  3200. #endif
  3201. .enable_device_hook = pnv_pci_enable_device_hook,
  3202. .window_alignment = pnv_pci_window_alignment,
  3203. .reset_secondary_bus = pnv_pci_reset_secondary_bus,
  3204. .dma_set_mask = pnv_npu_dma_set_mask,
  3205. .shutdown = pnv_pci_ioda_shutdown,
  3206. };
  3207. static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
  3208. .enable_device_hook = pnv_pci_enable_device_hook,
  3209. .window_alignment = pnv_pci_window_alignment,
  3210. .reset_secondary_bus = pnv_pci_reset_secondary_bus,
  3211. .shutdown = pnv_pci_ioda_shutdown,
  3212. };
  3213. static void __init pnv_pci_init_ioda_phb(struct device_node *np,
  3214. u64 hub_id, int ioda_type)
  3215. {
  3216. struct pci_controller *hose;
  3217. struct pnv_phb *phb;
  3218. unsigned long size, m64map_off, m32map_off, pemap_off;
  3219. unsigned long iomap_off = 0, dma32map_off = 0;
  3220. struct resource r;
  3221. const __be64 *prop64;
  3222. const __be32 *prop32;
  3223. int len;
  3224. unsigned int segno;
  3225. u64 phb_id;
  3226. void *aux;
  3227. long rc;
  3228. if (!of_device_is_available(np))
  3229. return;
  3230. pr_info("Initializing %s PHB (%pOF)\n", pnv_phb_names[ioda_type], np);
  3231. prop64 = of_get_property(np, "ibm,opal-phbid", NULL);
  3232. if (!prop64) {
  3233. pr_err(" Missing \"ibm,opal-phbid\" property !\n");
  3234. return;
  3235. }
  3236. phb_id = be64_to_cpup(prop64);
  3237. pr_debug(" PHB-ID : 0x%016llx\n", phb_id);
  3238. phb = memblock_virt_alloc(sizeof(*phb), 0);
  3239. /* Allocate PCI controller */
  3240. phb->hose = hose = pcibios_alloc_controller(np);
  3241. if (!phb->hose) {
  3242. pr_err(" Can't allocate PCI controller for %pOF\n",
  3243. np);
  3244. memblock_free(__pa(phb), sizeof(struct pnv_phb));
  3245. return;
  3246. }
  3247. spin_lock_init(&phb->lock);
  3248. prop32 = of_get_property(np, "bus-range", &len);
  3249. if (prop32 && len == 8) {
  3250. hose->first_busno = be32_to_cpu(prop32[0]);
  3251. hose->last_busno = be32_to_cpu(prop32[1]);
  3252. } else {
  3253. pr_warn(" Broken <bus-range> on %pOF\n", np);
  3254. hose->first_busno = 0;
  3255. hose->last_busno = 0xff;
  3256. }
  3257. hose->private_data = phb;
  3258. phb->hub_id = hub_id;
  3259. phb->opal_id = phb_id;
  3260. phb->type = ioda_type;
  3261. mutex_init(&phb->ioda.pe_alloc_mutex);
  3262. /* Detect specific models for error handling */
  3263. if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
  3264. phb->model = PNV_PHB_MODEL_P7IOC;
  3265. else if (of_device_is_compatible(np, "ibm,power8-pciex"))
  3266. phb->model = PNV_PHB_MODEL_PHB3;
  3267. else if (of_device_is_compatible(np, "ibm,power8-npu-pciex"))
  3268. phb->model = PNV_PHB_MODEL_NPU;
  3269. else if (of_device_is_compatible(np, "ibm,power9-npu-pciex"))
  3270. phb->model = PNV_PHB_MODEL_NPU2;
  3271. else
  3272. phb->model = PNV_PHB_MODEL_UNKNOWN;
  3273. /* Initialize diagnostic data buffer */
  3274. prop32 = of_get_property(np, "ibm,phb-diag-data-size", NULL);
  3275. if (prop32)
  3276. phb->diag_data_size = be32_to_cpup(prop32);
  3277. else
  3278. phb->diag_data_size = PNV_PCI_DIAG_BUF_SIZE;
  3279. phb->diag_data = memblock_virt_alloc(phb->diag_data_size, 0);
  3280. /* Parse 32-bit and IO ranges (if any) */
  3281. pci_process_bridge_OF_ranges(hose, np, !hose->global_number);
  3282. /* Get registers */
  3283. if (!of_address_to_resource(np, 0, &r)) {
  3284. phb->regs_phys = r.start;
  3285. phb->regs = ioremap(r.start, resource_size(&r));
  3286. if (phb->regs == NULL)
  3287. pr_err(" Failed to map registers !\n");
  3288. }
  3289. /* Initialize more IODA stuff */
  3290. phb->ioda.total_pe_num = 1;
  3291. prop32 = of_get_property(np, "ibm,opal-num-pes", NULL);
  3292. if (prop32)
  3293. phb->ioda.total_pe_num = be32_to_cpup(prop32);
  3294. prop32 = of_get_property(np, "ibm,opal-reserved-pe", NULL);
  3295. if (prop32)
  3296. phb->ioda.reserved_pe_idx = be32_to_cpup(prop32);
  3297. /* Invalidate RID to PE# mapping */
  3298. for (segno = 0; segno < ARRAY_SIZE(phb->ioda.pe_rmap); segno++)
  3299. phb->ioda.pe_rmap[segno] = IODA_INVALID_PE;
  3300. /* Parse 64-bit MMIO range */
  3301. pnv_ioda_parse_m64_window(phb);
  3302. phb->ioda.m32_size = resource_size(&hose->mem_resources[0]);
  3303. /* FW Has already off top 64k of M32 space (MSI space) */
  3304. phb->ioda.m32_size += 0x10000;
  3305. phb->ioda.m32_segsize = phb->ioda.m32_size / phb->ioda.total_pe_num;
  3306. phb->ioda.m32_pci_base = hose->mem_resources[0].start - hose->mem_offset[0];
  3307. phb->ioda.io_size = hose->pci_io_size;
  3308. phb->ioda.io_segsize = phb->ioda.io_size / phb->ioda.total_pe_num;
  3309. phb->ioda.io_pci_base = 0; /* XXX calculate this ? */
  3310. /* Calculate how many 32-bit TCE segments we have */
  3311. phb->ioda.dma32_count = phb->ioda.m32_pci_base /
  3312. PNV_IODA1_DMA32_SEGSIZE;
  3313. /* Allocate aux data & arrays. We don't have IO ports on PHB3 */
  3314. size = _ALIGN_UP(max_t(unsigned, phb->ioda.total_pe_num, 8) / 8,
  3315. sizeof(unsigned long));
  3316. m64map_off = size;
  3317. size += phb->ioda.total_pe_num * sizeof(phb->ioda.m64_segmap[0]);
  3318. m32map_off = size;
  3319. size += phb->ioda.total_pe_num * sizeof(phb->ioda.m32_segmap[0]);
  3320. if (phb->type == PNV_PHB_IODA1) {
  3321. iomap_off = size;
  3322. size += phb->ioda.total_pe_num * sizeof(phb->ioda.io_segmap[0]);
  3323. dma32map_off = size;
  3324. size += phb->ioda.dma32_count *
  3325. sizeof(phb->ioda.dma32_segmap[0]);
  3326. }
  3327. pemap_off = size;
  3328. size += phb->ioda.total_pe_num * sizeof(struct pnv_ioda_pe);
  3329. aux = memblock_virt_alloc(size, 0);
  3330. phb->ioda.pe_alloc = aux;
  3331. phb->ioda.m64_segmap = aux + m64map_off;
  3332. phb->ioda.m32_segmap = aux + m32map_off;
  3333. for (segno = 0; segno < phb->ioda.total_pe_num; segno++) {
  3334. phb->ioda.m64_segmap[segno] = IODA_INVALID_PE;
  3335. phb->ioda.m32_segmap[segno] = IODA_INVALID_PE;
  3336. }
  3337. if (phb->type == PNV_PHB_IODA1) {
  3338. phb->ioda.io_segmap = aux + iomap_off;
  3339. for (segno = 0; segno < phb->ioda.total_pe_num; segno++)
  3340. phb->ioda.io_segmap[segno] = IODA_INVALID_PE;
  3341. phb->ioda.dma32_segmap = aux + dma32map_off;
  3342. for (segno = 0; segno < phb->ioda.dma32_count; segno++)
  3343. phb->ioda.dma32_segmap[segno] = IODA_INVALID_PE;
  3344. }
  3345. phb->ioda.pe_array = aux + pemap_off;
  3346. /*
  3347. * Choose PE number for root bus, which shouldn't have
  3348. * M64 resources consumed by its child devices. To pick
  3349. * the PE number adjacent to the reserved one if possible.
  3350. */
  3351. pnv_ioda_reserve_pe(phb, phb->ioda.reserved_pe_idx);
  3352. if (phb->ioda.reserved_pe_idx == 0) {
  3353. phb->ioda.root_pe_idx = 1;
  3354. pnv_ioda_reserve_pe(phb, phb->ioda.root_pe_idx);
  3355. } else if (phb->ioda.reserved_pe_idx == (phb->ioda.total_pe_num - 1)) {
  3356. phb->ioda.root_pe_idx = phb->ioda.reserved_pe_idx - 1;
  3357. pnv_ioda_reserve_pe(phb, phb->ioda.root_pe_idx);
  3358. } else {
  3359. phb->ioda.root_pe_idx = IODA_INVALID_PE;
  3360. }
  3361. INIT_LIST_HEAD(&phb->ioda.pe_list);
  3362. mutex_init(&phb->ioda.pe_list_mutex);
  3363. /* Calculate how many 32-bit TCE segments we have */
  3364. phb->ioda.dma32_count = phb->ioda.m32_pci_base /
  3365. PNV_IODA1_DMA32_SEGSIZE;
  3366. #if 0 /* We should really do that ... */
  3367. rc = opal_pci_set_phb_mem_window(opal->phb_id,
  3368. window_type,
  3369. window_num,
  3370. starting_real_address,
  3371. starting_pci_address,
  3372. segment_size);
  3373. #endif
  3374. pr_info(" %03d (%03d) PE's M32: 0x%x [segment=0x%x]\n",
  3375. phb->ioda.total_pe_num, phb->ioda.reserved_pe_idx,
  3376. phb->ioda.m32_size, phb->ioda.m32_segsize);
  3377. if (phb->ioda.m64_size)
  3378. pr_info(" M64: 0x%lx [segment=0x%lx]\n",
  3379. phb->ioda.m64_size, phb->ioda.m64_segsize);
  3380. if (phb->ioda.io_size)
  3381. pr_info(" IO: 0x%x [segment=0x%x]\n",
  3382. phb->ioda.io_size, phb->ioda.io_segsize);
  3383. phb->hose->ops = &pnv_pci_ops;
  3384. phb->get_pe_state = pnv_ioda_get_pe_state;
  3385. phb->freeze_pe = pnv_ioda_freeze_pe;
  3386. phb->unfreeze_pe = pnv_ioda_unfreeze_pe;
  3387. /* Setup MSI support */
  3388. pnv_pci_init_ioda_msis(phb);
  3389. /*
  3390. * We pass the PCI probe flag PCI_REASSIGN_ALL_RSRC here
  3391. * to let the PCI core do resource assignment. It's supposed
  3392. * that the PCI core will do correct I/O and MMIO alignment
  3393. * for the P2P bridge bars so that each PCI bus (excluding
  3394. * the child P2P bridges) can form individual PE.
  3395. */
  3396. ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
  3397. switch (phb->type) {
  3398. case PNV_PHB_NPU_NVLINK:
  3399. hose->controller_ops = pnv_npu_ioda_controller_ops;
  3400. break;
  3401. case PNV_PHB_NPU_OCAPI:
  3402. hose->controller_ops = pnv_npu_ocapi_ioda_controller_ops;
  3403. break;
  3404. default:
  3405. phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
  3406. hose->controller_ops = pnv_pci_ioda_controller_ops;
  3407. }
  3408. ppc_md.pcibios_default_alignment = pnv_pci_default_alignment;
  3409. #ifdef CONFIG_PCI_IOV
  3410. ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov;
  3411. ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
  3412. ppc_md.pcibios_sriov_enable = pnv_pcibios_sriov_enable;
  3413. ppc_md.pcibios_sriov_disable = pnv_pcibios_sriov_disable;
  3414. #endif
  3415. pci_add_flags(PCI_REASSIGN_ALL_RSRC);
  3416. /* Reset IODA tables to a clean state */
  3417. rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
  3418. if (rc)
  3419. pr_warn(" OPAL Error %ld performing IODA table reset !\n", rc);
  3420. /*
  3421. * If we're running in kdump kernel, the previous kernel never
  3422. * shutdown PCI devices correctly. We already got IODA table
  3423. * cleaned out. So we have to issue PHB reset to stop all PCI
  3424. * transactions from previous kernel. The ppc_pci_reset_phbs
  3425. * kernel parameter will force this reset too.
  3426. */
  3427. if (is_kdump_kernel() || pci_reset_phbs) {
  3428. pr_info(" Issue PHB reset ...\n");
  3429. pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
  3430. pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
  3431. }
  3432. /* Remove M64 resource if we can't configure it successfully */
  3433. if (!phb->init_m64 || phb->init_m64(phb))
  3434. hose->mem_resources[1].flags = 0;
  3435. }
  3436. void __init pnv_pci_init_ioda2_phb(struct device_node *np)
  3437. {
  3438. pnv_pci_init_ioda_phb(np, 0, PNV_PHB_IODA2);
  3439. }
  3440. void __init pnv_pci_init_npu_phb(struct device_node *np)
  3441. {
  3442. pnv_pci_init_ioda_phb(np, 0, PNV_PHB_NPU_NVLINK);
  3443. }
  3444. void __init pnv_pci_init_npu2_opencapi_phb(struct device_node *np)
  3445. {
  3446. pnv_pci_init_ioda_phb(np, 0, PNV_PHB_NPU_OCAPI);
  3447. }
  3448. static void pnv_npu2_opencapi_cfg_size_fixup(struct pci_dev *dev)
  3449. {
  3450. struct pci_controller *hose = pci_bus_to_host(dev->bus);
  3451. struct pnv_phb *phb = hose->private_data;
  3452. if (!machine_is(powernv))
  3453. return;
  3454. if (phb->type == PNV_PHB_NPU_OCAPI)
  3455. dev->cfg_size = PCI_CFG_SPACE_EXP_SIZE;
  3456. }
  3457. DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, pnv_npu2_opencapi_cfg_size_fixup);
  3458. void __init pnv_pci_init_ioda_hub(struct device_node *np)
  3459. {
  3460. struct device_node *phbn;
  3461. const __be64 *prop64;
  3462. u64 hub_id;
  3463. pr_info("Probing IODA IO-Hub %pOF\n", np);
  3464. prop64 = of_get_property(np, "ibm,opal-hubid", NULL);
  3465. if (!prop64) {
  3466. pr_err(" Missing \"ibm,opal-hubid\" property !\n");
  3467. return;
  3468. }
  3469. hub_id = be64_to_cpup(prop64);
  3470. pr_devel(" HUB-ID : 0x%016llx\n", hub_id);
  3471. /* Count child PHBs */
  3472. for_each_child_of_node(np, phbn) {
  3473. /* Look for IODA1 PHBs */
  3474. if (of_device_is_compatible(phbn, "ibm,ioda-phb"))
  3475. pnv_pci_init_ioda_phb(phbn, hub_id, PNV_PHB_IODA1);
  3476. }
  3477. }