book3s_hv_rmhandlers.S 86 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541
  1. /*
  2. * This program is free software; you can redistribute it and/or modify
  3. * it under the terms of the GNU General Public License, version 2, as
  4. * published by the Free Software Foundation.
  5. *
  6. * This program is distributed in the hope that it will be useful,
  7. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  8. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  9. * GNU General Public License for more details.
  10. *
  11. * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  12. *
  13. * Derived from book3s_rmhandlers.S and other files, which are:
  14. *
  15. * Copyright SUSE Linux Products GmbH 2009
  16. *
  17. * Authors: Alexander Graf <agraf@suse.de>
  18. */
  19. #include <asm/ppc_asm.h>
  20. #include <asm/code-patching-asm.h>
  21. #include <asm/kvm_asm.h>
  22. #include <asm/reg.h>
  23. #include <asm/mmu.h>
  24. #include <asm/page.h>
  25. #include <asm/ptrace.h>
  26. #include <asm/hvcall.h>
  27. #include <asm/asm-offsets.h>
  28. #include <asm/exception-64s.h>
  29. #include <asm/kvm_book3s_asm.h>
  30. #include <asm/book3s/64/mmu-hash.h>
  31. #include <asm/tm.h>
  32. #include <asm/opal.h>
  33. #include <asm/xive-regs.h>
  34. #include <asm/thread_info.h>
  35. #include <asm/asm-compat.h>
  36. #include <asm/feature-fixups.h>
  37. /* Sign-extend HDEC if not on POWER9 */
  38. #define EXTEND_HDEC(reg) \
  39. BEGIN_FTR_SECTION; \
  40. extsw reg, reg; \
  41. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
  42. /* Values in HSTATE_NAPPING(r13) */
  43. #define NAPPING_CEDE 1
  44. #define NAPPING_NOVCPU 2
  45. /* Stack frame offsets for kvmppc_hv_entry */
  46. #define SFS 160
  47. #define STACK_SLOT_TRAP (SFS-4)
  48. #define STACK_SLOT_TID (SFS-16)
  49. #define STACK_SLOT_PSSCR (SFS-24)
  50. #define STACK_SLOT_PID (SFS-32)
  51. #define STACK_SLOT_IAMR (SFS-40)
  52. #define STACK_SLOT_CIABR (SFS-48)
  53. #define STACK_SLOT_DAWR (SFS-56)
  54. #define STACK_SLOT_DAWRX (SFS-64)
  55. #define STACK_SLOT_HFSCR (SFS-72)
  56. #define STACK_SLOT_AMR (SFS-80)
  57. #define STACK_SLOT_UAMOR (SFS-88)
  58. /*
  59. * Call kvmppc_hv_entry in real mode.
  60. * Must be called with interrupts hard-disabled.
  61. *
  62. * Input Registers:
  63. *
  64. * LR = return address to continue at after eventually re-enabling MMU
  65. */
  66. _GLOBAL_TOC(kvmppc_hv_entry_trampoline)
  67. mflr r0
  68. std r0, PPC_LR_STKOFF(r1)
  69. stdu r1, -112(r1)
  70. mfmsr r10
  71. std r10, HSTATE_HOST_MSR(r13)
  72. LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
  73. li r0,MSR_RI
  74. andc r0,r10,r0
  75. li r6,MSR_IR | MSR_DR
  76. andc r6,r10,r6
  77. mtmsrd r0,1 /* clear RI in MSR */
  78. mtsrr0 r5
  79. mtsrr1 r6
  80. RFI_TO_KERNEL
  81. kvmppc_call_hv_entry:
  82. BEGIN_FTR_SECTION
  83. /* On P9, do LPCR setting, if necessary */
  84. ld r3, HSTATE_SPLIT_MODE(r13)
  85. cmpdi r3, 0
  86. beq 46f
  87. lwz r4, KVM_SPLIT_DO_SET(r3)
  88. cmpwi r4, 0
  89. beq 46f
  90. bl kvmhv_p9_set_lpcr
  91. nop
  92. 46:
  93. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  94. ld r4, HSTATE_KVM_VCPU(r13)
  95. bl kvmppc_hv_entry
  96. /* Back from guest - restore host state and return to caller */
  97. BEGIN_FTR_SECTION
  98. /* Restore host DABR and DABRX */
  99. ld r5,HSTATE_DABR(r13)
  100. li r6,7
  101. mtspr SPRN_DABR,r5
  102. mtspr SPRN_DABRX,r6
  103. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
  104. /* Restore SPRG3 */
  105. ld r3,PACA_SPRG_VDSO(r13)
  106. mtspr SPRN_SPRG_VDSO_WRITE,r3
  107. /* Reload the host's PMU registers */
  108. lbz r4, PACA_PMCINUSE(r13) /* is the host using the PMU? */
  109. cmpwi r4, 0
  110. beq 23f /* skip if not */
  111. BEGIN_FTR_SECTION
  112. ld r3, HSTATE_MMCR0(r13)
  113. andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
  114. cmpwi r4, MMCR0_PMAO
  115. beql kvmppc_fix_pmao
  116. END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
  117. lwz r3, HSTATE_PMC1(r13)
  118. lwz r4, HSTATE_PMC2(r13)
  119. lwz r5, HSTATE_PMC3(r13)
  120. lwz r6, HSTATE_PMC4(r13)
  121. lwz r8, HSTATE_PMC5(r13)
  122. lwz r9, HSTATE_PMC6(r13)
  123. mtspr SPRN_PMC1, r3
  124. mtspr SPRN_PMC2, r4
  125. mtspr SPRN_PMC3, r5
  126. mtspr SPRN_PMC4, r6
  127. mtspr SPRN_PMC5, r8
  128. mtspr SPRN_PMC6, r9
  129. ld r3, HSTATE_MMCR0(r13)
  130. ld r4, HSTATE_MMCR1(r13)
  131. ld r5, HSTATE_MMCRA(r13)
  132. ld r6, HSTATE_SIAR(r13)
  133. ld r7, HSTATE_SDAR(r13)
  134. mtspr SPRN_MMCR1, r4
  135. mtspr SPRN_MMCRA, r5
  136. mtspr SPRN_SIAR, r6
  137. mtspr SPRN_SDAR, r7
  138. BEGIN_FTR_SECTION
  139. ld r8, HSTATE_MMCR2(r13)
  140. ld r9, HSTATE_SIER(r13)
  141. mtspr SPRN_MMCR2, r8
  142. mtspr SPRN_SIER, r9
  143. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  144. mtspr SPRN_MMCR0, r3
  145. isync
  146. 23:
  147. /*
  148. * Reload DEC. HDEC interrupts were disabled when
  149. * we reloaded the host's LPCR value.
  150. */
  151. ld r3, HSTATE_DECEXP(r13)
  152. mftb r4
  153. subf r4, r4, r3
  154. mtspr SPRN_DEC, r4
  155. /* hwthread_req may have got set by cede or no vcpu, so clear it */
  156. li r0, 0
  157. stb r0, HSTATE_HWTHREAD_REQ(r13)
  158. /*
  159. * For external interrupts we need to call the Linux
  160. * handler to process the interrupt. We do that by jumping
  161. * to absolute address 0x500 for external interrupts.
  162. * The [h]rfid at the end of the handler will return to
  163. * the book3s_hv_interrupts.S code. For other interrupts
  164. * we do the rfid to get back to the book3s_hv_interrupts.S
  165. * code here.
  166. */
  167. ld r8, 112+PPC_LR_STKOFF(r1)
  168. addi r1, r1, 112
  169. ld r7, HSTATE_HOST_MSR(r13)
  170. /* Return the trap number on this thread as the return value */
  171. mr r3, r12
  172. /*
  173. * If we came back from the guest via a relocation-on interrupt,
  174. * we will be in virtual mode at this point, which makes it a
  175. * little easier to get back to the caller.
  176. */
  177. mfmsr r0
  178. andi. r0, r0, MSR_IR /* in real mode? */
  179. bne .Lvirt_return
  180. /* RFI into the highmem handler */
  181. mfmsr r6
  182. li r0, MSR_RI
  183. andc r6, r6, r0
  184. mtmsrd r6, 1 /* Clear RI in MSR */
  185. mtsrr0 r8
  186. mtsrr1 r7
  187. RFI_TO_KERNEL
  188. /* Virtual-mode return */
  189. .Lvirt_return:
  190. mtlr r8
  191. blr
  192. kvmppc_primary_no_guest:
  193. /* We handle this much like a ceded vcpu */
  194. /* put the HDEC into the DEC, since HDEC interrupts don't wake us */
  195. /* HDEC may be larger than DEC for arch >= v3.00, but since the */
  196. /* HDEC value came from DEC in the first place, it will fit */
  197. mfspr r3, SPRN_HDEC
  198. mtspr SPRN_DEC, r3
  199. /*
  200. * Make sure the primary has finished the MMU switch.
  201. * We should never get here on a secondary thread, but
  202. * check it for robustness' sake.
  203. */
  204. ld r5, HSTATE_KVM_VCORE(r13)
  205. 65: lbz r0, VCORE_IN_GUEST(r5)
  206. cmpwi r0, 0
  207. beq 65b
  208. /* Set LPCR. */
  209. ld r8,VCORE_LPCR(r5)
  210. mtspr SPRN_LPCR,r8
  211. isync
  212. /* set our bit in napping_threads */
  213. ld r5, HSTATE_KVM_VCORE(r13)
  214. lbz r7, HSTATE_PTID(r13)
  215. li r0, 1
  216. sld r0, r0, r7
  217. addi r6, r5, VCORE_NAPPING_THREADS
  218. 1: lwarx r3, 0, r6
  219. or r3, r3, r0
  220. stwcx. r3, 0, r6
  221. bne 1b
  222. /* order napping_threads update vs testing entry_exit_map */
  223. isync
  224. li r12, 0
  225. lwz r7, VCORE_ENTRY_EXIT(r5)
  226. cmpwi r7, 0x100
  227. bge kvm_novcpu_exit /* another thread already exiting */
  228. li r3, NAPPING_NOVCPU
  229. stb r3, HSTATE_NAPPING(r13)
  230. li r3, 0 /* Don't wake on privileged (OS) doorbell */
  231. b kvm_do_nap
  232. /*
  233. * kvm_novcpu_wakeup
  234. * Entered from kvm_start_guest if kvm_hstate.napping is set
  235. * to NAPPING_NOVCPU
  236. * r2 = kernel TOC
  237. * r13 = paca
  238. */
  239. kvm_novcpu_wakeup:
  240. ld r1, HSTATE_HOST_R1(r13)
  241. ld r5, HSTATE_KVM_VCORE(r13)
  242. li r0, 0
  243. stb r0, HSTATE_NAPPING(r13)
  244. /* check the wake reason */
  245. bl kvmppc_check_wake_reason
  246. /*
  247. * Restore volatile registers since we could have called
  248. * a C routine in kvmppc_check_wake_reason.
  249. * r5 = VCORE
  250. */
  251. ld r5, HSTATE_KVM_VCORE(r13)
  252. /* see if any other thread is already exiting */
  253. lwz r0, VCORE_ENTRY_EXIT(r5)
  254. cmpwi r0, 0x100
  255. bge kvm_novcpu_exit
  256. /* clear our bit in napping_threads */
  257. lbz r7, HSTATE_PTID(r13)
  258. li r0, 1
  259. sld r0, r0, r7
  260. addi r6, r5, VCORE_NAPPING_THREADS
  261. 4: lwarx r7, 0, r6
  262. andc r7, r7, r0
  263. stwcx. r7, 0, r6
  264. bne 4b
  265. /* See if the wake reason means we need to exit */
  266. cmpdi r3, 0
  267. bge kvm_novcpu_exit
  268. /* See if our timeslice has expired (HDEC is negative) */
  269. mfspr r0, SPRN_HDEC
  270. EXTEND_HDEC(r0)
  271. li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
  272. cmpdi r0, 0
  273. blt kvm_novcpu_exit
  274. /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
  275. ld r4, HSTATE_KVM_VCPU(r13)
  276. cmpdi r4, 0
  277. beq kvmppc_primary_no_guest
  278. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  279. addi r3, r4, VCPU_TB_RMENTRY
  280. bl kvmhv_start_timing
  281. #endif
  282. b kvmppc_got_guest
  283. kvm_novcpu_exit:
  284. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  285. ld r4, HSTATE_KVM_VCPU(r13)
  286. cmpdi r4, 0
  287. beq 13f
  288. addi r3, r4, VCPU_TB_RMEXIT
  289. bl kvmhv_accumulate_time
  290. #endif
  291. 13: mr r3, r12
  292. stw r12, STACK_SLOT_TRAP(r1)
  293. bl kvmhv_commence_exit
  294. nop
  295. b kvmhv_switch_to_host
  296. /*
  297. * We come in here when wakened from nap mode.
  298. * Relocation is off and most register values are lost.
  299. * r13 points to the PACA.
  300. * r3 contains the SRR1 wakeup value, SRR1 is trashed.
  301. */
  302. .globl kvm_start_guest
  303. kvm_start_guest:
  304. /* Set runlatch bit the minute you wake up from nap */
  305. mfspr r0, SPRN_CTRLF
  306. ori r0, r0, 1
  307. mtspr SPRN_CTRLT, r0
  308. /*
  309. * Could avoid this and pass it through in r3. For now,
  310. * code expects it to be in SRR1.
  311. */
  312. mtspr SPRN_SRR1,r3
  313. ld r2,PACATOC(r13)
  314. li r0,0
  315. stb r0,PACA_FTRACE_ENABLED(r13)
  316. li r0,KVM_HWTHREAD_IN_KVM
  317. stb r0,HSTATE_HWTHREAD_STATE(r13)
  318. /* NV GPR values from power7_idle() will no longer be valid */
  319. li r0,1
  320. stb r0,PACA_NAPSTATELOST(r13)
  321. /* were we napping due to cede? */
  322. lbz r0,HSTATE_NAPPING(r13)
  323. cmpwi r0,NAPPING_CEDE
  324. beq kvm_end_cede
  325. cmpwi r0,NAPPING_NOVCPU
  326. beq kvm_novcpu_wakeup
  327. ld r1,PACAEMERGSP(r13)
  328. subi r1,r1,STACK_FRAME_OVERHEAD
  329. /*
  330. * We weren't napping due to cede, so this must be a secondary
  331. * thread being woken up to run a guest, or being woken up due
  332. * to a stray IPI. (Or due to some machine check or hypervisor
  333. * maintenance interrupt while the core is in KVM.)
  334. */
  335. /* Check the wake reason in SRR1 to see why we got here */
  336. bl kvmppc_check_wake_reason
  337. /*
  338. * kvmppc_check_wake_reason could invoke a C routine, but we
  339. * have no volatile registers to restore when we return.
  340. */
  341. cmpdi r3, 0
  342. bge kvm_no_guest
  343. /* get vcore pointer, NULL if we have nothing to run */
  344. ld r5,HSTATE_KVM_VCORE(r13)
  345. cmpdi r5,0
  346. /* if we have no vcore to run, go back to sleep */
  347. beq kvm_no_guest
  348. kvm_secondary_got_guest:
  349. /* Set HSTATE_DSCR(r13) to something sensible */
  350. ld r6, PACA_DSCR_DEFAULT(r13)
  351. std r6, HSTATE_DSCR(r13)
  352. /* On thread 0 of a subcore, set HDEC to max */
  353. lbz r4, HSTATE_PTID(r13)
  354. cmpwi r4, 0
  355. bne 63f
  356. LOAD_REG_ADDR(r6, decrementer_max)
  357. ld r6, 0(r6)
  358. mtspr SPRN_HDEC, r6
  359. /* and set per-LPAR registers, if doing dynamic micro-threading */
  360. ld r6, HSTATE_SPLIT_MODE(r13)
  361. cmpdi r6, 0
  362. beq 63f
  363. BEGIN_FTR_SECTION
  364. ld r0, KVM_SPLIT_RPR(r6)
  365. mtspr SPRN_RPR, r0
  366. ld r0, KVM_SPLIT_PMMAR(r6)
  367. mtspr SPRN_PMMAR, r0
  368. ld r0, KVM_SPLIT_LDBAR(r6)
  369. mtspr SPRN_LDBAR, r0
  370. isync
  371. FTR_SECTION_ELSE
  372. /* On P9 we use the split_info for coordinating LPCR changes */
  373. lwz r4, KVM_SPLIT_DO_SET(r6)
  374. cmpwi r4, 0
  375. beq 1f
  376. mr r3, r6
  377. bl kvmhv_p9_set_lpcr
  378. nop
  379. 1:
  380. ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
  381. 63:
  382. /* Order load of vcpu after load of vcore */
  383. lwsync
  384. ld r4, HSTATE_KVM_VCPU(r13)
  385. bl kvmppc_hv_entry
  386. /* Back from the guest, go back to nap */
  387. /* Clear our vcpu and vcore pointers so we don't come back in early */
  388. li r0, 0
  389. std r0, HSTATE_KVM_VCPU(r13)
  390. /*
  391. * Once we clear HSTATE_KVM_VCORE(r13), the code in
  392. * kvmppc_run_core() is going to assume that all our vcpu
  393. * state is visible in memory. This lwsync makes sure
  394. * that that is true.
  395. */
  396. lwsync
  397. std r0, HSTATE_KVM_VCORE(r13)
  398. /*
  399. * All secondaries exiting guest will fall through this path.
  400. * Before proceeding, just check for HMI interrupt and
  401. * invoke opal hmi handler. By now we are sure that the
  402. * primary thread on this core/subcore has already made partition
  403. * switch/TB resync and we are good to call opal hmi handler.
  404. */
  405. cmpwi r12, BOOK3S_INTERRUPT_HMI
  406. bne kvm_no_guest
  407. li r3,0 /* NULL argument */
  408. bl hmi_exception_realmode
  409. /*
  410. * At this point we have finished executing in the guest.
  411. * We need to wait for hwthread_req to become zero, since
  412. * we may not turn on the MMU while hwthread_req is non-zero.
  413. * While waiting we also need to check if we get given a vcpu to run.
  414. */
  415. kvm_no_guest:
  416. lbz r3, HSTATE_HWTHREAD_REQ(r13)
  417. cmpwi r3, 0
  418. bne 53f
  419. HMT_MEDIUM
  420. li r0, KVM_HWTHREAD_IN_KERNEL
  421. stb r0, HSTATE_HWTHREAD_STATE(r13)
  422. /* need to recheck hwthread_req after a barrier, to avoid race */
  423. sync
  424. lbz r3, HSTATE_HWTHREAD_REQ(r13)
  425. cmpwi r3, 0
  426. bne 54f
  427. /*
  428. * We jump to pnv_wakeup_loss, which will return to the caller
  429. * of power7_nap in the powernv cpu offline loop. The value we
  430. * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss
  431. * requires SRR1 in r12.
  432. */
  433. li r3, LPCR_PECE0
  434. mfspr r4, SPRN_LPCR
  435. rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
  436. mtspr SPRN_LPCR, r4
  437. li r3, 0
  438. mfspr r12,SPRN_SRR1
  439. b pnv_wakeup_loss
  440. 53: HMT_LOW
  441. ld r5, HSTATE_KVM_VCORE(r13)
  442. cmpdi r5, 0
  443. bne 60f
  444. ld r3, HSTATE_SPLIT_MODE(r13)
  445. cmpdi r3, 0
  446. beq kvm_no_guest
  447. lwz r0, KVM_SPLIT_DO_SET(r3)
  448. cmpwi r0, 0
  449. bne kvmhv_do_set
  450. lwz r0, KVM_SPLIT_DO_RESTORE(r3)
  451. cmpwi r0, 0
  452. bne kvmhv_do_restore
  453. lbz r0, KVM_SPLIT_DO_NAP(r3)
  454. cmpwi r0, 0
  455. beq kvm_no_guest
  456. HMT_MEDIUM
  457. b kvm_unsplit_nap
  458. 60: HMT_MEDIUM
  459. b kvm_secondary_got_guest
  460. 54: li r0, KVM_HWTHREAD_IN_KVM
  461. stb r0, HSTATE_HWTHREAD_STATE(r13)
  462. b kvm_no_guest
  463. kvmhv_do_set:
  464. /* Set LPCR, LPIDR etc. on P9 */
  465. HMT_MEDIUM
  466. bl kvmhv_p9_set_lpcr
  467. nop
  468. b kvm_no_guest
  469. kvmhv_do_restore:
  470. HMT_MEDIUM
  471. bl kvmhv_p9_restore_lpcr
  472. nop
  473. b kvm_no_guest
  474. /*
  475. * Here the primary thread is trying to return the core to
  476. * whole-core mode, so we need to nap.
  477. */
  478. kvm_unsplit_nap:
  479. /*
  480. * When secondaries are napping in kvm_unsplit_nap() with
  481. * hwthread_req = 1, HMI goes ignored even though subcores are
  482. * already exited the guest. Hence HMI keeps waking up secondaries
  483. * from nap in a loop and secondaries always go back to nap since
  484. * no vcore is assigned to them. This makes impossible for primary
  485. * thread to get hold of secondary threads resulting into a soft
  486. * lockup in KVM path.
  487. *
  488. * Let us check if HMI is pending and handle it before we go to nap.
  489. */
  490. cmpwi r12, BOOK3S_INTERRUPT_HMI
  491. bne 55f
  492. li r3, 0 /* NULL argument */
  493. bl hmi_exception_realmode
  494. 55:
  495. /*
  496. * Ensure that secondary doesn't nap when it has
  497. * its vcore pointer set.
  498. */
  499. sync /* matches smp_mb() before setting split_info.do_nap */
  500. ld r0, HSTATE_KVM_VCORE(r13)
  501. cmpdi r0, 0
  502. bne kvm_no_guest
  503. /* clear any pending message */
  504. BEGIN_FTR_SECTION
  505. lis r6, (PPC_DBELL_SERVER << (63-36))@h
  506. PPC_MSGCLR(6)
  507. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  508. /* Set kvm_split_mode.napped[tid] = 1 */
  509. ld r3, HSTATE_SPLIT_MODE(r13)
  510. li r0, 1
  511. lbz r4, HSTATE_TID(r13)
  512. addi r4, r4, KVM_SPLIT_NAPPED
  513. stbx r0, r3, r4
  514. /* Check the do_nap flag again after setting napped[] */
  515. sync
  516. lbz r0, KVM_SPLIT_DO_NAP(r3)
  517. cmpwi r0, 0
  518. beq 57f
  519. li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
  520. mfspr r5, SPRN_LPCR
  521. rlwimi r5, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
  522. b kvm_nap_sequence
  523. 57: li r0, 0
  524. stbx r0, r3, r4
  525. b kvm_no_guest
  526. /******************************************************************************
  527. * *
  528. * Entry code *
  529. * *
  530. *****************************************************************************/
  531. .global kvmppc_hv_entry
  532. kvmppc_hv_entry:
  533. /* Required state:
  534. *
  535. * R4 = vcpu pointer (or NULL)
  536. * MSR = ~IR|DR
  537. * R13 = PACA
  538. * R1 = host R1
  539. * R2 = TOC
  540. * all other volatile GPRS = free
  541. * Does not preserve non-volatile GPRs or CR fields
  542. */
  543. mflr r0
  544. std r0, PPC_LR_STKOFF(r1)
  545. stdu r1, -SFS(r1)
  546. /* Save R1 in the PACA */
  547. std r1, HSTATE_HOST_R1(r13)
  548. li r6, KVM_GUEST_MODE_HOST_HV
  549. stb r6, HSTATE_IN_GUEST(r13)
  550. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  551. /* Store initial timestamp */
  552. cmpdi r4, 0
  553. beq 1f
  554. addi r3, r4, VCPU_TB_RMENTRY
  555. bl kvmhv_start_timing
  556. 1:
  557. #endif
  558. /* Use cr7 as an indication of radix mode */
  559. ld r5, HSTATE_KVM_VCORE(r13)
  560. ld r9, VCORE_KVM(r5) /* pointer to struct kvm */
  561. lbz r0, KVM_RADIX(r9)
  562. cmpwi cr7, r0, 0
  563. /*
  564. * POWER7/POWER8 host -> guest partition switch code.
  565. * We don't have to lock against concurrent tlbies,
  566. * but we do have to coordinate across hardware threads.
  567. */
  568. /* Set bit in entry map iff exit map is zero. */
  569. li r7, 1
  570. lbz r6, HSTATE_PTID(r13)
  571. sld r7, r7, r6
  572. addi r8, r5, VCORE_ENTRY_EXIT
  573. 21: lwarx r3, 0, r8
  574. cmpwi r3, 0x100 /* any threads starting to exit? */
  575. bge secondary_too_late /* if so we're too late to the party */
  576. or r3, r3, r7
  577. stwcx. r3, 0, r8
  578. bne 21b
  579. /* Primary thread switches to guest partition. */
  580. cmpwi r6,0
  581. bne 10f
  582. /* Radix has already switched LPID and flushed core TLB */
  583. bne cr7, 22f
  584. lwz r7,KVM_LPID(r9)
  585. BEGIN_FTR_SECTION
  586. ld r6,KVM_SDR1(r9)
  587. li r0,LPID_RSVD /* switch to reserved LPID */
  588. mtspr SPRN_LPID,r0
  589. ptesync
  590. mtspr SPRN_SDR1,r6 /* switch to partition page table */
  591. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
  592. mtspr SPRN_LPID,r7
  593. isync
  594. /* See if we need to flush the TLB. Hash has to be done in RM */
  595. lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */
  596. BEGIN_FTR_SECTION
  597. /*
  598. * On POWER9, individual threads can come in here, but the
  599. * TLB is shared between the 4 threads in a core, hence
  600. * invalidating on one thread invalidates for all.
  601. * Thus we make all 4 threads use the same bit here.
  602. */
  603. clrrdi r6,r6,2
  604. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  605. clrldi r7,r6,64-6 /* extract bit number (6 bits) */
  606. srdi r6,r6,6 /* doubleword number */
  607. sldi r6,r6,3 /* address offset */
  608. add r6,r6,r9
  609. addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */
  610. li r8,1
  611. sld r8,r8,r7
  612. ld r7,0(r6)
  613. and. r7,r7,r8
  614. beq 22f
  615. /* Flush the TLB of any entries for this LPID */
  616. lwz r0,KVM_TLB_SETS(r9)
  617. mtctr r0
  618. li r7,0x800 /* IS field = 0b10 */
  619. ptesync
  620. li r0,0 /* RS for P9 version of tlbiel */
  621. 28: tlbiel r7 /* On P9, rs=0, RIC=0, PRS=0, R=0 */
  622. addi r7,r7,0x1000
  623. bdnz 28b
  624. ptesync
  625. 23: ldarx r7,0,r6 /* clear the bit after TLB flushed */
  626. andc r7,r7,r8
  627. stdcx. r7,0,r6
  628. bne 23b
  629. /* Add timebase offset onto timebase */
  630. 22: ld r8,VCORE_TB_OFFSET(r5)
  631. cmpdi r8,0
  632. beq 37f
  633. std r8, VCORE_TB_OFFSET_APPL(r5)
  634. mftb r6 /* current host timebase */
  635. add r8,r8,r6
  636. mtspr SPRN_TBU40,r8 /* update upper 40 bits */
  637. mftb r7 /* check if lower 24 bits overflowed */
  638. clrldi r6,r6,40
  639. clrldi r7,r7,40
  640. cmpld r7,r6
  641. bge 37f
  642. addis r8,r8,0x100 /* if so, increment upper 40 bits */
  643. mtspr SPRN_TBU40,r8
  644. /* Load guest PCR value to select appropriate compat mode */
  645. 37: ld r7, VCORE_PCR(r5)
  646. cmpdi r7, 0
  647. beq 38f
  648. mtspr SPRN_PCR, r7
  649. 38:
  650. BEGIN_FTR_SECTION
  651. /* DPDES and VTB are shared between threads */
  652. ld r8, VCORE_DPDES(r5)
  653. ld r7, VCORE_VTB(r5)
  654. mtspr SPRN_DPDES, r8
  655. mtspr SPRN_VTB, r7
  656. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  657. /* Mark the subcore state as inside guest */
  658. bl kvmppc_subcore_enter_guest
  659. nop
  660. ld r5, HSTATE_KVM_VCORE(r13)
  661. ld r4, HSTATE_KVM_VCPU(r13)
  662. li r0,1
  663. stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
  664. /* Do we have a guest vcpu to run? */
  665. 10: cmpdi r4, 0
  666. beq kvmppc_primary_no_guest
  667. kvmppc_got_guest:
  668. /* Increment yield count if they have a VPA */
  669. ld r3, VCPU_VPA(r4)
  670. cmpdi r3, 0
  671. beq 25f
  672. li r6, LPPACA_YIELDCOUNT
  673. LWZX_BE r5, r3, r6
  674. addi r5, r5, 1
  675. STWX_BE r5, r3, r6
  676. li r6, 1
  677. stb r6, VCPU_VPA_DIRTY(r4)
  678. 25:
  679. /* Save purr/spurr */
  680. mfspr r5,SPRN_PURR
  681. mfspr r6,SPRN_SPURR
  682. std r5,HSTATE_PURR(r13)
  683. std r6,HSTATE_SPURR(r13)
  684. ld r7,VCPU_PURR(r4)
  685. ld r8,VCPU_SPURR(r4)
  686. mtspr SPRN_PURR,r7
  687. mtspr SPRN_SPURR,r8
  688. /* Save host values of some registers */
  689. BEGIN_FTR_SECTION
  690. mfspr r5, SPRN_TIDR
  691. mfspr r6, SPRN_PSSCR
  692. mfspr r7, SPRN_PID
  693. std r5, STACK_SLOT_TID(r1)
  694. std r6, STACK_SLOT_PSSCR(r1)
  695. std r7, STACK_SLOT_PID(r1)
  696. mfspr r5, SPRN_HFSCR
  697. std r5, STACK_SLOT_HFSCR(r1)
  698. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  699. BEGIN_FTR_SECTION
  700. mfspr r5, SPRN_CIABR
  701. mfspr r6, SPRN_DAWR
  702. mfspr r7, SPRN_DAWRX
  703. mfspr r8, SPRN_IAMR
  704. std r5, STACK_SLOT_CIABR(r1)
  705. std r6, STACK_SLOT_DAWR(r1)
  706. std r7, STACK_SLOT_DAWRX(r1)
  707. std r8, STACK_SLOT_IAMR(r1)
  708. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  709. mfspr r5, SPRN_AMR
  710. std r5, STACK_SLOT_AMR(r1)
  711. mfspr r6, SPRN_UAMOR
  712. std r6, STACK_SLOT_UAMOR(r1)
  713. BEGIN_FTR_SECTION
  714. /* Set partition DABR */
  715. /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
  716. lwz r5,VCPU_DABRX(r4)
  717. ld r6,VCPU_DABR(r4)
  718. mtspr SPRN_DABRX,r5
  719. mtspr SPRN_DABR,r6
  720. isync
  721. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
  722. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  723. /*
  724. * Branch around the call if both CPU_FTR_TM and
  725. * CPU_FTR_P9_TM_HV_ASSIST are off.
  726. */
  727. BEGIN_FTR_SECTION
  728. b 91f
  729. END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0)
  730. /*
  731. * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR
  732. */
  733. mr r3, r4
  734. ld r4, VCPU_MSR(r3)
  735. bl kvmppc_restore_tm_hv
  736. ld r4, HSTATE_KVM_VCPU(r13)
  737. 91:
  738. #endif
  739. /* Load guest PMU registers */
  740. /* R4 is live here (vcpu pointer) */
  741. li r3, 1
  742. sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
  743. mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
  744. isync
  745. BEGIN_FTR_SECTION
  746. ld r3, VCPU_MMCR(r4)
  747. andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
  748. cmpwi r5, MMCR0_PMAO
  749. beql kvmppc_fix_pmao
  750. END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
  751. lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
  752. lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
  753. lwz r6, VCPU_PMC + 8(r4)
  754. lwz r7, VCPU_PMC + 12(r4)
  755. lwz r8, VCPU_PMC + 16(r4)
  756. lwz r9, VCPU_PMC + 20(r4)
  757. mtspr SPRN_PMC1, r3
  758. mtspr SPRN_PMC2, r5
  759. mtspr SPRN_PMC3, r6
  760. mtspr SPRN_PMC4, r7
  761. mtspr SPRN_PMC5, r8
  762. mtspr SPRN_PMC6, r9
  763. ld r3, VCPU_MMCR(r4)
  764. ld r5, VCPU_MMCR + 8(r4)
  765. ld r6, VCPU_MMCR + 16(r4)
  766. ld r7, VCPU_SIAR(r4)
  767. ld r8, VCPU_SDAR(r4)
  768. mtspr SPRN_MMCR1, r5
  769. mtspr SPRN_MMCRA, r6
  770. mtspr SPRN_SIAR, r7
  771. mtspr SPRN_SDAR, r8
  772. BEGIN_FTR_SECTION
  773. ld r5, VCPU_MMCR + 24(r4)
  774. ld r6, VCPU_SIER(r4)
  775. mtspr SPRN_MMCR2, r5
  776. mtspr SPRN_SIER, r6
  777. BEGIN_FTR_SECTION_NESTED(96)
  778. lwz r7, VCPU_PMC + 24(r4)
  779. lwz r8, VCPU_PMC + 28(r4)
  780. ld r9, VCPU_MMCR + 32(r4)
  781. mtspr SPRN_SPMC1, r7
  782. mtspr SPRN_SPMC2, r8
  783. mtspr SPRN_MMCRS, r9
  784. END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96)
  785. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  786. mtspr SPRN_MMCR0, r3
  787. isync
  788. /* Load up FP, VMX and VSX registers */
  789. bl kvmppc_load_fp
  790. ld r14, VCPU_GPR(R14)(r4)
  791. ld r15, VCPU_GPR(R15)(r4)
  792. ld r16, VCPU_GPR(R16)(r4)
  793. ld r17, VCPU_GPR(R17)(r4)
  794. ld r18, VCPU_GPR(R18)(r4)
  795. ld r19, VCPU_GPR(R19)(r4)
  796. ld r20, VCPU_GPR(R20)(r4)
  797. ld r21, VCPU_GPR(R21)(r4)
  798. ld r22, VCPU_GPR(R22)(r4)
  799. ld r23, VCPU_GPR(R23)(r4)
  800. ld r24, VCPU_GPR(R24)(r4)
  801. ld r25, VCPU_GPR(R25)(r4)
  802. ld r26, VCPU_GPR(R26)(r4)
  803. ld r27, VCPU_GPR(R27)(r4)
  804. ld r28, VCPU_GPR(R28)(r4)
  805. ld r29, VCPU_GPR(R29)(r4)
  806. ld r30, VCPU_GPR(R30)(r4)
  807. ld r31, VCPU_GPR(R31)(r4)
  808. /* Switch DSCR to guest value */
  809. ld r5, VCPU_DSCR(r4)
  810. mtspr SPRN_DSCR, r5
  811. BEGIN_FTR_SECTION
  812. /* Skip next section on POWER7 */
  813. b 8f
  814. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
  815. /* Load up POWER8-specific registers */
  816. ld r5, VCPU_IAMR(r4)
  817. lwz r6, VCPU_PSPB(r4)
  818. ld r7, VCPU_FSCR(r4)
  819. mtspr SPRN_IAMR, r5
  820. mtspr SPRN_PSPB, r6
  821. mtspr SPRN_FSCR, r7
  822. ld r5, VCPU_DAWR(r4)
  823. ld r6, VCPU_DAWRX(r4)
  824. ld r7, VCPU_CIABR(r4)
  825. ld r8, VCPU_TAR(r4)
  826. /*
  827. * Handle broken DAWR case by not writing it. This means we
  828. * can still store the DAWR register for migration.
  829. */
  830. BEGIN_FTR_SECTION
  831. mtspr SPRN_DAWR, r5
  832. mtspr SPRN_DAWRX, r6
  833. END_FTR_SECTION_IFSET(CPU_FTR_DAWR)
  834. mtspr SPRN_CIABR, r7
  835. mtspr SPRN_TAR, r8
  836. ld r5, VCPU_IC(r4)
  837. ld r8, VCPU_EBBHR(r4)
  838. mtspr SPRN_IC, r5
  839. mtspr SPRN_EBBHR, r8
  840. ld r5, VCPU_EBBRR(r4)
  841. ld r6, VCPU_BESCR(r4)
  842. lwz r7, VCPU_GUEST_PID(r4)
  843. ld r8, VCPU_WORT(r4)
  844. mtspr SPRN_EBBRR, r5
  845. mtspr SPRN_BESCR, r6
  846. mtspr SPRN_PID, r7
  847. mtspr SPRN_WORT, r8
  848. BEGIN_FTR_SECTION
  849. /* POWER8-only registers */
  850. ld r5, VCPU_TCSCR(r4)
  851. ld r6, VCPU_ACOP(r4)
  852. ld r7, VCPU_CSIGR(r4)
  853. ld r8, VCPU_TACR(r4)
  854. mtspr SPRN_TCSCR, r5
  855. mtspr SPRN_ACOP, r6
  856. mtspr SPRN_CSIGR, r7
  857. mtspr SPRN_TACR, r8
  858. nop
  859. FTR_SECTION_ELSE
  860. /* POWER9-only registers */
  861. ld r5, VCPU_TID(r4)
  862. ld r6, VCPU_PSSCR(r4)
  863. lbz r8, HSTATE_FAKE_SUSPEND(r13)
  864. oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */
  865. rldimi r6, r8, PSSCR_FAKE_SUSPEND_LG, 63 - PSSCR_FAKE_SUSPEND_LG
  866. ld r7, VCPU_HFSCR(r4)
  867. mtspr SPRN_TIDR, r5
  868. mtspr SPRN_PSSCR, r6
  869. mtspr SPRN_HFSCR, r7
  870. ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
  871. 8:
  872. ld r5, VCPU_SPRG0(r4)
  873. ld r6, VCPU_SPRG1(r4)
  874. ld r7, VCPU_SPRG2(r4)
  875. ld r8, VCPU_SPRG3(r4)
  876. mtspr SPRN_SPRG0, r5
  877. mtspr SPRN_SPRG1, r6
  878. mtspr SPRN_SPRG2, r7
  879. mtspr SPRN_SPRG3, r8
  880. /* Load up DAR and DSISR */
  881. ld r5, VCPU_DAR(r4)
  882. lwz r6, VCPU_DSISR(r4)
  883. mtspr SPRN_DAR, r5
  884. mtspr SPRN_DSISR, r6
  885. /* Restore AMR and UAMOR, set AMOR to all 1s */
  886. ld r5,VCPU_AMR(r4)
  887. ld r6,VCPU_UAMOR(r4)
  888. li r7,-1
  889. mtspr SPRN_AMR,r5
  890. mtspr SPRN_UAMOR,r6
  891. mtspr SPRN_AMOR,r7
  892. /* Restore state of CTRL run bit; assume 1 on entry */
  893. lwz r5,VCPU_CTRL(r4)
  894. andi. r5,r5,1
  895. bne 4f
  896. mfspr r6,SPRN_CTRLF
  897. clrrdi r6,r6,1
  898. mtspr SPRN_CTRLT,r6
  899. 4:
  900. /* Secondary threads wait for primary to have done partition switch */
  901. ld r5, HSTATE_KVM_VCORE(r13)
  902. lbz r6, HSTATE_PTID(r13)
  903. cmpwi r6, 0
  904. beq 21f
  905. lbz r0, VCORE_IN_GUEST(r5)
  906. cmpwi r0, 0
  907. bne 21f
  908. HMT_LOW
  909. 20: lwz r3, VCORE_ENTRY_EXIT(r5)
  910. cmpwi r3, 0x100
  911. bge no_switch_exit
  912. lbz r0, VCORE_IN_GUEST(r5)
  913. cmpwi r0, 0
  914. beq 20b
  915. HMT_MEDIUM
  916. 21:
  917. /* Set LPCR. */
  918. ld r8,VCORE_LPCR(r5)
  919. mtspr SPRN_LPCR,r8
  920. isync
  921. /*
  922. * Set the decrementer to the guest decrementer.
  923. */
  924. ld r8,VCPU_DEC_EXPIRES(r4)
  925. /* r8 is a host timebase value here, convert to guest TB */
  926. ld r5,HSTATE_KVM_VCORE(r13)
  927. ld r6,VCORE_TB_OFFSET_APPL(r5)
  928. add r8,r8,r6
  929. mftb r7
  930. subf r3,r7,r8
  931. mtspr SPRN_DEC,r3
  932. /* Check if HDEC expires soon */
  933. mfspr r3, SPRN_HDEC
  934. EXTEND_HDEC(r3)
  935. cmpdi r3, 512 /* 1 microsecond */
  936. blt hdec_soon
  937. /* For hash guest, clear out and reload the SLB */
  938. ld r6, VCPU_KVM(r4)
  939. lbz r0, KVM_RADIX(r6)
  940. cmpwi r0, 0
  941. bne 9f
  942. li r6, 0
  943. slbmte r6, r6
  944. slbia
  945. ptesync
  946. /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */
  947. lwz r5,VCPU_SLB_MAX(r4)
  948. cmpwi r5,0
  949. beq 9f
  950. mtctr r5
  951. addi r6,r4,VCPU_SLB
  952. 1: ld r8,VCPU_SLB_E(r6)
  953. ld r9,VCPU_SLB_V(r6)
  954. slbmte r9,r8
  955. addi r6,r6,VCPU_SLB_SIZE
  956. bdnz 1b
  957. 9:
  958. #ifdef CONFIG_KVM_XICS
  959. /* We are entering the guest on that thread, push VCPU to XIVE */
  960. ld r10, HSTATE_XIVE_TIMA_PHYS(r13)
  961. cmpldi cr0, r10, 0
  962. beq no_xive
  963. ld r11, VCPU_XIVE_SAVED_STATE(r4)
  964. li r9, TM_QW1_OS
  965. eieio
  966. stdcix r11,r9,r10
  967. lwz r11, VCPU_XIVE_CAM_WORD(r4)
  968. li r9, TM_QW1_OS + TM_WORD2
  969. stwcix r11,r9,r10
  970. li r9, 1
  971. stb r9, VCPU_XIVE_PUSHED(r4)
  972. eieio
  973. /*
  974. * We clear the irq_pending flag. There is a small chance of a
  975. * race vs. the escalation interrupt happening on another
  976. * processor setting it again, but the only consequence is to
  977. * cause a spurrious wakeup on the next H_CEDE which is not an
  978. * issue.
  979. */
  980. li r0,0
  981. stb r0, VCPU_IRQ_PENDING(r4)
  982. /*
  983. * In single escalation mode, if the escalation interrupt is
  984. * on, we mask it.
  985. */
  986. lbz r0, VCPU_XIVE_ESC_ON(r4)
  987. cmpwi r0,0
  988. beq 1f
  989. ld r10, VCPU_XIVE_ESC_RADDR(r4)
  990. li r9, XIVE_ESB_SET_PQ_01
  991. ldcix r0, r10, r9
  992. sync
  993. /* We have a possible subtle race here: The escalation interrupt might
  994. * have fired and be on its way to the host queue while we mask it,
  995. * and if we unmask it early enough (re-cede right away), there is
  996. * a theorical possibility that it fires again, thus landing in the
  997. * target queue more than once which is a big no-no.
  998. *
  999. * Fortunately, solving this is rather easy. If the above load setting
  1000. * PQ to 01 returns a previous value where P is set, then we know the
  1001. * escalation interrupt is somewhere on its way to the host. In that
  1002. * case we simply don't clear the xive_esc_on flag below. It will be
  1003. * eventually cleared by the handler for the escalation interrupt.
  1004. *
  1005. * Then, when doing a cede, we check that flag again before re-enabling
  1006. * the escalation interrupt, and if set, we abort the cede.
  1007. */
  1008. andi. r0, r0, XIVE_ESB_VAL_P
  1009. bne- 1f
  1010. /* Now P is 0, we can clear the flag */
  1011. li r0, 0
  1012. stb r0, VCPU_XIVE_ESC_ON(r4)
  1013. 1:
  1014. no_xive:
  1015. #endif /* CONFIG_KVM_XICS */
  1016. deliver_guest_interrupt:
  1017. ld r6, VCPU_CTR(r4)
  1018. ld r7, VCPU_XER(r4)
  1019. mtctr r6
  1020. mtxer r7
  1021. kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */
  1022. ld r10, VCPU_PC(r4)
  1023. ld r11, VCPU_MSR(r4)
  1024. ld r6, VCPU_SRR0(r4)
  1025. ld r7, VCPU_SRR1(r4)
  1026. mtspr SPRN_SRR0, r6
  1027. mtspr SPRN_SRR1, r7
  1028. /* r11 = vcpu->arch.msr & ~MSR_HV */
  1029. rldicl r11, r11, 63 - MSR_HV_LG, 1
  1030. rotldi r11, r11, 1 + MSR_HV_LG
  1031. ori r11, r11, MSR_ME
  1032. /* Check if we can deliver an external or decrementer interrupt now */
  1033. ld r0, VCPU_PENDING_EXC(r4)
  1034. rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
  1035. cmpdi cr1, r0, 0
  1036. andi. r8, r11, MSR_EE
  1037. mfspr r8, SPRN_LPCR
  1038. /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
  1039. rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
  1040. mtspr SPRN_LPCR, r8
  1041. isync
  1042. beq 5f
  1043. li r0, BOOK3S_INTERRUPT_EXTERNAL
  1044. bne cr1, 12f
  1045. mfspr r0, SPRN_DEC
  1046. BEGIN_FTR_SECTION
  1047. /* On POWER9 check whether the guest has large decrementer enabled */
  1048. andis. r8, r8, LPCR_LD@h
  1049. bne 15f
  1050. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  1051. extsw r0, r0
  1052. 15: cmpdi r0, 0
  1053. li r0, BOOK3S_INTERRUPT_DECREMENTER
  1054. bge 5f
  1055. 12: mtspr SPRN_SRR0, r10
  1056. mr r10,r0
  1057. mtspr SPRN_SRR1, r11
  1058. mr r9, r4
  1059. bl kvmppc_msr_interrupt
  1060. 5:
  1061. BEGIN_FTR_SECTION
  1062. b fast_guest_return
  1063. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
  1064. /* On POWER9, check for pending doorbell requests */
  1065. lbz r0, VCPU_DBELL_REQ(r4)
  1066. cmpwi r0, 0
  1067. beq fast_guest_return
  1068. ld r5, HSTATE_KVM_VCORE(r13)
  1069. /* Set DPDES register so the CPU will take a doorbell interrupt */
  1070. li r0, 1
  1071. mtspr SPRN_DPDES, r0
  1072. std r0, VCORE_DPDES(r5)
  1073. /* Make sure other cpus see vcore->dpdes set before dbell req clear */
  1074. lwsync
  1075. /* Clear the pending doorbell request */
  1076. li r0, 0
  1077. stb r0, VCPU_DBELL_REQ(r4)
  1078. /*
  1079. * Required state:
  1080. * R4 = vcpu
  1081. * R10: value for HSRR0
  1082. * R11: value for HSRR1
  1083. * R13 = PACA
  1084. */
  1085. fast_guest_return:
  1086. li r0,0
  1087. stb r0,VCPU_CEDED(r4) /* cancel cede */
  1088. mtspr SPRN_HSRR0,r10
  1089. mtspr SPRN_HSRR1,r11
  1090. /* Activate guest mode, so faults get handled by KVM */
  1091. li r9, KVM_GUEST_MODE_GUEST_HV
  1092. stb r9, HSTATE_IN_GUEST(r13)
  1093. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  1094. /* Accumulate timing */
  1095. addi r3, r4, VCPU_TB_GUEST
  1096. bl kvmhv_accumulate_time
  1097. #endif
  1098. /* Enter guest */
  1099. BEGIN_FTR_SECTION
  1100. ld r5, VCPU_CFAR(r4)
  1101. mtspr SPRN_CFAR, r5
  1102. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  1103. BEGIN_FTR_SECTION
  1104. ld r0, VCPU_PPR(r4)
  1105. END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  1106. ld r5, VCPU_LR(r4)
  1107. ld r6, VCPU_CR(r4)
  1108. mtlr r5
  1109. mtcr r6
  1110. ld r1, VCPU_GPR(R1)(r4)
  1111. ld r2, VCPU_GPR(R2)(r4)
  1112. ld r3, VCPU_GPR(R3)(r4)
  1113. ld r5, VCPU_GPR(R5)(r4)
  1114. ld r6, VCPU_GPR(R6)(r4)
  1115. ld r7, VCPU_GPR(R7)(r4)
  1116. ld r8, VCPU_GPR(R8)(r4)
  1117. ld r9, VCPU_GPR(R9)(r4)
  1118. ld r10, VCPU_GPR(R10)(r4)
  1119. ld r11, VCPU_GPR(R11)(r4)
  1120. ld r12, VCPU_GPR(R12)(r4)
  1121. ld r13, VCPU_GPR(R13)(r4)
  1122. BEGIN_FTR_SECTION
  1123. mtspr SPRN_PPR, r0
  1124. END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  1125. /* Move canary into DSISR to check for later */
  1126. BEGIN_FTR_SECTION
  1127. li r0, 0x7fff
  1128. mtspr SPRN_HDSISR, r0
  1129. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  1130. ld r0, VCPU_GPR(R0)(r4)
  1131. ld r4, VCPU_GPR(R4)(r4)
  1132. HRFI_TO_GUEST
  1133. b .
  1134. secondary_too_late:
  1135. li r12, 0
  1136. stw r12, STACK_SLOT_TRAP(r1)
  1137. cmpdi r4, 0
  1138. beq 11f
  1139. stw r12, VCPU_TRAP(r4)
  1140. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  1141. addi r3, r4, VCPU_TB_RMEXIT
  1142. bl kvmhv_accumulate_time
  1143. #endif
  1144. 11: b kvmhv_switch_to_host
  1145. no_switch_exit:
  1146. HMT_MEDIUM
  1147. li r12, 0
  1148. b 12f
  1149. hdec_soon:
  1150. li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
  1151. 12: stw r12, VCPU_TRAP(r4)
  1152. mr r9, r4
  1153. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  1154. addi r3, r4, VCPU_TB_RMEXIT
  1155. bl kvmhv_accumulate_time
  1156. #endif
  1157. b guest_bypass
  1158. /******************************************************************************
  1159. * *
  1160. * Exit code *
  1161. * *
  1162. *****************************************************************************/
  1163. /*
  1164. * We come here from the first-level interrupt handlers.
  1165. */
  1166. .globl kvmppc_interrupt_hv
  1167. kvmppc_interrupt_hv:
  1168. /*
  1169. * Register contents:
  1170. * R12 = (guest CR << 32) | interrupt vector
  1171. * R13 = PACA
  1172. * guest R12 saved in shadow VCPU SCRATCH0
  1173. * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE
  1174. * guest R13 saved in SPRN_SCRATCH0
  1175. */
  1176. std r9, HSTATE_SCRATCH2(r13)
  1177. lbz r9, HSTATE_IN_GUEST(r13)
  1178. cmpwi r9, KVM_GUEST_MODE_HOST_HV
  1179. beq kvmppc_bad_host_intr
  1180. #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
  1181. cmpwi r9, KVM_GUEST_MODE_GUEST
  1182. ld r9, HSTATE_SCRATCH2(r13)
  1183. beq kvmppc_interrupt_pr
  1184. #endif
  1185. /* We're now back in the host but in guest MMU context */
  1186. li r9, KVM_GUEST_MODE_HOST_HV
  1187. stb r9, HSTATE_IN_GUEST(r13)
  1188. ld r9, HSTATE_KVM_VCPU(r13)
  1189. /* Save registers */
  1190. std r0, VCPU_GPR(R0)(r9)
  1191. std r1, VCPU_GPR(R1)(r9)
  1192. std r2, VCPU_GPR(R2)(r9)
  1193. std r3, VCPU_GPR(R3)(r9)
  1194. std r4, VCPU_GPR(R4)(r9)
  1195. std r5, VCPU_GPR(R5)(r9)
  1196. std r6, VCPU_GPR(R6)(r9)
  1197. std r7, VCPU_GPR(R7)(r9)
  1198. std r8, VCPU_GPR(R8)(r9)
  1199. ld r0, HSTATE_SCRATCH2(r13)
  1200. std r0, VCPU_GPR(R9)(r9)
  1201. std r10, VCPU_GPR(R10)(r9)
  1202. std r11, VCPU_GPR(R11)(r9)
  1203. ld r3, HSTATE_SCRATCH0(r13)
  1204. std r3, VCPU_GPR(R12)(r9)
  1205. /* CR is in the high half of r12 */
  1206. srdi r4, r12, 32
  1207. std r4, VCPU_CR(r9)
  1208. BEGIN_FTR_SECTION
  1209. ld r3, HSTATE_CFAR(r13)
  1210. std r3, VCPU_CFAR(r9)
  1211. END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
  1212. BEGIN_FTR_SECTION
  1213. ld r4, HSTATE_PPR(r13)
  1214. std r4, VCPU_PPR(r9)
  1215. END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
  1216. /* Restore R1/R2 so we can handle faults */
  1217. ld r1, HSTATE_HOST_R1(r13)
  1218. ld r2, PACATOC(r13)
  1219. mfspr r10, SPRN_SRR0
  1220. mfspr r11, SPRN_SRR1
  1221. std r10, VCPU_SRR0(r9)
  1222. std r11, VCPU_SRR1(r9)
  1223. /* trap is in the low half of r12, clear CR from the high half */
  1224. clrldi r12, r12, 32
  1225. andi. r0, r12, 2 /* need to read HSRR0/1? */
  1226. beq 1f
  1227. mfspr r10, SPRN_HSRR0
  1228. mfspr r11, SPRN_HSRR1
  1229. clrrdi r12, r12, 2
  1230. 1: std r10, VCPU_PC(r9)
  1231. std r11, VCPU_MSR(r9)
  1232. GET_SCRATCH0(r3)
  1233. mflr r4
  1234. std r3, VCPU_GPR(R13)(r9)
  1235. std r4, VCPU_LR(r9)
  1236. stw r12,VCPU_TRAP(r9)
  1237. /*
  1238. * Now that we have saved away SRR0/1 and HSRR0/1,
  1239. * interrupts are recoverable in principle, so set MSR_RI.
  1240. * This becomes important for relocation-on interrupts from
  1241. * the guest, which we can get in radix mode on POWER9.
  1242. */
  1243. li r0, MSR_RI
  1244. mtmsrd r0, 1
  1245. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  1246. addi r3, r9, VCPU_TB_RMINTR
  1247. mr r4, r9
  1248. bl kvmhv_accumulate_time
  1249. ld r5, VCPU_GPR(R5)(r9)
  1250. ld r6, VCPU_GPR(R6)(r9)
  1251. ld r7, VCPU_GPR(R7)(r9)
  1252. ld r8, VCPU_GPR(R8)(r9)
  1253. #endif
  1254. /* Save HEIR (HV emulation assist reg) in emul_inst
  1255. if this is an HEI (HV emulation interrupt, e40) */
  1256. li r3,KVM_INST_FETCH_FAILED
  1257. stw r3,VCPU_LAST_INST(r9)
  1258. cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
  1259. bne 11f
  1260. mfspr r3,SPRN_HEIR
  1261. 11: stw r3,VCPU_HEIR(r9)
  1262. /* these are volatile across C function calls */
  1263. #ifdef CONFIG_RELOCATABLE
  1264. ld r3, HSTATE_SCRATCH1(r13)
  1265. mtctr r3
  1266. #else
  1267. mfctr r3
  1268. #endif
  1269. mfxer r4
  1270. std r3, VCPU_CTR(r9)
  1271. std r4, VCPU_XER(r9)
  1272. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1273. /* For softpatch interrupt, go off and do TM instruction emulation */
  1274. cmpwi r12, BOOK3S_INTERRUPT_HV_SOFTPATCH
  1275. beq kvmppc_tm_emul
  1276. #endif
  1277. /* If this is a page table miss then see if it's theirs or ours */
  1278. cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
  1279. beq kvmppc_hdsi
  1280. cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
  1281. beq kvmppc_hisi
  1282. /* See if this is a leftover HDEC interrupt */
  1283. cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
  1284. bne 2f
  1285. mfspr r3,SPRN_HDEC
  1286. EXTEND_HDEC(r3)
  1287. cmpdi r3,0
  1288. mr r4,r9
  1289. bge fast_guest_return
  1290. 2:
  1291. /* See if this is an hcall we can handle in real mode */
  1292. cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
  1293. beq hcall_try_real_mode
  1294. /* Hypervisor doorbell - exit only if host IPI flag set */
  1295. cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
  1296. bne 3f
  1297. BEGIN_FTR_SECTION
  1298. PPC_MSGSYNC
  1299. lwsync
  1300. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  1301. lbz r0, HSTATE_HOST_IPI(r13)
  1302. cmpwi r0, 0
  1303. beq 4f
  1304. b guest_exit_cont
  1305. 3:
  1306. /* If it's a hypervisor facility unavailable interrupt, save HFSCR */
  1307. cmpwi r12, BOOK3S_INTERRUPT_H_FAC_UNAVAIL
  1308. bne 14f
  1309. mfspr r3, SPRN_HFSCR
  1310. std r3, VCPU_HFSCR(r9)
  1311. b guest_exit_cont
  1312. 14:
  1313. /* External interrupt ? */
  1314. cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
  1315. bne+ guest_exit_cont
  1316. /* External interrupt, first check for host_ipi. If this is
  1317. * set, we know the host wants us out so let's do it now
  1318. */
  1319. bl kvmppc_read_intr
  1320. /*
  1321. * Restore the active volatile registers after returning from
  1322. * a C function.
  1323. */
  1324. ld r9, HSTATE_KVM_VCPU(r13)
  1325. li r12, BOOK3S_INTERRUPT_EXTERNAL
  1326. /*
  1327. * kvmppc_read_intr return codes:
  1328. *
  1329. * Exit to host (r3 > 0)
  1330. * 1 An interrupt is pending that needs to be handled by the host
  1331. * Exit guest and return to host by branching to guest_exit_cont
  1332. *
  1333. * 2 Passthrough that needs completion in the host
  1334. * Exit guest and return to host by branching to guest_exit_cont
  1335. * However, we also set r12 to BOOK3S_INTERRUPT_HV_RM_HARD
  1336. * to indicate to the host to complete handling the interrupt
  1337. *
  1338. * Before returning to guest, we check if any CPU is heading out
  1339. * to the host and if so, we head out also. If no CPUs are heading
  1340. * check return values <= 0.
  1341. *
  1342. * Return to guest (r3 <= 0)
  1343. * 0 No external interrupt is pending
  1344. * -1 A guest wakeup IPI (which has now been cleared)
  1345. * In either case, we return to guest to deliver any pending
  1346. * guest interrupts.
  1347. *
  1348. * -2 A PCI passthrough external interrupt was handled
  1349. * (interrupt was delivered directly to guest)
  1350. * Return to guest to deliver any pending guest interrupts.
  1351. */
  1352. cmpdi r3, 1
  1353. ble 1f
  1354. /* Return code = 2 */
  1355. li r12, BOOK3S_INTERRUPT_HV_RM_HARD
  1356. stw r12, VCPU_TRAP(r9)
  1357. b guest_exit_cont
  1358. 1: /* Return code <= 1 */
  1359. cmpdi r3, 0
  1360. bgt guest_exit_cont
  1361. /* Return code <= 0 */
  1362. 4: ld r5, HSTATE_KVM_VCORE(r13)
  1363. lwz r0, VCORE_ENTRY_EXIT(r5)
  1364. cmpwi r0, 0x100
  1365. mr r4, r9
  1366. blt deliver_guest_interrupt
  1367. guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
  1368. /* Save more register state */
  1369. mfdar r6
  1370. mfdsisr r7
  1371. std r6, VCPU_DAR(r9)
  1372. stw r7, VCPU_DSISR(r9)
  1373. /* don't overwrite fault_dar/fault_dsisr if HDSI */
  1374. cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
  1375. beq mc_cont
  1376. std r6, VCPU_FAULT_DAR(r9)
  1377. stw r7, VCPU_FAULT_DSISR(r9)
  1378. /* See if it is a machine check */
  1379. cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
  1380. beq machine_check_realmode
  1381. mc_cont:
  1382. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  1383. addi r3, r9, VCPU_TB_RMEXIT
  1384. mr r4, r9
  1385. bl kvmhv_accumulate_time
  1386. #endif
  1387. #ifdef CONFIG_KVM_XICS
  1388. /* We are exiting, pull the VP from the XIVE */
  1389. lbz r0, VCPU_XIVE_PUSHED(r9)
  1390. cmpwi cr0, r0, 0
  1391. beq 1f
  1392. li r7, TM_SPC_PULL_OS_CTX
  1393. li r6, TM_QW1_OS
  1394. mfmsr r0
  1395. andi. r0, r0, MSR_DR /* in real mode? */
  1396. beq 2f
  1397. ld r10, HSTATE_XIVE_TIMA_VIRT(r13)
  1398. cmpldi cr0, r10, 0
  1399. beq 1f
  1400. /* First load to pull the context, we ignore the value */
  1401. eieio
  1402. lwzx r11, r7, r10
  1403. /* Second load to recover the context state (Words 0 and 1) */
  1404. ldx r11, r6, r10
  1405. b 3f
  1406. 2: ld r10, HSTATE_XIVE_TIMA_PHYS(r13)
  1407. cmpldi cr0, r10, 0
  1408. beq 1f
  1409. /* First load to pull the context, we ignore the value */
  1410. eieio
  1411. lwzcix r11, r7, r10
  1412. /* Second load to recover the context state (Words 0 and 1) */
  1413. ldcix r11, r6, r10
  1414. 3: std r11, VCPU_XIVE_SAVED_STATE(r9)
  1415. /* Fixup some of the state for the next load */
  1416. li r10, 0
  1417. li r0, 0xff
  1418. stb r10, VCPU_XIVE_PUSHED(r9)
  1419. stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9)
  1420. stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9)
  1421. eieio
  1422. 1:
  1423. #endif /* CONFIG_KVM_XICS */
  1424. /* Possibly flush the link stack here. */
  1425. 1: nop
  1426. patch_site 1b patch__call_kvm_flush_link_stack
  1427. /* For hash guest, read the guest SLB and save it away */
  1428. ld r5, VCPU_KVM(r9)
  1429. lbz r0, KVM_RADIX(r5)
  1430. li r5, 0
  1431. cmpwi r0, 0
  1432. bne 3f /* for radix, save 0 entries */
  1433. lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
  1434. mtctr r0
  1435. li r6,0
  1436. addi r7,r9,VCPU_SLB
  1437. 1: slbmfee r8,r6
  1438. andis. r0,r8,SLB_ESID_V@h
  1439. beq 2f
  1440. add r8,r8,r6 /* put index in */
  1441. slbmfev r3,r6
  1442. std r8,VCPU_SLB_E(r7)
  1443. std r3,VCPU_SLB_V(r7)
  1444. addi r7,r7,VCPU_SLB_SIZE
  1445. addi r5,r5,1
  1446. 2: addi r6,r6,1
  1447. bdnz 1b
  1448. /* Finally clear out the SLB */
  1449. li r0,0
  1450. slbmte r0,r0
  1451. slbia
  1452. ptesync
  1453. 3: stw r5,VCPU_SLB_MAX(r9)
  1454. /* load host SLB entries */
  1455. BEGIN_MMU_FTR_SECTION
  1456. b 0f
  1457. END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
  1458. ld r8,PACA_SLBSHADOWPTR(r13)
  1459. .rept SLB_NUM_BOLTED
  1460. li r3, SLBSHADOW_SAVEAREA
  1461. LDX_BE r5, r8, r3
  1462. addi r3, r3, 8
  1463. LDX_BE r6, r8, r3
  1464. andis. r7,r5,SLB_ESID_V@h
  1465. beq 1f
  1466. slbmte r6,r5
  1467. 1: addi r8,r8,16
  1468. .endr
  1469. 0:
  1470. guest_bypass:
  1471. stw r12, STACK_SLOT_TRAP(r1)
  1472. /* Save DEC */
  1473. /* Do this before kvmhv_commence_exit so we know TB is guest TB */
  1474. ld r3, HSTATE_KVM_VCORE(r13)
  1475. mfspr r5,SPRN_DEC
  1476. mftb r6
  1477. /* On P9, if the guest has large decr enabled, don't sign extend */
  1478. BEGIN_FTR_SECTION
  1479. ld r4, VCORE_LPCR(r3)
  1480. andis. r4, r4, LPCR_LD@h
  1481. bne 16f
  1482. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  1483. extsw r5,r5
  1484. 16: add r5,r5,r6
  1485. /* r5 is a guest timebase value here, convert to host TB */
  1486. ld r4,VCORE_TB_OFFSET_APPL(r3)
  1487. subf r5,r4,r5
  1488. std r5,VCPU_DEC_EXPIRES(r9)
  1489. /* Increment exit count, poke other threads to exit */
  1490. mr r3, r12
  1491. bl kvmhv_commence_exit
  1492. nop
  1493. ld r9, HSTATE_KVM_VCPU(r13)
  1494. /* Stop others sending VCPU interrupts to this physical CPU */
  1495. li r0, -1
  1496. stw r0, VCPU_CPU(r9)
  1497. stw r0, VCPU_THREAD_CPU(r9)
  1498. /* Save guest CTRL register, set runlatch to 1 */
  1499. mfspr r6,SPRN_CTRLF
  1500. stw r6,VCPU_CTRL(r9)
  1501. andi. r0,r6,1
  1502. bne 4f
  1503. ori r6,r6,1
  1504. mtspr SPRN_CTRLT,r6
  1505. 4:
  1506. /*
  1507. * Save the guest PURR/SPURR
  1508. */
  1509. mfspr r5,SPRN_PURR
  1510. mfspr r6,SPRN_SPURR
  1511. ld r7,VCPU_PURR(r9)
  1512. ld r8,VCPU_SPURR(r9)
  1513. std r5,VCPU_PURR(r9)
  1514. std r6,VCPU_SPURR(r9)
  1515. subf r5,r7,r5
  1516. subf r6,r8,r6
  1517. /*
  1518. * Restore host PURR/SPURR and add guest times
  1519. * so that the time in the guest gets accounted.
  1520. */
  1521. ld r3,HSTATE_PURR(r13)
  1522. ld r4,HSTATE_SPURR(r13)
  1523. add r3,r3,r5
  1524. add r4,r4,r6
  1525. mtspr SPRN_PURR,r3
  1526. mtspr SPRN_SPURR,r4
  1527. BEGIN_FTR_SECTION
  1528. b 8f
  1529. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
  1530. /* Save POWER8-specific registers */
  1531. mfspr r5, SPRN_IAMR
  1532. mfspr r6, SPRN_PSPB
  1533. mfspr r7, SPRN_FSCR
  1534. std r5, VCPU_IAMR(r9)
  1535. stw r6, VCPU_PSPB(r9)
  1536. std r7, VCPU_FSCR(r9)
  1537. mfspr r5, SPRN_IC
  1538. mfspr r7, SPRN_TAR
  1539. std r5, VCPU_IC(r9)
  1540. std r7, VCPU_TAR(r9)
  1541. mfspr r8, SPRN_EBBHR
  1542. std r8, VCPU_EBBHR(r9)
  1543. mfspr r5, SPRN_EBBRR
  1544. mfspr r6, SPRN_BESCR
  1545. mfspr r7, SPRN_PID
  1546. mfspr r8, SPRN_WORT
  1547. std r5, VCPU_EBBRR(r9)
  1548. std r6, VCPU_BESCR(r9)
  1549. stw r7, VCPU_GUEST_PID(r9)
  1550. std r8, VCPU_WORT(r9)
  1551. BEGIN_FTR_SECTION
  1552. mfspr r5, SPRN_TCSCR
  1553. mfspr r6, SPRN_ACOP
  1554. mfspr r7, SPRN_CSIGR
  1555. mfspr r8, SPRN_TACR
  1556. std r5, VCPU_TCSCR(r9)
  1557. std r6, VCPU_ACOP(r9)
  1558. std r7, VCPU_CSIGR(r9)
  1559. std r8, VCPU_TACR(r9)
  1560. FTR_SECTION_ELSE
  1561. mfspr r5, SPRN_TIDR
  1562. mfspr r6, SPRN_PSSCR
  1563. std r5, VCPU_TID(r9)
  1564. rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */
  1565. rotldi r6, r6, 60
  1566. std r6, VCPU_PSSCR(r9)
  1567. /* Restore host HFSCR value */
  1568. ld r7, STACK_SLOT_HFSCR(r1)
  1569. mtspr SPRN_HFSCR, r7
  1570. ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
  1571. /*
  1572. * Restore various registers to 0, where non-zero values
  1573. * set by the guest could disrupt the host.
  1574. */
  1575. li r0, 0
  1576. mtspr SPRN_PSPB, r0
  1577. mtspr SPRN_WORT, r0
  1578. BEGIN_FTR_SECTION
  1579. mtspr SPRN_TCSCR, r0
  1580. /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
  1581. li r0, 1
  1582. sldi r0, r0, 31
  1583. mtspr SPRN_MMCRS, r0
  1584. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
  1585. /* Save and restore AMR, IAMR and UAMOR before turning on the MMU */
  1586. ld r8, STACK_SLOT_IAMR(r1)
  1587. mtspr SPRN_IAMR, r8
  1588. 8: /* Power7 jumps back in here */
  1589. mfspr r5,SPRN_AMR
  1590. mfspr r6,SPRN_UAMOR
  1591. std r5,VCPU_AMR(r9)
  1592. std r6,VCPU_UAMOR(r9)
  1593. ld r5,STACK_SLOT_AMR(r1)
  1594. ld r6,STACK_SLOT_UAMOR(r1)
  1595. mtspr SPRN_AMR, r5
  1596. mtspr SPRN_UAMOR, r6
  1597. /* Switch DSCR back to host value */
  1598. mfspr r8, SPRN_DSCR
  1599. ld r7, HSTATE_DSCR(r13)
  1600. std r8, VCPU_DSCR(r9)
  1601. mtspr SPRN_DSCR, r7
  1602. /* Save non-volatile GPRs */
  1603. std r14, VCPU_GPR(R14)(r9)
  1604. std r15, VCPU_GPR(R15)(r9)
  1605. std r16, VCPU_GPR(R16)(r9)
  1606. std r17, VCPU_GPR(R17)(r9)
  1607. std r18, VCPU_GPR(R18)(r9)
  1608. std r19, VCPU_GPR(R19)(r9)
  1609. std r20, VCPU_GPR(R20)(r9)
  1610. std r21, VCPU_GPR(R21)(r9)
  1611. std r22, VCPU_GPR(R22)(r9)
  1612. std r23, VCPU_GPR(R23)(r9)
  1613. std r24, VCPU_GPR(R24)(r9)
  1614. std r25, VCPU_GPR(R25)(r9)
  1615. std r26, VCPU_GPR(R26)(r9)
  1616. std r27, VCPU_GPR(R27)(r9)
  1617. std r28, VCPU_GPR(R28)(r9)
  1618. std r29, VCPU_GPR(R29)(r9)
  1619. std r30, VCPU_GPR(R30)(r9)
  1620. std r31, VCPU_GPR(R31)(r9)
  1621. /* Save SPRGs */
  1622. mfspr r3, SPRN_SPRG0
  1623. mfspr r4, SPRN_SPRG1
  1624. mfspr r5, SPRN_SPRG2
  1625. mfspr r6, SPRN_SPRG3
  1626. std r3, VCPU_SPRG0(r9)
  1627. std r4, VCPU_SPRG1(r9)
  1628. std r5, VCPU_SPRG2(r9)
  1629. std r6, VCPU_SPRG3(r9)
  1630. /* save FP state */
  1631. mr r3, r9
  1632. bl kvmppc_save_fp
  1633. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1634. /*
  1635. * Branch around the call if both CPU_FTR_TM and
  1636. * CPU_FTR_P9_TM_HV_ASSIST are off.
  1637. */
  1638. BEGIN_FTR_SECTION
  1639. b 91f
  1640. END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0)
  1641. /*
  1642. * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR
  1643. */
  1644. mr r3, r9
  1645. ld r4, VCPU_MSR(r3)
  1646. bl kvmppc_save_tm_hv
  1647. ld r9, HSTATE_KVM_VCPU(r13)
  1648. 91:
  1649. #endif
  1650. /* Increment yield count if they have a VPA */
  1651. ld r8, VCPU_VPA(r9) /* do they have a VPA? */
  1652. cmpdi r8, 0
  1653. beq 25f
  1654. li r4, LPPACA_YIELDCOUNT
  1655. LWZX_BE r3, r8, r4
  1656. addi r3, r3, 1
  1657. STWX_BE r3, r8, r4
  1658. li r3, 1
  1659. stb r3, VCPU_VPA_DIRTY(r9)
  1660. 25:
  1661. /* Save PMU registers if requested */
  1662. /* r8 and cr0.eq are live here */
  1663. BEGIN_FTR_SECTION
  1664. /*
  1665. * POWER8 seems to have a hardware bug where setting
  1666. * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
  1667. * when some counters are already negative doesn't seem
  1668. * to cause a performance monitor alert (and hence interrupt).
  1669. * The effect of this is that when saving the PMU state,
  1670. * if there is no PMU alert pending when we read MMCR0
  1671. * before freezing the counters, but one becomes pending
  1672. * before we read the counters, we lose it.
  1673. * To work around this, we need a way to freeze the counters
  1674. * before reading MMCR0. Normally, freezing the counters
  1675. * is done by writing MMCR0 (to set MMCR0[FC]) which
  1676. * unavoidably writes MMCR0[PMA0] as well. On POWER8,
  1677. * we can also freeze the counters using MMCR2, by writing
  1678. * 1s to all the counter freeze condition bits (there are
  1679. * 9 bits each for 6 counters).
  1680. */
  1681. li r3, -1 /* set all freeze bits */
  1682. clrrdi r3, r3, 10
  1683. mfspr r10, SPRN_MMCR2
  1684. mtspr SPRN_MMCR2, r3
  1685. isync
  1686. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  1687. li r3, 1
  1688. sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
  1689. mfspr r4, SPRN_MMCR0 /* save MMCR0 */
  1690. mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
  1691. mfspr r6, SPRN_MMCRA
  1692. /* Clear MMCRA in order to disable SDAR updates */
  1693. li r7, 0
  1694. mtspr SPRN_MMCRA, r7
  1695. isync
  1696. beq 21f /* if no VPA, save PMU stuff anyway */
  1697. lbz r7, LPPACA_PMCINUSE(r8)
  1698. cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */
  1699. bne 21f
  1700. std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
  1701. b 22f
  1702. 21: mfspr r5, SPRN_MMCR1
  1703. mfspr r7, SPRN_SIAR
  1704. mfspr r8, SPRN_SDAR
  1705. std r4, VCPU_MMCR(r9)
  1706. std r5, VCPU_MMCR + 8(r9)
  1707. std r6, VCPU_MMCR + 16(r9)
  1708. BEGIN_FTR_SECTION
  1709. std r10, VCPU_MMCR + 24(r9)
  1710. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  1711. std r7, VCPU_SIAR(r9)
  1712. std r8, VCPU_SDAR(r9)
  1713. mfspr r3, SPRN_PMC1
  1714. mfspr r4, SPRN_PMC2
  1715. mfspr r5, SPRN_PMC3
  1716. mfspr r6, SPRN_PMC4
  1717. mfspr r7, SPRN_PMC5
  1718. mfspr r8, SPRN_PMC6
  1719. stw r3, VCPU_PMC(r9)
  1720. stw r4, VCPU_PMC + 4(r9)
  1721. stw r5, VCPU_PMC + 8(r9)
  1722. stw r6, VCPU_PMC + 12(r9)
  1723. stw r7, VCPU_PMC + 16(r9)
  1724. stw r8, VCPU_PMC + 20(r9)
  1725. BEGIN_FTR_SECTION
  1726. mfspr r5, SPRN_SIER
  1727. std r5, VCPU_SIER(r9)
  1728. BEGIN_FTR_SECTION_NESTED(96)
  1729. mfspr r6, SPRN_SPMC1
  1730. mfspr r7, SPRN_SPMC2
  1731. mfspr r8, SPRN_MMCRS
  1732. stw r6, VCPU_PMC + 24(r9)
  1733. stw r7, VCPU_PMC + 28(r9)
  1734. std r8, VCPU_MMCR + 32(r9)
  1735. lis r4, 0x8000
  1736. mtspr SPRN_MMCRS, r4
  1737. END_FTR_SECTION_NESTED(CPU_FTR_ARCH_300, 0, 96)
  1738. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  1739. 22:
  1740. /* Restore host values of some registers */
  1741. BEGIN_FTR_SECTION
  1742. ld r5, STACK_SLOT_CIABR(r1)
  1743. ld r6, STACK_SLOT_DAWR(r1)
  1744. ld r7, STACK_SLOT_DAWRX(r1)
  1745. mtspr SPRN_CIABR, r5
  1746. /*
  1747. * If the DAWR doesn't work, it's ok to write these here as
  1748. * this value should always be zero
  1749. */
  1750. mtspr SPRN_DAWR, r6
  1751. mtspr SPRN_DAWRX, r7
  1752. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  1753. BEGIN_FTR_SECTION
  1754. ld r5, STACK_SLOT_TID(r1)
  1755. ld r6, STACK_SLOT_PSSCR(r1)
  1756. ld r7, STACK_SLOT_PID(r1)
  1757. mtspr SPRN_TIDR, r5
  1758. mtspr SPRN_PSSCR, r6
  1759. mtspr SPRN_PID, r7
  1760. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  1761. #ifdef CONFIG_PPC_RADIX_MMU
  1762. /*
  1763. * Are we running hash or radix ?
  1764. */
  1765. ld r5, VCPU_KVM(r9)
  1766. lbz r0, KVM_RADIX(r5)
  1767. cmpwi cr2, r0, 0
  1768. beq cr2, 2f
  1769. /*
  1770. * Radix: do eieio; tlbsync; ptesync sequence in case we
  1771. * interrupted the guest between a tlbie and a ptesync.
  1772. */
  1773. eieio
  1774. tlbsync
  1775. ptesync
  1776. /* Radix: Handle the case where the guest used an illegal PID */
  1777. LOAD_REG_ADDR(r4, mmu_base_pid)
  1778. lwz r3, VCPU_GUEST_PID(r9)
  1779. lwz r5, 0(r4)
  1780. cmpw cr0,r3,r5
  1781. blt 2f
  1782. /*
  1783. * Illegal PID, the HW might have prefetched and cached in the TLB
  1784. * some translations for the LPID 0 / guest PID combination which
  1785. * Linux doesn't know about, so we need to flush that PID out of
  1786. * the TLB. First we need to set LPIDR to 0 so tlbiel applies to
  1787. * the right context.
  1788. */
  1789. li r0,0
  1790. mtspr SPRN_LPID,r0
  1791. isync
  1792. /* Then do a congruence class local flush */
  1793. ld r6,VCPU_KVM(r9)
  1794. lwz r0,KVM_TLB_SETS(r6)
  1795. mtctr r0
  1796. li r7,0x400 /* IS field = 0b01 */
  1797. ptesync
  1798. sldi r0,r3,32 /* RS has PID */
  1799. 1: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */
  1800. addi r7,r7,0x1000
  1801. bdnz 1b
  1802. ptesync
  1803. 2:
  1804. #endif /* CONFIG_PPC_RADIX_MMU */
  1805. /*
  1806. * POWER7/POWER8 guest -> host partition switch code.
  1807. * We don't have to lock against tlbies but we do
  1808. * have to coordinate the hardware threads.
  1809. * Here STACK_SLOT_TRAP(r1) contains the trap number.
  1810. */
  1811. kvmhv_switch_to_host:
  1812. /* Secondary threads wait for primary to do partition switch */
  1813. ld r5,HSTATE_KVM_VCORE(r13)
  1814. ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
  1815. lbz r3,HSTATE_PTID(r13)
  1816. cmpwi r3,0
  1817. beq 15f
  1818. HMT_LOW
  1819. 13: lbz r3,VCORE_IN_GUEST(r5)
  1820. cmpwi r3,0
  1821. bne 13b
  1822. HMT_MEDIUM
  1823. b 16f
  1824. /* Primary thread waits for all the secondaries to exit guest */
  1825. 15: lwz r3,VCORE_ENTRY_EXIT(r5)
  1826. rlwinm r0,r3,32-8,0xff
  1827. clrldi r3,r3,56
  1828. cmpw r3,r0
  1829. bne 15b
  1830. isync
  1831. /* Did we actually switch to the guest at all? */
  1832. lbz r6, VCORE_IN_GUEST(r5)
  1833. cmpwi r6, 0
  1834. beq 19f
  1835. /* Primary thread switches back to host partition */
  1836. lwz r7,KVM_HOST_LPID(r4)
  1837. BEGIN_FTR_SECTION
  1838. ld r6,KVM_HOST_SDR1(r4)
  1839. li r8,LPID_RSVD /* switch to reserved LPID */
  1840. mtspr SPRN_LPID,r8
  1841. ptesync
  1842. mtspr SPRN_SDR1,r6 /* switch to host page table */
  1843. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
  1844. mtspr SPRN_LPID,r7
  1845. isync
  1846. BEGIN_FTR_SECTION
  1847. /* DPDES and VTB are shared between threads */
  1848. mfspr r7, SPRN_DPDES
  1849. mfspr r8, SPRN_VTB
  1850. std r7, VCORE_DPDES(r5)
  1851. std r8, VCORE_VTB(r5)
  1852. /* clear DPDES so we don't get guest doorbells in the host */
  1853. li r8, 0
  1854. mtspr SPRN_DPDES, r8
  1855. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  1856. /* If HMI, call kvmppc_realmode_hmi_handler() */
  1857. lwz r12, STACK_SLOT_TRAP(r1)
  1858. cmpwi r12, BOOK3S_INTERRUPT_HMI
  1859. bne 27f
  1860. bl kvmppc_realmode_hmi_handler
  1861. nop
  1862. cmpdi r3, 0
  1863. /*
  1864. * At this point kvmppc_realmode_hmi_handler may have resync-ed
  1865. * the TB, and if it has, we must not subtract the guest timebase
  1866. * offset from the timebase. So, skip it.
  1867. *
  1868. * Also, do not call kvmppc_subcore_exit_guest() because it has
  1869. * been invoked as part of kvmppc_realmode_hmi_handler().
  1870. */
  1871. beq 30f
  1872. 27:
  1873. /* Subtract timebase offset from timebase */
  1874. ld r8, VCORE_TB_OFFSET_APPL(r5)
  1875. cmpdi r8,0
  1876. beq 17f
  1877. li r0, 0
  1878. std r0, VCORE_TB_OFFSET_APPL(r5)
  1879. mftb r6 /* current guest timebase */
  1880. subf r8,r8,r6
  1881. mtspr SPRN_TBU40,r8 /* update upper 40 bits */
  1882. mftb r7 /* check if lower 24 bits overflowed */
  1883. clrldi r6,r6,40
  1884. clrldi r7,r7,40
  1885. cmpld r7,r6
  1886. bge 17f
  1887. addis r8,r8,0x100 /* if so, increment upper 40 bits */
  1888. mtspr SPRN_TBU40,r8
  1889. 17: bl kvmppc_subcore_exit_guest
  1890. nop
  1891. 30: ld r5,HSTATE_KVM_VCORE(r13)
  1892. ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
  1893. /* Reset PCR */
  1894. ld r0, VCORE_PCR(r5)
  1895. cmpdi r0, 0
  1896. beq 18f
  1897. li r0, 0
  1898. mtspr SPRN_PCR, r0
  1899. 18:
  1900. /* Signal secondary CPUs to continue */
  1901. stb r0,VCORE_IN_GUEST(r5)
  1902. 19: lis r8,0x7fff /* MAX_INT@h */
  1903. mtspr SPRN_HDEC,r8
  1904. 16:
  1905. BEGIN_FTR_SECTION
  1906. /* On POWER9 with HPT-on-radix we need to wait for all other threads */
  1907. ld r3, HSTATE_SPLIT_MODE(r13)
  1908. cmpdi r3, 0
  1909. beq 47f
  1910. lwz r8, KVM_SPLIT_DO_RESTORE(r3)
  1911. cmpwi r8, 0
  1912. beq 47f
  1913. bl kvmhv_p9_restore_lpcr
  1914. nop
  1915. b 48f
  1916. 47:
  1917. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  1918. ld r8,KVM_HOST_LPCR(r4)
  1919. mtspr SPRN_LPCR,r8
  1920. isync
  1921. 48:
  1922. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  1923. /* Finish timing, if we have a vcpu */
  1924. ld r4, HSTATE_KVM_VCPU(r13)
  1925. cmpdi r4, 0
  1926. li r3, 0
  1927. beq 2f
  1928. bl kvmhv_accumulate_time
  1929. 2:
  1930. #endif
  1931. /* Unset guest mode */
  1932. li r0, KVM_GUEST_MODE_NONE
  1933. stb r0, HSTATE_IN_GUEST(r13)
  1934. lwz r12, STACK_SLOT_TRAP(r1) /* return trap # in r12 */
  1935. ld r0, SFS+PPC_LR_STKOFF(r1)
  1936. addi r1, r1, SFS
  1937. mtlr r0
  1938. blr
  1939. .balign 32
  1940. .global kvm_flush_link_stack
  1941. kvm_flush_link_stack:
  1942. /* Save LR into r0 */
  1943. mflr r0
  1944. /* Flush the link stack. On Power8 it's up to 32 entries in size. */
  1945. .rept 32
  1946. bl .+4
  1947. .endr
  1948. /* And on Power9 it's up to 64. */
  1949. BEGIN_FTR_SECTION
  1950. .rept 32
  1951. bl .+4
  1952. .endr
  1953. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  1954. /* Restore LR */
  1955. mtlr r0
  1956. blr
  1957. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1958. /*
  1959. * Softpatch interrupt for transactional memory emulation cases
  1960. * on POWER9 DD2.2. This is early in the guest exit path - we
  1961. * haven't saved registers or done a treclaim yet.
  1962. */
  1963. kvmppc_tm_emul:
  1964. /* Save instruction image in HEIR */
  1965. mfspr r3, SPRN_HEIR
  1966. stw r3, VCPU_HEIR(r9)
  1967. /*
  1968. * The cases we want to handle here are those where the guest
  1969. * is in real suspend mode and is trying to transition to
  1970. * transactional mode.
  1971. */
  1972. lbz r0, HSTATE_FAKE_SUSPEND(r13)
  1973. cmpwi r0, 0 /* keep exiting guest if in fake suspend */
  1974. bne guest_exit_cont
  1975. rldicl r3, r11, 64 - MSR_TS_S_LG, 62
  1976. cmpwi r3, 1 /* or if not in suspend state */
  1977. bne guest_exit_cont
  1978. /* Call C code to do the emulation */
  1979. mr r3, r9
  1980. bl kvmhv_p9_tm_emulation_early
  1981. nop
  1982. ld r9, HSTATE_KVM_VCPU(r13)
  1983. li r12, BOOK3S_INTERRUPT_HV_SOFTPATCH
  1984. cmpwi r3, 0
  1985. beq guest_exit_cont /* continue exiting if not handled */
  1986. ld r10, VCPU_PC(r9)
  1987. ld r11, VCPU_MSR(r9)
  1988. b fast_interrupt_c_return /* go back to guest if handled */
  1989. #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
  1990. /*
  1991. * Check whether an HDSI is an HPTE not found fault or something else.
  1992. * If it is an HPTE not found fault that is due to the guest accessing
  1993. * a page that they have mapped but which we have paged out, then
  1994. * we continue on with the guest exit path. In all other cases,
  1995. * reflect the HDSI to the guest as a DSI.
  1996. */
  1997. kvmppc_hdsi:
  1998. ld r3, VCPU_KVM(r9)
  1999. lbz r0, KVM_RADIX(r3)
  2000. mfspr r4, SPRN_HDAR
  2001. mfspr r6, SPRN_HDSISR
  2002. BEGIN_FTR_SECTION
  2003. /* Look for DSISR canary. If we find it, retry instruction */
  2004. cmpdi r6, 0x7fff
  2005. beq 6f
  2006. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  2007. cmpwi r0, 0
  2008. bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */
  2009. /* HPTE not found fault or protection fault? */
  2010. andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
  2011. beq 1f /* if not, send it to the guest */
  2012. andi. r0, r11, MSR_DR /* data relocation enabled? */
  2013. beq 3f
  2014. BEGIN_FTR_SECTION
  2015. mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */
  2016. b 4f
  2017. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  2018. clrrdi r0, r4, 28
  2019. PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
  2020. li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
  2021. bne 7f /* if no SLB entry found */
  2022. 4: std r4, VCPU_FAULT_DAR(r9)
  2023. stw r6, VCPU_FAULT_DSISR(r9)
  2024. /* Search the hash table. */
  2025. mr r3, r9 /* vcpu pointer */
  2026. li r7, 1 /* data fault */
  2027. bl kvmppc_hpte_hv_fault
  2028. ld r9, HSTATE_KVM_VCPU(r13)
  2029. ld r10, VCPU_PC(r9)
  2030. ld r11, VCPU_MSR(r9)
  2031. li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
  2032. cmpdi r3, 0 /* retry the instruction */
  2033. beq 6f
  2034. cmpdi r3, -1 /* handle in kernel mode */
  2035. beq guest_exit_cont
  2036. cmpdi r3, -2 /* MMIO emulation; need instr word */
  2037. beq 2f
  2038. /* Synthesize a DSI (or DSegI) for the guest */
  2039. ld r4, VCPU_FAULT_DAR(r9)
  2040. mr r6, r3
  2041. 1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE
  2042. mtspr SPRN_DSISR, r6
  2043. 7: mtspr SPRN_DAR, r4
  2044. mtspr SPRN_SRR0, r10
  2045. mtspr SPRN_SRR1, r11
  2046. mr r10, r0
  2047. bl kvmppc_msr_interrupt
  2048. fast_interrupt_c_return:
  2049. 6: ld r7, VCPU_CTR(r9)
  2050. ld r8, VCPU_XER(r9)
  2051. mtctr r7
  2052. mtxer r8
  2053. mr r4, r9
  2054. b fast_guest_return
  2055. 3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
  2056. ld r5, KVM_VRMA_SLB_V(r5)
  2057. b 4b
  2058. /* If this is for emulated MMIO, load the instruction word */
  2059. 2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
  2060. /* Set guest mode to 'jump over instruction' so if lwz faults
  2061. * we'll just continue at the next IP. */
  2062. li r0, KVM_GUEST_MODE_SKIP
  2063. stb r0, HSTATE_IN_GUEST(r13)
  2064. /* Do the access with MSR:DR enabled */
  2065. mfmsr r3
  2066. ori r4, r3, MSR_DR /* Enable paging for data */
  2067. mtmsrd r4
  2068. lwz r8, 0(r10)
  2069. mtmsrd r3
  2070. /* Store the result */
  2071. stw r8, VCPU_LAST_INST(r9)
  2072. /* Unset guest mode. */
  2073. li r0, KVM_GUEST_MODE_HOST_HV
  2074. stb r0, HSTATE_IN_GUEST(r13)
  2075. b guest_exit_cont
  2076. .Lradix_hdsi:
  2077. std r4, VCPU_FAULT_DAR(r9)
  2078. stw r6, VCPU_FAULT_DSISR(r9)
  2079. .Lradix_hisi:
  2080. mfspr r5, SPRN_ASDR
  2081. std r5, VCPU_FAULT_GPA(r9)
  2082. b guest_exit_cont
  2083. /*
  2084. * Similarly for an HISI, reflect it to the guest as an ISI unless
  2085. * it is an HPTE not found fault for a page that we have paged out.
  2086. */
  2087. kvmppc_hisi:
  2088. ld r3, VCPU_KVM(r9)
  2089. lbz r0, KVM_RADIX(r3)
  2090. cmpwi r0, 0
  2091. bne .Lradix_hisi /* for radix, just save ASDR */
  2092. andis. r0, r11, SRR1_ISI_NOPT@h
  2093. beq 1f
  2094. andi. r0, r11, MSR_IR /* instruction relocation enabled? */
  2095. beq 3f
  2096. BEGIN_FTR_SECTION
  2097. mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */
  2098. b 4f
  2099. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  2100. clrrdi r0, r10, 28
  2101. PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
  2102. li r0, BOOK3S_INTERRUPT_INST_SEGMENT
  2103. bne 7f /* if no SLB entry found */
  2104. 4:
  2105. /* Search the hash table. */
  2106. mr r3, r9 /* vcpu pointer */
  2107. mr r4, r10
  2108. mr r6, r11
  2109. li r7, 0 /* instruction fault */
  2110. bl kvmppc_hpte_hv_fault
  2111. ld r9, HSTATE_KVM_VCPU(r13)
  2112. ld r10, VCPU_PC(r9)
  2113. ld r11, VCPU_MSR(r9)
  2114. li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
  2115. cmpdi r3, 0 /* retry the instruction */
  2116. beq fast_interrupt_c_return
  2117. cmpdi r3, -1 /* handle in kernel mode */
  2118. beq guest_exit_cont
  2119. /* Synthesize an ISI (or ISegI) for the guest */
  2120. mr r11, r3
  2121. 1: li r0, BOOK3S_INTERRUPT_INST_STORAGE
  2122. 7: mtspr SPRN_SRR0, r10
  2123. mtspr SPRN_SRR1, r11
  2124. mr r10, r0
  2125. bl kvmppc_msr_interrupt
  2126. b fast_interrupt_c_return
  2127. 3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
  2128. ld r5, KVM_VRMA_SLB_V(r6)
  2129. b 4b
  2130. /*
  2131. * Try to handle an hcall in real mode.
  2132. * Returns to the guest if we handle it, or continues on up to
  2133. * the kernel if we can't (i.e. if we don't have a handler for
  2134. * it, or if the handler returns H_TOO_HARD).
  2135. *
  2136. * r5 - r8 contain hcall args,
  2137. * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
  2138. */
  2139. hcall_try_real_mode:
  2140. ld r3,VCPU_GPR(R3)(r9)
  2141. andi. r0,r11,MSR_PR
  2142. /* sc 1 from userspace - reflect to guest syscall */
  2143. bne sc_1_fast_return
  2144. clrrdi r3,r3,2
  2145. cmpldi r3,hcall_real_table_end - hcall_real_table
  2146. bge guest_exit_cont
  2147. /* See if this hcall is enabled for in-kernel handling */
  2148. ld r4, VCPU_KVM(r9)
  2149. srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */
  2150. sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */
  2151. add r4, r4, r0
  2152. ld r0, KVM_ENABLED_HCALLS(r4)
  2153. rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */
  2154. srd r0, r0, r4
  2155. andi. r0, r0, 1
  2156. beq guest_exit_cont
  2157. /* Get pointer to handler, if any, and call it */
  2158. LOAD_REG_ADDR(r4, hcall_real_table)
  2159. lwax r3,r3,r4
  2160. cmpwi r3,0
  2161. beq guest_exit_cont
  2162. add r12,r3,r4
  2163. mtctr r12
  2164. mr r3,r9 /* get vcpu pointer */
  2165. ld r4,VCPU_GPR(R4)(r9)
  2166. bctrl
  2167. cmpdi r3,H_TOO_HARD
  2168. beq hcall_real_fallback
  2169. ld r4,HSTATE_KVM_VCPU(r13)
  2170. std r3,VCPU_GPR(R3)(r4)
  2171. ld r10,VCPU_PC(r4)
  2172. ld r11,VCPU_MSR(r4)
  2173. b fast_guest_return
  2174. sc_1_fast_return:
  2175. mtspr SPRN_SRR0,r10
  2176. mtspr SPRN_SRR1,r11
  2177. li r10, BOOK3S_INTERRUPT_SYSCALL
  2178. bl kvmppc_msr_interrupt
  2179. mr r4,r9
  2180. b fast_guest_return
  2181. /* We've attempted a real mode hcall, but it's punted it back
  2182. * to userspace. We need to restore some clobbered volatiles
  2183. * before resuming the pass-it-to-qemu path */
  2184. hcall_real_fallback:
  2185. li r12,BOOK3S_INTERRUPT_SYSCALL
  2186. ld r9, HSTATE_KVM_VCPU(r13)
  2187. b guest_exit_cont
  2188. .globl hcall_real_table
  2189. hcall_real_table:
  2190. .long 0 /* 0 - unused */
  2191. .long DOTSYM(kvmppc_h_remove) - hcall_real_table
  2192. .long DOTSYM(kvmppc_h_enter) - hcall_real_table
  2193. .long DOTSYM(kvmppc_h_read) - hcall_real_table
  2194. .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
  2195. .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
  2196. .long DOTSYM(kvmppc_h_protect) - hcall_real_table
  2197. .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table
  2198. .long DOTSYM(kvmppc_rm_h_put_tce) - hcall_real_table
  2199. .long 0 /* 0x24 - H_SET_SPRG0 */
  2200. .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
  2201. .long 0 /* 0x2c */
  2202. .long 0 /* 0x30 */
  2203. .long 0 /* 0x34 */
  2204. .long 0 /* 0x38 */
  2205. .long 0 /* 0x3c */
  2206. .long 0 /* 0x40 */
  2207. .long 0 /* 0x44 */
  2208. .long 0 /* 0x48 */
  2209. .long 0 /* 0x4c */
  2210. .long 0 /* 0x50 */
  2211. .long 0 /* 0x54 */
  2212. .long 0 /* 0x58 */
  2213. .long 0 /* 0x5c */
  2214. .long 0 /* 0x60 */
  2215. #ifdef CONFIG_KVM_XICS
  2216. .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
  2217. .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
  2218. .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
  2219. .long DOTSYM(kvmppc_rm_h_ipoll) - hcall_real_table
  2220. .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
  2221. #else
  2222. .long 0 /* 0x64 - H_EOI */
  2223. .long 0 /* 0x68 - H_CPPR */
  2224. .long 0 /* 0x6c - H_IPI */
  2225. .long 0 /* 0x70 - H_IPOLL */
  2226. .long 0 /* 0x74 - H_XIRR */
  2227. #endif
  2228. .long 0 /* 0x78 */
  2229. .long 0 /* 0x7c */
  2230. .long 0 /* 0x80 */
  2231. .long 0 /* 0x84 */
  2232. .long 0 /* 0x88 */
  2233. .long 0 /* 0x8c */
  2234. .long 0 /* 0x90 */
  2235. .long 0 /* 0x94 */
  2236. .long 0 /* 0x98 */
  2237. .long 0 /* 0x9c */
  2238. .long 0 /* 0xa0 */
  2239. .long 0 /* 0xa4 */
  2240. .long 0 /* 0xa8 */
  2241. .long 0 /* 0xac */
  2242. .long 0 /* 0xb0 */
  2243. .long 0 /* 0xb4 */
  2244. .long 0 /* 0xb8 */
  2245. .long 0 /* 0xbc */
  2246. .long 0 /* 0xc0 */
  2247. .long 0 /* 0xc4 */
  2248. .long 0 /* 0xc8 */
  2249. .long 0 /* 0xcc */
  2250. .long 0 /* 0xd0 */
  2251. .long 0 /* 0xd4 */
  2252. .long 0 /* 0xd8 */
  2253. .long 0 /* 0xdc */
  2254. .long DOTSYM(kvmppc_h_cede) - hcall_real_table
  2255. .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
  2256. .long 0 /* 0xe8 */
  2257. .long 0 /* 0xec */
  2258. .long 0 /* 0xf0 */
  2259. .long 0 /* 0xf4 */
  2260. .long 0 /* 0xf8 */
  2261. .long 0 /* 0xfc */
  2262. .long 0 /* 0x100 */
  2263. .long 0 /* 0x104 */
  2264. .long 0 /* 0x108 */
  2265. .long 0 /* 0x10c */
  2266. .long 0 /* 0x110 */
  2267. .long 0 /* 0x114 */
  2268. .long 0 /* 0x118 */
  2269. .long 0 /* 0x11c */
  2270. .long 0 /* 0x120 */
  2271. .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
  2272. .long 0 /* 0x128 */
  2273. .long 0 /* 0x12c */
  2274. .long 0 /* 0x130 */
  2275. .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
  2276. .long DOTSYM(kvmppc_rm_h_stuff_tce) - hcall_real_table
  2277. .long DOTSYM(kvmppc_rm_h_put_tce_indirect) - hcall_real_table
  2278. .long 0 /* 0x140 */
  2279. .long 0 /* 0x144 */
  2280. .long 0 /* 0x148 */
  2281. .long 0 /* 0x14c */
  2282. .long 0 /* 0x150 */
  2283. .long 0 /* 0x154 */
  2284. .long 0 /* 0x158 */
  2285. .long 0 /* 0x15c */
  2286. .long 0 /* 0x160 */
  2287. .long 0 /* 0x164 */
  2288. .long 0 /* 0x168 */
  2289. .long 0 /* 0x16c */
  2290. .long 0 /* 0x170 */
  2291. .long 0 /* 0x174 */
  2292. .long 0 /* 0x178 */
  2293. .long 0 /* 0x17c */
  2294. .long 0 /* 0x180 */
  2295. .long 0 /* 0x184 */
  2296. .long 0 /* 0x188 */
  2297. .long 0 /* 0x18c */
  2298. .long 0 /* 0x190 */
  2299. .long 0 /* 0x194 */
  2300. .long 0 /* 0x198 */
  2301. .long 0 /* 0x19c */
  2302. .long 0 /* 0x1a0 */
  2303. .long 0 /* 0x1a4 */
  2304. .long 0 /* 0x1a8 */
  2305. .long 0 /* 0x1ac */
  2306. .long 0 /* 0x1b0 */
  2307. .long 0 /* 0x1b4 */
  2308. .long 0 /* 0x1b8 */
  2309. .long 0 /* 0x1bc */
  2310. .long 0 /* 0x1c0 */
  2311. .long 0 /* 0x1c4 */
  2312. .long 0 /* 0x1c8 */
  2313. .long 0 /* 0x1cc */
  2314. .long 0 /* 0x1d0 */
  2315. .long 0 /* 0x1d4 */
  2316. .long 0 /* 0x1d8 */
  2317. .long 0 /* 0x1dc */
  2318. .long 0 /* 0x1e0 */
  2319. .long 0 /* 0x1e4 */
  2320. .long 0 /* 0x1e8 */
  2321. .long 0 /* 0x1ec */
  2322. .long 0 /* 0x1f0 */
  2323. .long 0 /* 0x1f4 */
  2324. .long 0 /* 0x1f8 */
  2325. .long 0 /* 0x1fc */
  2326. .long 0 /* 0x200 */
  2327. .long 0 /* 0x204 */
  2328. .long 0 /* 0x208 */
  2329. .long 0 /* 0x20c */
  2330. .long 0 /* 0x210 */
  2331. .long 0 /* 0x214 */
  2332. .long 0 /* 0x218 */
  2333. .long 0 /* 0x21c */
  2334. .long 0 /* 0x220 */
  2335. .long 0 /* 0x224 */
  2336. .long 0 /* 0x228 */
  2337. .long 0 /* 0x22c */
  2338. .long 0 /* 0x230 */
  2339. .long 0 /* 0x234 */
  2340. .long 0 /* 0x238 */
  2341. .long 0 /* 0x23c */
  2342. .long 0 /* 0x240 */
  2343. .long 0 /* 0x244 */
  2344. .long 0 /* 0x248 */
  2345. .long 0 /* 0x24c */
  2346. .long 0 /* 0x250 */
  2347. .long 0 /* 0x254 */
  2348. .long 0 /* 0x258 */
  2349. .long 0 /* 0x25c */
  2350. .long 0 /* 0x260 */
  2351. .long 0 /* 0x264 */
  2352. .long 0 /* 0x268 */
  2353. .long 0 /* 0x26c */
  2354. .long 0 /* 0x270 */
  2355. .long 0 /* 0x274 */
  2356. .long 0 /* 0x278 */
  2357. .long 0 /* 0x27c */
  2358. .long 0 /* 0x280 */
  2359. .long 0 /* 0x284 */
  2360. .long 0 /* 0x288 */
  2361. .long 0 /* 0x28c */
  2362. .long 0 /* 0x290 */
  2363. .long 0 /* 0x294 */
  2364. .long 0 /* 0x298 */
  2365. .long 0 /* 0x29c */
  2366. .long 0 /* 0x2a0 */
  2367. .long 0 /* 0x2a4 */
  2368. .long 0 /* 0x2a8 */
  2369. .long 0 /* 0x2ac */
  2370. .long 0 /* 0x2b0 */
  2371. .long 0 /* 0x2b4 */
  2372. .long 0 /* 0x2b8 */
  2373. .long 0 /* 0x2bc */
  2374. .long 0 /* 0x2c0 */
  2375. .long 0 /* 0x2c4 */
  2376. .long 0 /* 0x2c8 */
  2377. .long 0 /* 0x2cc */
  2378. .long 0 /* 0x2d0 */
  2379. .long 0 /* 0x2d4 */
  2380. .long 0 /* 0x2d8 */
  2381. .long 0 /* 0x2dc */
  2382. .long 0 /* 0x2e0 */
  2383. .long 0 /* 0x2e4 */
  2384. .long 0 /* 0x2e8 */
  2385. .long 0 /* 0x2ec */
  2386. .long 0 /* 0x2f0 */
  2387. .long 0 /* 0x2f4 */
  2388. .long 0 /* 0x2f8 */
  2389. #ifdef CONFIG_KVM_XICS
  2390. .long DOTSYM(kvmppc_rm_h_xirr_x) - hcall_real_table
  2391. #else
  2392. .long 0 /* 0x2fc - H_XIRR_X*/
  2393. #endif
  2394. .long DOTSYM(kvmppc_h_random) - hcall_real_table
  2395. .globl hcall_real_table_end
  2396. hcall_real_table_end:
  2397. _GLOBAL(kvmppc_h_set_xdabr)
  2398. andi. r0, r5, DABRX_USER | DABRX_KERNEL
  2399. beq 6f
  2400. li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
  2401. andc. r0, r5, r0
  2402. beq 3f
  2403. 6: li r3, H_PARAMETER
  2404. blr
  2405. _GLOBAL(kvmppc_h_set_dabr)
  2406. li r5, DABRX_USER | DABRX_KERNEL
  2407. 3:
  2408. BEGIN_FTR_SECTION
  2409. b 2f
  2410. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  2411. std r4,VCPU_DABR(r3)
  2412. stw r5, VCPU_DABRX(r3)
  2413. mtspr SPRN_DABRX, r5
  2414. /* Work around P7 bug where DABR can get corrupted on mtspr */
  2415. 1: mtspr SPRN_DABR,r4
  2416. mfspr r5, SPRN_DABR
  2417. cmpd r4, r5
  2418. bne 1b
  2419. isync
  2420. li r3,0
  2421. blr
  2422. 2:
  2423. BEGIN_FTR_SECTION
  2424. /* POWER9 with disabled DAWR */
  2425. li r3, H_HARDWARE
  2426. blr
  2427. END_FTR_SECTION_IFCLR(CPU_FTR_DAWR)
  2428. /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
  2429. rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
  2430. rlwimi r5, r4, 2, DAWRX_WT
  2431. clrrdi r4, r4, 3
  2432. std r4, VCPU_DAWR(r3)
  2433. std r5, VCPU_DAWRX(r3)
  2434. mtspr SPRN_DAWR, r4
  2435. mtspr SPRN_DAWRX, r5
  2436. li r3, 0
  2437. blr
  2438. _GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */
  2439. ori r11,r11,MSR_EE
  2440. std r11,VCPU_MSR(r3)
  2441. li r0,1
  2442. stb r0,VCPU_CEDED(r3)
  2443. sync /* order setting ceded vs. testing prodded */
  2444. lbz r5,VCPU_PRODDED(r3)
  2445. cmpwi r5,0
  2446. bne kvm_cede_prodded
  2447. li r12,0 /* set trap to 0 to say hcall is handled */
  2448. stw r12,VCPU_TRAP(r3)
  2449. li r0,H_SUCCESS
  2450. std r0,VCPU_GPR(R3)(r3)
  2451. /*
  2452. * Set our bit in the bitmask of napping threads unless all the
  2453. * other threads are already napping, in which case we send this
  2454. * up to the host.
  2455. */
  2456. ld r5,HSTATE_KVM_VCORE(r13)
  2457. lbz r6,HSTATE_PTID(r13)
  2458. lwz r8,VCORE_ENTRY_EXIT(r5)
  2459. clrldi r8,r8,56
  2460. li r0,1
  2461. sld r0,r0,r6
  2462. addi r6,r5,VCORE_NAPPING_THREADS
  2463. 31: lwarx r4,0,r6
  2464. or r4,r4,r0
  2465. cmpw r4,r8
  2466. beq kvm_cede_exit
  2467. stwcx. r4,0,r6
  2468. bne 31b
  2469. /* order napping_threads update vs testing entry_exit_map */
  2470. isync
  2471. li r0,NAPPING_CEDE
  2472. stb r0,HSTATE_NAPPING(r13)
  2473. lwz r7,VCORE_ENTRY_EXIT(r5)
  2474. cmpwi r7,0x100
  2475. bge 33f /* another thread already exiting */
  2476. /*
  2477. * Although not specifically required by the architecture, POWER7
  2478. * preserves the following registers in nap mode, even if an SMT mode
  2479. * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
  2480. * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
  2481. */
  2482. /* Save non-volatile GPRs */
  2483. std r14, VCPU_GPR(R14)(r3)
  2484. std r15, VCPU_GPR(R15)(r3)
  2485. std r16, VCPU_GPR(R16)(r3)
  2486. std r17, VCPU_GPR(R17)(r3)
  2487. std r18, VCPU_GPR(R18)(r3)
  2488. std r19, VCPU_GPR(R19)(r3)
  2489. std r20, VCPU_GPR(R20)(r3)
  2490. std r21, VCPU_GPR(R21)(r3)
  2491. std r22, VCPU_GPR(R22)(r3)
  2492. std r23, VCPU_GPR(R23)(r3)
  2493. std r24, VCPU_GPR(R24)(r3)
  2494. std r25, VCPU_GPR(R25)(r3)
  2495. std r26, VCPU_GPR(R26)(r3)
  2496. std r27, VCPU_GPR(R27)(r3)
  2497. std r28, VCPU_GPR(R28)(r3)
  2498. std r29, VCPU_GPR(R29)(r3)
  2499. std r30, VCPU_GPR(R30)(r3)
  2500. std r31, VCPU_GPR(R31)(r3)
  2501. /* save FP state */
  2502. bl kvmppc_save_fp
  2503. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  2504. /*
  2505. * Branch around the call if both CPU_FTR_TM and
  2506. * CPU_FTR_P9_TM_HV_ASSIST are off.
  2507. */
  2508. BEGIN_FTR_SECTION
  2509. b 91f
  2510. END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0)
  2511. /*
  2512. * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR
  2513. */
  2514. ld r3, HSTATE_KVM_VCPU(r13)
  2515. ld r4, VCPU_MSR(r3)
  2516. bl kvmppc_save_tm_hv
  2517. 91:
  2518. #endif
  2519. /*
  2520. * Set DEC to the smaller of DEC and HDEC, so that we wake
  2521. * no later than the end of our timeslice (HDEC interrupts
  2522. * don't wake us from nap).
  2523. */
  2524. mfspr r3, SPRN_DEC
  2525. mfspr r4, SPRN_HDEC
  2526. mftb r5
  2527. BEGIN_FTR_SECTION
  2528. /* On P9 check whether the guest has large decrementer mode enabled */
  2529. ld r6, HSTATE_KVM_VCORE(r13)
  2530. ld r6, VCORE_LPCR(r6)
  2531. andis. r6, r6, LPCR_LD@h
  2532. bne 68f
  2533. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  2534. extsw r3, r3
  2535. 68: EXTEND_HDEC(r4)
  2536. cmpd r3, r4
  2537. ble 67f
  2538. mtspr SPRN_DEC, r4
  2539. 67:
  2540. /* save expiry time of guest decrementer */
  2541. add r3, r3, r5
  2542. ld r4, HSTATE_KVM_VCPU(r13)
  2543. ld r5, HSTATE_KVM_VCORE(r13)
  2544. ld r6, VCORE_TB_OFFSET_APPL(r5)
  2545. subf r3, r6, r3 /* convert to host TB value */
  2546. std r3, VCPU_DEC_EXPIRES(r4)
  2547. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  2548. ld r4, HSTATE_KVM_VCPU(r13)
  2549. addi r3, r4, VCPU_TB_CEDE
  2550. bl kvmhv_accumulate_time
  2551. #endif
  2552. lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */
  2553. /*
  2554. * Take a nap until a decrementer or external or doobell interrupt
  2555. * occurs, with PECE1 and PECE0 set in LPCR.
  2556. * On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
  2557. * Also clear the runlatch bit before napping.
  2558. */
  2559. kvm_do_nap:
  2560. mfspr r0, SPRN_CTRLF
  2561. clrrdi r0, r0, 1
  2562. mtspr SPRN_CTRLT, r0
  2563. li r0,1
  2564. stb r0,HSTATE_HWTHREAD_REQ(r13)
  2565. mfspr r5,SPRN_LPCR
  2566. ori r5,r5,LPCR_PECE0 | LPCR_PECE1
  2567. BEGIN_FTR_SECTION
  2568. ori r5, r5, LPCR_PECEDH
  2569. rlwimi r5, r3, 0, LPCR_PECEDP
  2570. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  2571. kvm_nap_sequence: /* desired LPCR value in r5 */
  2572. BEGIN_FTR_SECTION
  2573. /*
  2574. * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset)
  2575. * enable state loss = 1 (allow SMT mode switch)
  2576. * requested level = 0 (just stop dispatching)
  2577. */
  2578. lis r3, (PSSCR_EC | PSSCR_ESL)@h
  2579. mtspr SPRN_PSSCR, r3
  2580. /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */
  2581. li r4, LPCR_PECE_HVEE@higher
  2582. sldi r4, r4, 32
  2583. or r5, r5, r4
  2584. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  2585. mtspr SPRN_LPCR,r5
  2586. isync
  2587. li r0, 0
  2588. std r0, HSTATE_SCRATCH0(r13)
  2589. ptesync
  2590. ld r0, HSTATE_SCRATCH0(r13)
  2591. 1: cmpd r0, r0
  2592. bne 1b
  2593. BEGIN_FTR_SECTION
  2594. nap
  2595. FTR_SECTION_ELSE
  2596. PPC_STOP
  2597. ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
  2598. b .
  2599. 33: mr r4, r3
  2600. li r3, 0
  2601. li r12, 0
  2602. b 34f
  2603. kvm_end_cede:
  2604. /* get vcpu pointer */
  2605. ld r4, HSTATE_KVM_VCPU(r13)
  2606. /* Woken by external or decrementer interrupt */
  2607. ld r1, HSTATE_HOST_R1(r13)
  2608. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  2609. addi r3, r4, VCPU_TB_RMINTR
  2610. bl kvmhv_accumulate_time
  2611. #endif
  2612. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  2613. /*
  2614. * Branch around the call if both CPU_FTR_TM and
  2615. * CPU_FTR_P9_TM_HV_ASSIST are off.
  2616. */
  2617. BEGIN_FTR_SECTION
  2618. b 91f
  2619. END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0)
  2620. /*
  2621. * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS INCLUDING CR
  2622. */
  2623. mr r3, r4
  2624. ld r4, VCPU_MSR(r3)
  2625. bl kvmppc_restore_tm_hv
  2626. ld r4, HSTATE_KVM_VCPU(r13)
  2627. 91:
  2628. #endif
  2629. /* load up FP state */
  2630. bl kvmppc_load_fp
  2631. /* Restore guest decrementer */
  2632. ld r3, VCPU_DEC_EXPIRES(r4)
  2633. ld r5, HSTATE_KVM_VCORE(r13)
  2634. ld r6, VCORE_TB_OFFSET_APPL(r5)
  2635. add r3, r3, r6 /* convert host TB to guest TB value */
  2636. mftb r7
  2637. subf r3, r7, r3
  2638. mtspr SPRN_DEC, r3
  2639. /* Load NV GPRS */
  2640. ld r14, VCPU_GPR(R14)(r4)
  2641. ld r15, VCPU_GPR(R15)(r4)
  2642. ld r16, VCPU_GPR(R16)(r4)
  2643. ld r17, VCPU_GPR(R17)(r4)
  2644. ld r18, VCPU_GPR(R18)(r4)
  2645. ld r19, VCPU_GPR(R19)(r4)
  2646. ld r20, VCPU_GPR(R20)(r4)
  2647. ld r21, VCPU_GPR(R21)(r4)
  2648. ld r22, VCPU_GPR(R22)(r4)
  2649. ld r23, VCPU_GPR(R23)(r4)
  2650. ld r24, VCPU_GPR(R24)(r4)
  2651. ld r25, VCPU_GPR(R25)(r4)
  2652. ld r26, VCPU_GPR(R26)(r4)
  2653. ld r27, VCPU_GPR(R27)(r4)
  2654. ld r28, VCPU_GPR(R28)(r4)
  2655. ld r29, VCPU_GPR(R29)(r4)
  2656. ld r30, VCPU_GPR(R30)(r4)
  2657. ld r31, VCPU_GPR(R31)(r4)
  2658. /* Check the wake reason in SRR1 to see why we got here */
  2659. bl kvmppc_check_wake_reason
  2660. /*
  2661. * Restore volatile registers since we could have called a
  2662. * C routine in kvmppc_check_wake_reason
  2663. * r4 = VCPU
  2664. * r3 tells us whether we need to return to host or not
  2665. * WARNING: it gets checked further down:
  2666. * should not modify r3 until this check is done.
  2667. */
  2668. ld r4, HSTATE_KVM_VCPU(r13)
  2669. /* clear our bit in vcore->napping_threads */
  2670. 34: ld r5,HSTATE_KVM_VCORE(r13)
  2671. lbz r7,HSTATE_PTID(r13)
  2672. li r0,1
  2673. sld r0,r0,r7
  2674. addi r6,r5,VCORE_NAPPING_THREADS
  2675. 32: lwarx r7,0,r6
  2676. andc r7,r7,r0
  2677. stwcx. r7,0,r6
  2678. bne 32b
  2679. li r0,0
  2680. stb r0,HSTATE_NAPPING(r13)
  2681. /* See if the wake reason saved in r3 means we need to exit */
  2682. stw r12, VCPU_TRAP(r4)
  2683. mr r9, r4
  2684. cmpdi r3, 0
  2685. bgt guest_exit_cont
  2686. /* see if any other thread is already exiting */
  2687. lwz r0,VCORE_ENTRY_EXIT(r5)
  2688. cmpwi r0,0x100
  2689. bge guest_exit_cont
  2690. b kvmppc_cede_reentry /* if not go back to guest */
  2691. /* cede when already previously prodded case */
  2692. kvm_cede_prodded:
  2693. li r0,0
  2694. stb r0,VCPU_PRODDED(r3)
  2695. sync /* order testing prodded vs. clearing ceded */
  2696. stb r0,VCPU_CEDED(r3)
  2697. li r3,H_SUCCESS
  2698. blr
  2699. /* we've ceded but we want to give control to the host */
  2700. kvm_cede_exit:
  2701. ld r9, HSTATE_KVM_VCPU(r13)
  2702. #ifdef CONFIG_KVM_XICS
  2703. /* are we using XIVE with single escalation? */
  2704. ld r10, VCPU_XIVE_ESC_VADDR(r9)
  2705. cmpdi r10, 0
  2706. beq 3f
  2707. li r6, XIVE_ESB_SET_PQ_00
  2708. /*
  2709. * If we still have a pending escalation, abort the cede,
  2710. * and we must set PQ to 10 rather than 00 so that we don't
  2711. * potentially end up with two entries for the escalation
  2712. * interrupt in the XIVE interrupt queue. In that case
  2713. * we also don't want to set xive_esc_on to 1 here in
  2714. * case we race with xive_esc_irq().
  2715. */
  2716. lbz r5, VCPU_XIVE_ESC_ON(r9)
  2717. cmpwi r5, 0
  2718. beq 4f
  2719. li r0, 0
  2720. stb r0, VCPU_CEDED(r9)
  2721. li r6, XIVE_ESB_SET_PQ_10
  2722. b 5f
  2723. 4: li r0, 1
  2724. stb r0, VCPU_XIVE_ESC_ON(r9)
  2725. /* make sure store to xive_esc_on is seen before xive_esc_irq runs */
  2726. sync
  2727. 5: /* Enable XIVE escalation */
  2728. mfmsr r0
  2729. andi. r0, r0, MSR_DR /* in real mode? */
  2730. beq 1f
  2731. ldx r0, r10, r6
  2732. b 2f
  2733. 1: ld r10, VCPU_XIVE_ESC_RADDR(r9)
  2734. ldcix r0, r10, r6
  2735. 2: sync
  2736. #endif /* CONFIG_KVM_XICS */
  2737. 3: b guest_exit_cont
  2738. /* Try to handle a machine check in real mode */
  2739. machine_check_realmode:
  2740. mr r3, r9 /* get vcpu pointer */
  2741. bl kvmppc_realmode_machine_check
  2742. nop
  2743. ld r9, HSTATE_KVM_VCPU(r13)
  2744. li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
  2745. /*
  2746. * For the guest that is FWNMI capable, deliver all the MCE errors
  2747. * (handled/unhandled) by exiting the guest with KVM_EXIT_NMI exit
  2748. * reason. This new approach injects machine check errors in guest
  2749. * address space to guest with additional information in the form
  2750. * of RTAS event, thus enabling guest kernel to suitably handle
  2751. * such errors.
  2752. *
  2753. * For the guest that is not FWNMI capable (old QEMU) fallback
  2754. * to old behaviour for backward compatibility:
  2755. * Deliver unhandled/fatal (e.g. UE) MCE errors to guest either
  2756. * through machine check interrupt (set HSRR0 to 0x200).
  2757. * For handled errors (no-fatal), just go back to guest execution
  2758. * with current HSRR0.
  2759. * if we receive machine check with MSR(RI=0) then deliver it to
  2760. * guest as machine check causing guest to crash.
  2761. */
  2762. ld r11, VCPU_MSR(r9)
  2763. rldicl. r0, r11, 64-MSR_HV_LG, 63 /* check if it happened in HV mode */
  2764. bne mc_cont /* if so, exit to host */
  2765. /* Check if guest is capable of handling NMI exit */
  2766. ld r10, VCPU_KVM(r9)
  2767. lbz r10, KVM_FWNMI(r10)
  2768. cmpdi r10, 1 /* FWNMI capable? */
  2769. beq mc_cont /* if so, exit with KVM_EXIT_NMI. */
  2770. /* if not, fall through for backward compatibility. */
  2771. andi. r10, r11, MSR_RI /* check for unrecoverable exception */
  2772. beq 1f /* Deliver a machine check to guest */
  2773. ld r10, VCPU_PC(r9)
  2774. cmpdi r3, 0 /* Did we handle MCE ? */
  2775. bne 2f /* Continue guest execution. */
  2776. /* If not, deliver a machine check. SRR0/1 are already set */
  2777. 1: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK
  2778. bl kvmppc_msr_interrupt
  2779. 2: b fast_interrupt_c_return
  2780. /*
  2781. * Check the reason we woke from nap, and take appropriate action.
  2782. * Returns (in r3):
  2783. * 0 if nothing needs to be done
  2784. * 1 if something happened that needs to be handled by the host
  2785. * -1 if there was a guest wakeup (IPI or msgsnd)
  2786. * -2 if we handled a PCI passthrough interrupt (returned by
  2787. * kvmppc_read_intr only)
  2788. *
  2789. * Also sets r12 to the interrupt vector for any interrupt that needs
  2790. * to be handled now by the host (0x500 for external interrupt), or zero.
  2791. * Modifies all volatile registers (since it may call a C function).
  2792. * This routine calls kvmppc_read_intr, a C function, if an external
  2793. * interrupt is pending.
  2794. */
  2795. kvmppc_check_wake_reason:
  2796. mfspr r6, SPRN_SRR1
  2797. BEGIN_FTR_SECTION
  2798. rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */
  2799. FTR_SECTION_ELSE
  2800. rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */
  2801. ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
  2802. cmpwi r6, 8 /* was it an external interrupt? */
  2803. beq 7f /* if so, see what it was */
  2804. li r3, 0
  2805. li r12, 0
  2806. cmpwi r6, 6 /* was it the decrementer? */
  2807. beq 0f
  2808. BEGIN_FTR_SECTION
  2809. cmpwi r6, 5 /* privileged doorbell? */
  2810. beq 0f
  2811. cmpwi r6, 3 /* hypervisor doorbell? */
  2812. beq 3f
  2813. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
  2814. cmpwi r6, 0xa /* Hypervisor maintenance ? */
  2815. beq 4f
  2816. li r3, 1 /* anything else, return 1 */
  2817. 0: blr
  2818. /* hypervisor doorbell */
  2819. 3: li r12, BOOK3S_INTERRUPT_H_DOORBELL
  2820. /*
  2821. * Clear the doorbell as we will invoke the handler
  2822. * explicitly in the guest exit path.
  2823. */
  2824. lis r6, (PPC_DBELL_SERVER << (63-36))@h
  2825. PPC_MSGCLR(6)
  2826. /* see if it's a host IPI */
  2827. li r3, 1
  2828. BEGIN_FTR_SECTION
  2829. PPC_MSGSYNC
  2830. lwsync
  2831. END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
  2832. lbz r0, HSTATE_HOST_IPI(r13)
  2833. cmpwi r0, 0
  2834. bnelr
  2835. /* if not, return -1 */
  2836. li r3, -1
  2837. blr
  2838. /* Woken up due to Hypervisor maintenance interrupt */
  2839. 4: li r12, BOOK3S_INTERRUPT_HMI
  2840. li r3, 1
  2841. blr
  2842. /* external interrupt - create a stack frame so we can call C */
  2843. 7: mflr r0
  2844. std r0, PPC_LR_STKOFF(r1)
  2845. stdu r1, -PPC_MIN_STKFRM(r1)
  2846. bl kvmppc_read_intr
  2847. nop
  2848. li r12, BOOK3S_INTERRUPT_EXTERNAL
  2849. cmpdi r3, 1
  2850. ble 1f
  2851. /*
  2852. * Return code of 2 means PCI passthrough interrupt, but
  2853. * we need to return back to host to complete handling the
  2854. * interrupt. Trap reason is expected in r12 by guest
  2855. * exit code.
  2856. */
  2857. li r12, BOOK3S_INTERRUPT_HV_RM_HARD
  2858. 1:
  2859. ld r0, PPC_MIN_STKFRM+PPC_LR_STKOFF(r1)
  2860. addi r1, r1, PPC_MIN_STKFRM
  2861. mtlr r0
  2862. blr
  2863. /*
  2864. * Save away FP, VMX and VSX registers.
  2865. * r3 = vcpu pointer
  2866. * N.B. r30 and r31 are volatile across this function,
  2867. * thus it is not callable from C.
  2868. */
  2869. kvmppc_save_fp:
  2870. mflr r30
  2871. mr r31,r3
  2872. mfmsr r5
  2873. ori r8,r5,MSR_FP
  2874. #ifdef CONFIG_ALTIVEC
  2875. BEGIN_FTR_SECTION
  2876. oris r8,r8,MSR_VEC@h
  2877. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  2878. #endif
  2879. #ifdef CONFIG_VSX
  2880. BEGIN_FTR_SECTION
  2881. oris r8,r8,MSR_VSX@h
  2882. END_FTR_SECTION_IFSET(CPU_FTR_VSX)
  2883. #endif
  2884. mtmsrd r8
  2885. addi r3,r3,VCPU_FPRS
  2886. bl store_fp_state
  2887. #ifdef CONFIG_ALTIVEC
  2888. BEGIN_FTR_SECTION
  2889. addi r3,r31,VCPU_VRS
  2890. bl store_vr_state
  2891. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  2892. #endif
  2893. mfspr r6,SPRN_VRSAVE
  2894. stw r6,VCPU_VRSAVE(r31)
  2895. mtlr r30
  2896. blr
  2897. /*
  2898. * Load up FP, VMX and VSX registers
  2899. * r4 = vcpu pointer
  2900. * N.B. r30 and r31 are volatile across this function,
  2901. * thus it is not callable from C.
  2902. */
  2903. kvmppc_load_fp:
  2904. mflr r30
  2905. mr r31,r4
  2906. mfmsr r9
  2907. ori r8,r9,MSR_FP
  2908. #ifdef CONFIG_ALTIVEC
  2909. BEGIN_FTR_SECTION
  2910. oris r8,r8,MSR_VEC@h
  2911. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  2912. #endif
  2913. #ifdef CONFIG_VSX
  2914. BEGIN_FTR_SECTION
  2915. oris r8,r8,MSR_VSX@h
  2916. END_FTR_SECTION_IFSET(CPU_FTR_VSX)
  2917. #endif
  2918. mtmsrd r8
  2919. addi r3,r4,VCPU_FPRS
  2920. bl load_fp_state
  2921. #ifdef CONFIG_ALTIVEC
  2922. BEGIN_FTR_SECTION
  2923. addi r3,r31,VCPU_VRS
  2924. bl load_vr_state
  2925. END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  2926. #endif
  2927. lwz r7,VCPU_VRSAVE(r31)
  2928. mtspr SPRN_VRSAVE,r7
  2929. mtlr r30
  2930. mr r4,r31
  2931. blr
  2932. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  2933. /*
  2934. * Save transactional state and TM-related registers.
  2935. * Called with r3 pointing to the vcpu struct and r4 containing
  2936. * the guest MSR value.
  2937. * This can modify all checkpointed registers, but
  2938. * restores r1 and r2 before exit.
  2939. */
  2940. kvmppc_save_tm_hv:
  2941. /* See if we need to handle fake suspend mode */
  2942. BEGIN_FTR_SECTION
  2943. b __kvmppc_save_tm
  2944. END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
  2945. lbz r0, HSTATE_FAKE_SUSPEND(r13) /* Were we fake suspended? */
  2946. cmpwi r0, 0
  2947. beq __kvmppc_save_tm
  2948. /* The following code handles the fake_suspend = 1 case */
  2949. mflr r0
  2950. std r0, PPC_LR_STKOFF(r1)
  2951. stdu r1, -PPC_MIN_STKFRM(r1)
  2952. /* Turn on TM. */
  2953. mfmsr r8
  2954. li r0, 1
  2955. rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
  2956. mtmsrd r8
  2957. rldicl. r8, r8, 64 - MSR_TS_S_LG, 62 /* Did we actually hrfid? */
  2958. beq 4f
  2959. BEGIN_FTR_SECTION
  2960. bl pnv_power9_force_smt4_catch
  2961. END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
  2962. nop
  2963. std r1, HSTATE_HOST_R1(r13)
  2964. /* Clear the MSR RI since r1, r13 may be foobar. */
  2965. li r5, 0
  2966. mtmsrd r5, 1
  2967. /* We have to treclaim here because that's the only way to do S->N */
  2968. li r3, TM_CAUSE_KVM_RESCHED
  2969. TRECLAIM(R3)
  2970. /*
  2971. * We were in fake suspend, so we are not going to save the
  2972. * register state as the guest checkpointed state (since
  2973. * we already have it), therefore we can now use any volatile GPR.
  2974. */
  2975. /* Reload PACA pointer, stack pointer and TOC. */
  2976. GET_PACA(r13)
  2977. ld r1, HSTATE_HOST_R1(r13)
  2978. ld r2, PACATOC(r13)
  2979. /* Set MSR RI now we have r1 and r13 back. */
  2980. li r5, MSR_RI
  2981. mtmsrd r5, 1
  2982. HMT_MEDIUM
  2983. ld r6, HSTATE_DSCR(r13)
  2984. mtspr SPRN_DSCR, r6
  2985. BEGIN_FTR_SECTION_NESTED(96)
  2986. bl pnv_power9_force_smt4_release
  2987. END_FTR_SECTION_NESTED(CPU_FTR_P9_TM_XER_SO_BUG, CPU_FTR_P9_TM_XER_SO_BUG, 96)
  2988. nop
  2989. 4:
  2990. mfspr r3, SPRN_PSSCR
  2991. /* PSSCR_FAKE_SUSPEND is a write-only bit, but clear it anyway */
  2992. li r0, PSSCR_FAKE_SUSPEND
  2993. andc r3, r3, r0
  2994. mtspr SPRN_PSSCR, r3
  2995. /* Don't save TEXASR, use value from last exit in real suspend state */
  2996. ld r9, HSTATE_KVM_VCPU(r13)
  2997. mfspr r5, SPRN_TFHAR
  2998. mfspr r6, SPRN_TFIAR
  2999. std r5, VCPU_TFHAR(r9)
  3000. std r6, VCPU_TFIAR(r9)
  3001. addi r1, r1, PPC_MIN_STKFRM
  3002. ld r0, PPC_LR_STKOFF(r1)
  3003. mtlr r0
  3004. blr
  3005. /*
  3006. * Restore transactional state and TM-related registers.
  3007. * Called with r3 pointing to the vcpu struct
  3008. * and r4 containing the guest MSR value.
  3009. * This potentially modifies all checkpointed registers.
  3010. * It restores r1 and r2 from the PACA.
  3011. */
  3012. kvmppc_restore_tm_hv:
  3013. /*
  3014. * If we are doing TM emulation for the guest on a POWER9 DD2,
  3015. * then we don't actually do a trechkpt -- we either set up
  3016. * fake-suspend mode, or emulate a TM rollback.
  3017. */
  3018. BEGIN_FTR_SECTION
  3019. b __kvmppc_restore_tm
  3020. END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
  3021. mflr r0
  3022. std r0, PPC_LR_STKOFF(r1)
  3023. li r0, 0
  3024. stb r0, HSTATE_FAKE_SUSPEND(r13)
  3025. /* Turn on TM so we can restore TM SPRs */
  3026. mfmsr r5
  3027. li r0, 1
  3028. rldimi r5, r0, MSR_TM_LG, 63-MSR_TM_LG
  3029. mtmsrd r5
  3030. /*
  3031. * The user may change these outside of a transaction, so they must
  3032. * always be context switched.
  3033. */
  3034. ld r5, VCPU_TFHAR(r3)
  3035. ld r6, VCPU_TFIAR(r3)
  3036. ld r7, VCPU_TEXASR(r3)
  3037. mtspr SPRN_TFHAR, r5
  3038. mtspr SPRN_TFIAR, r6
  3039. mtspr SPRN_TEXASR, r7
  3040. rldicl. r5, r4, 64 - MSR_TS_S_LG, 62
  3041. beqlr /* TM not active in guest */
  3042. /* Make sure the failure summary is set */
  3043. oris r7, r7, (TEXASR_FS)@h
  3044. mtspr SPRN_TEXASR, r7
  3045. cmpwi r5, 1 /* check for suspended state */
  3046. bgt 10f
  3047. stb r5, HSTATE_FAKE_SUSPEND(r13)
  3048. b 9f /* and return */
  3049. 10: stdu r1, -PPC_MIN_STKFRM(r1)
  3050. /* guest is in transactional state, so simulate rollback */
  3051. bl kvmhv_emulate_tm_rollback
  3052. nop
  3053. addi r1, r1, PPC_MIN_STKFRM
  3054. 9: ld r0, PPC_LR_STKOFF(r1)
  3055. mtlr r0
  3056. blr
  3057. #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
  3058. /*
  3059. * We come here if we get any exception or interrupt while we are
  3060. * executing host real mode code while in guest MMU context.
  3061. * r12 is (CR << 32) | vector
  3062. * r13 points to our PACA
  3063. * r12 is saved in HSTATE_SCRATCH0(r13)
  3064. * ctr is saved in HSTATE_SCRATCH1(r13) if RELOCATABLE
  3065. * r9 is saved in HSTATE_SCRATCH2(r13)
  3066. * r13 is saved in HSPRG1
  3067. * cfar is saved in HSTATE_CFAR(r13)
  3068. * ppr is saved in HSTATE_PPR(r13)
  3069. */
  3070. kvmppc_bad_host_intr:
  3071. /*
  3072. * Switch to the emergency stack, but start half-way down in
  3073. * case we were already on it.
  3074. */
  3075. mr r9, r1
  3076. std r1, PACAR1(r13)
  3077. ld r1, PACAEMERGSP(r13)
  3078. subi r1, r1, THREAD_SIZE/2 + INT_FRAME_SIZE
  3079. std r9, 0(r1)
  3080. std r0, GPR0(r1)
  3081. std r9, GPR1(r1)
  3082. std r2, GPR2(r1)
  3083. SAVE_4GPRS(3, r1)
  3084. SAVE_2GPRS(7, r1)
  3085. srdi r0, r12, 32
  3086. clrldi r12, r12, 32
  3087. std r0, _CCR(r1)
  3088. std r12, _TRAP(r1)
  3089. andi. r0, r12, 2
  3090. beq 1f
  3091. mfspr r3, SPRN_HSRR0
  3092. mfspr r4, SPRN_HSRR1
  3093. mfspr r5, SPRN_HDAR
  3094. mfspr r6, SPRN_HDSISR
  3095. b 2f
  3096. 1: mfspr r3, SPRN_SRR0
  3097. mfspr r4, SPRN_SRR1
  3098. mfspr r5, SPRN_DAR
  3099. mfspr r6, SPRN_DSISR
  3100. 2: std r3, _NIP(r1)
  3101. std r4, _MSR(r1)
  3102. std r5, _DAR(r1)
  3103. std r6, _DSISR(r1)
  3104. ld r9, HSTATE_SCRATCH2(r13)
  3105. ld r12, HSTATE_SCRATCH0(r13)
  3106. GET_SCRATCH0(r0)
  3107. SAVE_4GPRS(9, r1)
  3108. std r0, GPR13(r1)
  3109. SAVE_NVGPRS(r1)
  3110. ld r5, HSTATE_CFAR(r13)
  3111. std r5, ORIG_GPR3(r1)
  3112. mflr r3
  3113. #ifdef CONFIG_RELOCATABLE
  3114. ld r4, HSTATE_SCRATCH1(r13)
  3115. #else
  3116. mfctr r4
  3117. #endif
  3118. mfxer r5
  3119. lbz r6, PACAIRQSOFTMASK(r13)
  3120. std r3, _LINK(r1)
  3121. std r4, _CTR(r1)
  3122. std r5, _XER(r1)
  3123. std r6, SOFTE(r1)
  3124. ld r2, PACATOC(r13)
  3125. LOAD_REG_IMMEDIATE(3, 0x7265677368657265)
  3126. std r3, STACK_FRAME_OVERHEAD-16(r1)
  3127. /*
  3128. * On POWER9 do a minimal restore of the MMU and call C code,
  3129. * which will print a message and panic.
  3130. * XXX On POWER7 and POWER8, we just spin here since we don't
  3131. * know what the other threads are doing (and we don't want to
  3132. * coordinate with them) - but at least we now have register state
  3133. * in memory that we might be able to look at from another CPU.
  3134. */
  3135. BEGIN_FTR_SECTION
  3136. b .
  3137. END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
  3138. ld r9, HSTATE_KVM_VCPU(r13)
  3139. ld r10, VCPU_KVM(r9)
  3140. li r0, 0
  3141. mtspr SPRN_AMR, r0
  3142. mtspr SPRN_IAMR, r0
  3143. mtspr SPRN_CIABR, r0
  3144. mtspr SPRN_DAWRX, r0
  3145. BEGIN_MMU_FTR_SECTION
  3146. b 4f
  3147. END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
  3148. slbmte r0, r0
  3149. slbia
  3150. ptesync
  3151. ld r8, PACA_SLBSHADOWPTR(r13)
  3152. .rept SLB_NUM_BOLTED
  3153. li r3, SLBSHADOW_SAVEAREA
  3154. LDX_BE r5, r8, r3
  3155. addi r3, r3, 8
  3156. LDX_BE r6, r8, r3
  3157. andis. r7, r5, SLB_ESID_V@h
  3158. beq 3f
  3159. slbmte r6, r5
  3160. 3: addi r8, r8, 16
  3161. .endr
  3162. 4: lwz r7, KVM_HOST_LPID(r10)
  3163. mtspr SPRN_LPID, r7
  3164. mtspr SPRN_PID, r0
  3165. ld r8, KVM_HOST_LPCR(r10)
  3166. mtspr SPRN_LPCR, r8
  3167. isync
  3168. li r0, KVM_GUEST_MODE_NONE
  3169. stb r0, HSTATE_IN_GUEST(r13)
  3170. /*
  3171. * Turn on the MMU and jump to C code
  3172. */
  3173. bcl 20, 31, .+4
  3174. 5: mflr r3
  3175. addi r3, r3, 9f - 5b
  3176. li r4, -1
  3177. rldimi r3, r4, 62, 0 /* ensure 0xc000000000000000 bits are set */
  3178. ld r4, PACAKMSR(r13)
  3179. mtspr SPRN_SRR0, r3
  3180. mtspr SPRN_SRR1, r4
  3181. RFI_TO_KERNEL
  3182. 9: addi r3, r1, STACK_FRAME_OVERHEAD
  3183. bl kvmppc_bad_interrupt
  3184. b 9b
  3185. /*
  3186. * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken
  3187. * from VCPU_INTR_MSR and is modified based on the required TM state changes.
  3188. * r11 has the guest MSR value (in/out)
  3189. * r9 has a vcpu pointer (in)
  3190. * r0 is used as a scratch register
  3191. */
  3192. kvmppc_msr_interrupt:
  3193. rldicl r0, r11, 64 - MSR_TS_S_LG, 62
  3194. cmpwi r0, 2 /* Check if we are in transactional state.. */
  3195. ld r11, VCPU_INTR_MSR(r9)
  3196. bne 1f
  3197. /* ... if transactional, change to suspended */
  3198. li r0, 1
  3199. 1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
  3200. blr
  3201. /*
  3202. * This works around a hardware bug on POWER8E processors, where
  3203. * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
  3204. * performance monitor interrupt. Instead, when we need to have
  3205. * an interrupt pending, we have to arrange for a counter to overflow.
  3206. */
  3207. kvmppc_fix_pmao:
  3208. li r3, 0
  3209. mtspr SPRN_MMCR2, r3
  3210. lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
  3211. ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
  3212. mtspr SPRN_MMCR0, r3
  3213. lis r3, 0x7fff
  3214. ori r3, r3, 0xffff
  3215. mtspr SPRN_PMC6, r3
  3216. isync
  3217. blr
  3218. #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
  3219. /*
  3220. * Start timing an activity
  3221. * r3 = pointer to time accumulation struct, r4 = vcpu
  3222. */
  3223. kvmhv_start_timing:
  3224. ld r5, HSTATE_KVM_VCORE(r13)
  3225. ld r6, VCORE_TB_OFFSET_APPL(r5)
  3226. mftb r5
  3227. subf r5, r6, r5 /* subtract current timebase offset */
  3228. std r3, VCPU_CUR_ACTIVITY(r4)
  3229. std r5, VCPU_ACTIVITY_START(r4)
  3230. blr
  3231. /*
  3232. * Accumulate time to one activity and start another.
  3233. * r3 = pointer to new time accumulation struct, r4 = vcpu
  3234. */
  3235. kvmhv_accumulate_time:
  3236. ld r5, HSTATE_KVM_VCORE(r13)
  3237. ld r8, VCORE_TB_OFFSET_APPL(r5)
  3238. ld r5, VCPU_CUR_ACTIVITY(r4)
  3239. ld r6, VCPU_ACTIVITY_START(r4)
  3240. std r3, VCPU_CUR_ACTIVITY(r4)
  3241. mftb r7
  3242. subf r7, r8, r7 /* subtract current timebase offset */
  3243. std r7, VCPU_ACTIVITY_START(r4)
  3244. cmpdi r5, 0
  3245. beqlr
  3246. subf r3, r6, r7
  3247. ld r8, TAS_SEQCOUNT(r5)
  3248. cmpdi r8, 0
  3249. addi r8, r8, 1
  3250. std r8, TAS_SEQCOUNT(r5)
  3251. lwsync
  3252. ld r7, TAS_TOTAL(r5)
  3253. add r7, r7, r3
  3254. std r7, TAS_TOTAL(r5)
  3255. ld r6, TAS_MIN(r5)
  3256. ld r7, TAS_MAX(r5)
  3257. beq 3f
  3258. cmpd r3, r6
  3259. bge 1f
  3260. 3: std r3, TAS_MIN(r5)
  3261. 1: cmpd r3, r7
  3262. ble 2f
  3263. std r3, TAS_MAX(r5)
  3264. 2: lwsync
  3265. addi r8, r8, 1
  3266. std r8, TAS_SEQCOUNT(r5)
  3267. blr
  3268. #endif