traps.c 56 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220
  1. /*
  2. * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  3. * Copyright 2007-2010 Freescale Semiconductor, Inc.
  4. *
  5. * This program is free software; you can redistribute it and/or
  6. * modify it under the terms of the GNU General Public License
  7. * as published by the Free Software Foundation; either version
  8. * 2 of the License, or (at your option) any later version.
  9. *
  10. * Modified by Cort Dougan (cort@cs.nmt.edu)
  11. * and Paul Mackerras (paulus@samba.org)
  12. */
  13. /*
  14. * This file handles the architecture-dependent parts of hardware exceptions
  15. */
  16. #include <linux/errno.h>
  17. #include <linux/sched.h>
  18. #include <linux/sched/debug.h>
  19. #include <linux/kernel.h>
  20. #include <linux/mm.h>
  21. #include <linux/pkeys.h>
  22. #include <linux/stddef.h>
  23. #include <linux/unistd.h>
  24. #include <linux/ptrace.h>
  25. #include <linux/user.h>
  26. #include <linux/interrupt.h>
  27. #include <linux/init.h>
  28. #include <linux/extable.h>
  29. #include <linux/module.h> /* print_modules */
  30. #include <linux/prctl.h>
  31. #include <linux/delay.h>
  32. #include <linux/kprobes.h>
  33. #include <linux/kexec.h>
  34. #include <linux/backlight.h>
  35. #include <linux/bug.h>
  36. #include <linux/kdebug.h>
  37. #include <linux/ratelimit.h>
  38. #include <linux/context_tracking.h>
  39. #include <linux/smp.h>
  40. #include <linux/console.h>
  41. #include <linux/kmsg_dump.h>
  42. #include <asm/emulated_ops.h>
  43. #include <asm/pgtable.h>
  44. #include <linux/uaccess.h>
  45. #include <asm/debugfs.h>
  46. #include <asm/io.h>
  47. #include <asm/machdep.h>
  48. #include <asm/rtas.h>
  49. #include <asm/pmc.h>
  50. #include <asm/reg.h>
  51. #ifdef CONFIG_PMAC_BACKLIGHT
  52. #include <asm/backlight.h>
  53. #endif
  54. #ifdef CONFIG_PPC64
  55. #include <asm/firmware.h>
  56. #include <asm/processor.h>
  57. #include <asm/tm.h>
  58. #endif
  59. #include <asm/kexec.h>
  60. #include <asm/ppc-opcode.h>
  61. #include <asm/rio.h>
  62. #include <asm/fadump.h>
  63. #include <asm/switch_to.h>
  64. #include <asm/tm.h>
  65. #include <asm/debug.h>
  66. #include <asm/asm-prototypes.h>
  67. #include <asm/hmi.h>
  68. #include <sysdev/fsl_pci.h>
  69. #include <asm/kprobes.h>
  70. #include <asm/stacktrace.h>
  71. #if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE)
  72. int (*__debugger)(struct pt_regs *regs) __read_mostly;
  73. int (*__debugger_ipi)(struct pt_regs *regs) __read_mostly;
  74. int (*__debugger_bpt)(struct pt_regs *regs) __read_mostly;
  75. int (*__debugger_sstep)(struct pt_regs *regs) __read_mostly;
  76. int (*__debugger_iabr_match)(struct pt_regs *regs) __read_mostly;
  77. int (*__debugger_break_match)(struct pt_regs *regs) __read_mostly;
  78. int (*__debugger_fault_handler)(struct pt_regs *regs) __read_mostly;
  79. EXPORT_SYMBOL(__debugger);
  80. EXPORT_SYMBOL(__debugger_ipi);
  81. EXPORT_SYMBOL(__debugger_bpt);
  82. EXPORT_SYMBOL(__debugger_sstep);
  83. EXPORT_SYMBOL(__debugger_iabr_match);
  84. EXPORT_SYMBOL(__debugger_break_match);
  85. EXPORT_SYMBOL(__debugger_fault_handler);
  86. #endif
  87. /* Transactional Memory trap debug */
  88. #ifdef TM_DEBUG_SW
  89. #define TM_DEBUG(x...) printk(KERN_INFO x)
  90. #else
  91. #define TM_DEBUG(x...) do { } while(0)
  92. #endif
  93. static const char *signame(int signr)
  94. {
  95. switch (signr) {
  96. case SIGBUS: return "bus error";
  97. case SIGFPE: return "floating point exception";
  98. case SIGILL: return "illegal instruction";
  99. case SIGSEGV: return "segfault";
  100. case SIGTRAP: return "unhandled trap";
  101. }
  102. return "unknown signal";
  103. }
  104. /*
  105. * Trap & Exception support
  106. */
  107. #ifdef CONFIG_PMAC_BACKLIGHT
  108. static void pmac_backlight_unblank(void)
  109. {
  110. mutex_lock(&pmac_backlight_mutex);
  111. if (pmac_backlight) {
  112. struct backlight_properties *props;
  113. props = &pmac_backlight->props;
  114. props->brightness = props->max_brightness;
  115. props->power = FB_BLANK_UNBLANK;
  116. backlight_update_status(pmac_backlight);
  117. }
  118. mutex_unlock(&pmac_backlight_mutex);
  119. }
  120. #else
  121. static inline void pmac_backlight_unblank(void) { }
  122. #endif
  123. /*
  124. * If oops/die is expected to crash the machine, return true here.
  125. *
  126. * This should not be expected to be 100% accurate, there may be
  127. * notifiers registered or other unexpected conditions that may bring
  128. * down the kernel. Or if the current process in the kernel is holding
  129. * locks or has other critical state, the kernel may become effectively
  130. * unusable anyway.
  131. */
  132. bool die_will_crash(void)
  133. {
  134. if (should_fadump_crash())
  135. return true;
  136. if (kexec_should_crash(current))
  137. return true;
  138. if (in_interrupt() || panic_on_oops ||
  139. !current->pid || is_global_init(current))
  140. return true;
  141. return false;
  142. }
  143. static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED;
  144. static int die_owner = -1;
  145. static unsigned int die_nest_count;
  146. static int die_counter;
  147. extern void panic_flush_kmsg_start(void)
  148. {
  149. /*
  150. * These are mostly taken from kernel/panic.c, but tries to do
  151. * relatively minimal work. Don't use delay functions (TB may
  152. * be broken), don't crash dump (need to set a firmware log),
  153. * don't run notifiers. We do want to get some information to
  154. * Linux console.
  155. */
  156. console_verbose();
  157. bust_spinlocks(1);
  158. }
  159. extern void panic_flush_kmsg_end(void)
  160. {
  161. printk_safe_flush_on_panic();
  162. kmsg_dump(KMSG_DUMP_PANIC);
  163. bust_spinlocks(0);
  164. debug_locks_off();
  165. console_flush_on_panic();
  166. }
  167. static unsigned long oops_begin(struct pt_regs *regs)
  168. {
  169. int cpu;
  170. unsigned long flags;
  171. oops_enter();
  172. /* racy, but better than risking deadlock. */
  173. raw_local_irq_save(flags);
  174. cpu = smp_processor_id();
  175. if (!arch_spin_trylock(&die_lock)) {
  176. if (cpu == die_owner)
  177. /* nested oops. should stop eventually */;
  178. else
  179. arch_spin_lock(&die_lock);
  180. }
  181. die_nest_count++;
  182. die_owner = cpu;
  183. console_verbose();
  184. bust_spinlocks(1);
  185. if (machine_is(powermac))
  186. pmac_backlight_unblank();
  187. return flags;
  188. }
  189. NOKPROBE_SYMBOL(oops_begin);
  190. static void oops_end(unsigned long flags, struct pt_regs *regs,
  191. int signr)
  192. {
  193. bust_spinlocks(0);
  194. add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
  195. die_nest_count--;
  196. oops_exit();
  197. printk("\n");
  198. if (!die_nest_count) {
  199. /* Nest count reaches zero, release the lock. */
  200. die_owner = -1;
  201. arch_spin_unlock(&die_lock);
  202. }
  203. raw_local_irq_restore(flags);
  204. /*
  205. * system_reset_excption handles debugger, crash dump, panic, for 0x100
  206. */
  207. if (TRAP(regs) == 0x100)
  208. return;
  209. crash_fadump(regs, "die oops");
  210. if (kexec_should_crash(current))
  211. crash_kexec(regs);
  212. if (!signr)
  213. return;
  214. /*
  215. * While our oops output is serialised by a spinlock, output
  216. * from panic() called below can race and corrupt it. If we
  217. * know we are going to panic, delay for 1 second so we have a
  218. * chance to get clean backtraces from all CPUs that are oopsing.
  219. */
  220. if (in_interrupt() || panic_on_oops || !current->pid ||
  221. is_global_init(current)) {
  222. mdelay(MSEC_PER_SEC);
  223. }
  224. if (in_interrupt())
  225. panic("Fatal exception in interrupt");
  226. if (panic_on_oops)
  227. panic("Fatal exception");
  228. do_exit(signr);
  229. }
  230. NOKPROBE_SYMBOL(oops_end);
  231. static int __die(const char *str, struct pt_regs *regs, long err)
  232. {
  233. printk("Oops: %s, sig: %ld [#%d]\n", str, err, ++die_counter);
  234. if (IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN))
  235. printk("LE ");
  236. else
  237. printk("BE ");
  238. if (IS_ENABLED(CONFIG_PREEMPT))
  239. pr_cont("PREEMPT ");
  240. if (IS_ENABLED(CONFIG_SMP))
  241. pr_cont("SMP NR_CPUS=%d ", NR_CPUS);
  242. if (debug_pagealloc_enabled())
  243. pr_cont("DEBUG_PAGEALLOC ");
  244. if (IS_ENABLED(CONFIG_NUMA))
  245. pr_cont("NUMA ");
  246. pr_cont("%s\n", ppc_md.name ? ppc_md.name : "");
  247. if (notify_die(DIE_OOPS, str, regs, err, 255, SIGSEGV) == NOTIFY_STOP)
  248. return 1;
  249. print_modules();
  250. show_regs(regs);
  251. return 0;
  252. }
  253. NOKPROBE_SYMBOL(__die);
  254. void die(const char *str, struct pt_regs *regs, long err)
  255. {
  256. unsigned long flags;
  257. /*
  258. * system_reset_excption handles debugger, crash dump, panic, for 0x100
  259. */
  260. if (TRAP(regs) != 0x100) {
  261. if (debugger(regs))
  262. return;
  263. }
  264. flags = oops_begin(regs);
  265. if (__die(str, regs, err))
  266. err = 0;
  267. oops_end(flags, regs, err);
  268. }
  269. NOKPROBE_SYMBOL(die);
  270. void user_single_step_siginfo(struct task_struct *tsk,
  271. struct pt_regs *regs, siginfo_t *info)
  272. {
  273. info->si_signo = SIGTRAP;
  274. info->si_code = TRAP_TRACE;
  275. info->si_addr = (void __user *)regs->nip;
  276. }
  277. static void show_signal_msg(int signr, struct pt_regs *regs, int code,
  278. unsigned long addr)
  279. {
  280. static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
  281. DEFAULT_RATELIMIT_BURST);
  282. if (!show_unhandled_signals)
  283. return;
  284. if (!unhandled_signal(current, signr))
  285. return;
  286. if (!__ratelimit(&rs))
  287. return;
  288. pr_info("%s[%d]: %s (%d) at %lx nip %lx lr %lx code %x",
  289. current->comm, current->pid, signame(signr), signr,
  290. addr, regs->nip, regs->link, code);
  291. print_vma_addr(KERN_CONT " in ", regs->nip);
  292. pr_cont("\n");
  293. show_user_instructions(regs);
  294. }
  295. void _exception_pkey(int signr, struct pt_regs *regs, int code,
  296. unsigned long addr, int key)
  297. {
  298. siginfo_t info;
  299. if (!user_mode(regs)) {
  300. die("Exception in kernel mode", regs, signr);
  301. return;
  302. }
  303. show_signal_msg(signr, regs, code, addr);
  304. if (arch_irqs_disabled() && !arch_irq_disabled_regs(regs))
  305. local_irq_enable();
  306. current->thread.trap_nr = code;
  307. /*
  308. * Save all the pkey registers AMR/IAMR/UAMOR. Eg: Core dumps need
  309. * to capture the content, if the task gets killed.
  310. */
  311. thread_pkey_regs_save(&current->thread);
  312. clear_siginfo(&info);
  313. info.si_signo = signr;
  314. info.si_code = code;
  315. info.si_addr = (void __user *) addr;
  316. info.si_pkey = key;
  317. force_sig_info(signr, &info, current);
  318. }
  319. void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr)
  320. {
  321. _exception_pkey(signr, regs, code, addr, 0);
  322. }
  323. void system_reset_exception(struct pt_regs *regs)
  324. {
  325. /*
  326. * Avoid crashes in case of nested NMI exceptions. Recoverability
  327. * is determined by RI and in_nmi
  328. */
  329. bool nested = in_nmi();
  330. if (!nested)
  331. nmi_enter();
  332. __this_cpu_inc(irq_stat.sreset_irqs);
  333. /* See if any machine dependent calls */
  334. if (ppc_md.system_reset_exception) {
  335. if (ppc_md.system_reset_exception(regs))
  336. goto out;
  337. }
  338. if (debugger(regs))
  339. goto out;
  340. kmsg_dump(KMSG_DUMP_OOPS);
  341. /*
  342. * A system reset is a request to dump, so we always send
  343. * it through the crashdump code (if fadump or kdump are
  344. * registered).
  345. */
  346. crash_fadump(regs, "System Reset");
  347. crash_kexec(regs);
  348. /*
  349. * We aren't the primary crash CPU. We need to send it
  350. * to a holding pattern to avoid it ending up in the panic
  351. * code.
  352. */
  353. crash_kexec_secondary(regs);
  354. /*
  355. * No debugger or crash dump registered, print logs then
  356. * panic.
  357. */
  358. die("System Reset", regs, SIGABRT);
  359. mdelay(2*MSEC_PER_SEC); /* Wait a little while for others to print */
  360. add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
  361. nmi_panic(regs, "System Reset");
  362. out:
  363. #ifdef CONFIG_PPC_BOOK3S_64
  364. BUG_ON(get_paca()->in_nmi == 0);
  365. if (get_paca()->in_nmi > 1)
  366. die("Unrecoverable nested System Reset", regs, SIGABRT);
  367. #endif
  368. /* Must die if the interrupt is not recoverable */
  369. if (!(regs->msr & MSR_RI)) {
  370. /* For the reason explained in die_mce, nmi_exit before die */
  371. nmi_exit();
  372. die("Unrecoverable System Reset", regs, SIGABRT);
  373. }
  374. if (!nested)
  375. nmi_exit();
  376. /* What should we do here? We could issue a shutdown or hard reset. */
  377. }
  378. /*
  379. * I/O accesses can cause machine checks on powermacs.
  380. * Check if the NIP corresponds to the address of a sync
  381. * instruction for which there is an entry in the exception
  382. * table.
  383. * Note that the 601 only takes a machine check on TEA
  384. * (transfer error ack) signal assertion, and does not
  385. * set any of the top 16 bits of SRR1.
  386. * -- paulus.
  387. */
  388. static inline int check_io_access(struct pt_regs *regs)
  389. {
  390. #ifdef CONFIG_PPC32
  391. unsigned long msr = regs->msr;
  392. const struct exception_table_entry *entry;
  393. unsigned int *nip = (unsigned int *)regs->nip;
  394. if (((msr & 0xffff0000) == 0 || (msr & (0x80000 | 0x40000)))
  395. && (entry = search_exception_tables(regs->nip)) != NULL) {
  396. /*
  397. * Check that it's a sync instruction, or somewhere
  398. * in the twi; isync; nop sequence that inb/inw/inl uses.
  399. * As the address is in the exception table
  400. * we should be able to read the instr there.
  401. * For the debug message, we look at the preceding
  402. * load or store.
  403. */
  404. if (*nip == PPC_INST_NOP)
  405. nip -= 2;
  406. else if (*nip == PPC_INST_ISYNC)
  407. --nip;
  408. if (*nip == PPC_INST_SYNC || (*nip >> 26) == OP_TRAP) {
  409. unsigned int rb;
  410. --nip;
  411. rb = (*nip >> 11) & 0x1f;
  412. printk(KERN_DEBUG "%s bad port %lx at %p\n",
  413. (*nip & 0x100)? "OUT to": "IN from",
  414. regs->gpr[rb] - _IO_BASE, nip);
  415. regs->msr |= MSR_RI;
  416. regs->nip = extable_fixup(entry);
  417. return 1;
  418. }
  419. }
  420. #endif /* CONFIG_PPC32 */
  421. return 0;
  422. }
  423. #ifdef CONFIG_PPC_ADV_DEBUG_REGS
  424. /* On 4xx, the reason for the machine check or program exception
  425. is in the ESR. */
  426. #define get_reason(regs) ((regs)->dsisr)
  427. #define REASON_FP ESR_FP
  428. #define REASON_ILLEGAL (ESR_PIL | ESR_PUO)
  429. #define REASON_PRIVILEGED ESR_PPR
  430. #define REASON_TRAP ESR_PTR
  431. /* single-step stuff */
  432. #define single_stepping(regs) (current->thread.debug.dbcr0 & DBCR0_IC)
  433. #define clear_single_step(regs) (current->thread.debug.dbcr0 &= ~DBCR0_IC)
  434. #define clear_br_trace(regs) do {} while(0)
  435. #else
  436. /* On non-4xx, the reason for the machine check or program
  437. exception is in the MSR. */
  438. #define get_reason(regs) ((regs)->msr)
  439. #define REASON_TM SRR1_PROGTM
  440. #define REASON_FP SRR1_PROGFPE
  441. #define REASON_ILLEGAL SRR1_PROGILL
  442. #define REASON_PRIVILEGED SRR1_PROGPRIV
  443. #define REASON_TRAP SRR1_PROGTRAP
  444. #define single_stepping(regs) ((regs)->msr & MSR_SE)
  445. #define clear_single_step(regs) ((regs)->msr &= ~MSR_SE)
  446. #define clear_br_trace(regs) ((regs)->msr &= ~MSR_BE)
  447. #endif
  448. #if defined(CONFIG_E500)
  449. int machine_check_e500mc(struct pt_regs *regs)
  450. {
  451. unsigned long mcsr = mfspr(SPRN_MCSR);
  452. unsigned long pvr = mfspr(SPRN_PVR);
  453. unsigned long reason = mcsr;
  454. int recoverable = 1;
  455. if (reason & MCSR_LD) {
  456. recoverable = fsl_rio_mcheck_exception(regs);
  457. if (recoverable == 1)
  458. goto silent_out;
  459. }
  460. printk("Machine check in kernel mode.\n");
  461. printk("Caused by (from MCSR=%lx): ", reason);
  462. if (reason & MCSR_MCP)
  463. printk("Machine Check Signal\n");
  464. if (reason & MCSR_ICPERR) {
  465. printk("Instruction Cache Parity Error\n");
  466. /*
  467. * This is recoverable by invalidating the i-cache.
  468. */
  469. mtspr(SPRN_L1CSR1, mfspr(SPRN_L1CSR1) | L1CSR1_ICFI);
  470. while (mfspr(SPRN_L1CSR1) & L1CSR1_ICFI)
  471. ;
  472. /*
  473. * This will generally be accompanied by an instruction
  474. * fetch error report -- only treat MCSR_IF as fatal
  475. * if it wasn't due to an L1 parity error.
  476. */
  477. reason &= ~MCSR_IF;
  478. }
  479. if (reason & MCSR_DCPERR_MC) {
  480. printk("Data Cache Parity Error\n");
  481. /*
  482. * In write shadow mode we auto-recover from the error, but it
  483. * may still get logged and cause a machine check. We should
  484. * only treat the non-write shadow case as non-recoverable.
  485. */
  486. /* On e6500 core, L1 DCWS (Data cache write shadow mode) bit
  487. * is not implemented but L1 data cache always runs in write
  488. * shadow mode. Hence on data cache parity errors HW will
  489. * automatically invalidate the L1 Data Cache.
  490. */
  491. if (PVR_VER(pvr) != PVR_VER_E6500) {
  492. if (!(mfspr(SPRN_L1CSR2) & L1CSR2_DCWS))
  493. recoverable = 0;
  494. }
  495. }
  496. if (reason & MCSR_L2MMU_MHIT) {
  497. printk("Hit on multiple TLB entries\n");
  498. recoverable = 0;
  499. }
  500. if (reason & MCSR_NMI)
  501. printk("Non-maskable interrupt\n");
  502. if (reason & MCSR_IF) {
  503. printk("Instruction Fetch Error Report\n");
  504. recoverable = 0;
  505. }
  506. if (reason & MCSR_LD) {
  507. printk("Load Error Report\n");
  508. recoverable = 0;
  509. }
  510. if (reason & MCSR_ST) {
  511. printk("Store Error Report\n");
  512. recoverable = 0;
  513. }
  514. if (reason & MCSR_LDG) {
  515. printk("Guarded Load Error Report\n");
  516. recoverable = 0;
  517. }
  518. if (reason & MCSR_TLBSYNC)
  519. printk("Simultaneous tlbsync operations\n");
  520. if (reason & MCSR_BSL2_ERR) {
  521. printk("Level 2 Cache Error\n");
  522. recoverable = 0;
  523. }
  524. if (reason & MCSR_MAV) {
  525. u64 addr;
  526. addr = mfspr(SPRN_MCAR);
  527. addr |= (u64)mfspr(SPRN_MCARU) << 32;
  528. printk("Machine Check %s Address: %#llx\n",
  529. reason & MCSR_MEA ? "Effective" : "Physical", addr);
  530. }
  531. silent_out:
  532. mtspr(SPRN_MCSR, mcsr);
  533. return mfspr(SPRN_MCSR) == 0 && recoverable;
  534. }
  535. int machine_check_e500(struct pt_regs *regs)
  536. {
  537. unsigned long reason = mfspr(SPRN_MCSR);
  538. if (reason & MCSR_BUS_RBERR) {
  539. if (fsl_rio_mcheck_exception(regs))
  540. return 1;
  541. if (fsl_pci_mcheck_exception(regs))
  542. return 1;
  543. }
  544. printk("Machine check in kernel mode.\n");
  545. printk("Caused by (from MCSR=%lx): ", reason);
  546. if (reason & MCSR_MCP)
  547. printk("Machine Check Signal\n");
  548. if (reason & MCSR_ICPERR)
  549. printk("Instruction Cache Parity Error\n");
  550. if (reason & MCSR_DCP_PERR)
  551. printk("Data Cache Push Parity Error\n");
  552. if (reason & MCSR_DCPERR)
  553. printk("Data Cache Parity Error\n");
  554. if (reason & MCSR_BUS_IAERR)
  555. printk("Bus - Instruction Address Error\n");
  556. if (reason & MCSR_BUS_RAERR)
  557. printk("Bus - Read Address Error\n");
  558. if (reason & MCSR_BUS_WAERR)
  559. printk("Bus - Write Address Error\n");
  560. if (reason & MCSR_BUS_IBERR)
  561. printk("Bus - Instruction Data Error\n");
  562. if (reason & MCSR_BUS_RBERR)
  563. printk("Bus - Read Data Bus Error\n");
  564. if (reason & MCSR_BUS_WBERR)
  565. printk("Bus - Write Data Bus Error\n");
  566. if (reason & MCSR_BUS_IPERR)
  567. printk("Bus - Instruction Parity Error\n");
  568. if (reason & MCSR_BUS_RPERR)
  569. printk("Bus - Read Parity Error\n");
  570. return 0;
  571. }
  572. int machine_check_generic(struct pt_regs *regs)
  573. {
  574. return 0;
  575. }
  576. #elif defined(CONFIG_E200)
  577. int machine_check_e200(struct pt_regs *regs)
  578. {
  579. unsigned long reason = mfspr(SPRN_MCSR);
  580. printk("Machine check in kernel mode.\n");
  581. printk("Caused by (from MCSR=%lx): ", reason);
  582. if (reason & MCSR_MCP)
  583. printk("Machine Check Signal\n");
  584. if (reason & MCSR_CP_PERR)
  585. printk("Cache Push Parity Error\n");
  586. if (reason & MCSR_CPERR)
  587. printk("Cache Parity Error\n");
  588. if (reason & MCSR_EXCP_ERR)
  589. printk("ISI, ITLB, or Bus Error on first instruction fetch for an exception handler\n");
  590. if (reason & MCSR_BUS_IRERR)
  591. printk("Bus - Read Bus Error on instruction fetch\n");
  592. if (reason & MCSR_BUS_DRERR)
  593. printk("Bus - Read Bus Error on data load\n");
  594. if (reason & MCSR_BUS_WRERR)
  595. printk("Bus - Write Bus Error on buffered store or cache line push\n");
  596. return 0;
  597. }
  598. #elif defined(CONFIG_PPC32)
  599. int machine_check_generic(struct pt_regs *regs)
  600. {
  601. unsigned long reason = regs->msr;
  602. printk("Machine check in kernel mode.\n");
  603. printk("Caused by (from SRR1=%lx): ", reason);
  604. switch (reason & 0x601F0000) {
  605. case 0x80000:
  606. printk("Machine check signal\n");
  607. break;
  608. case 0: /* for 601 */
  609. case 0x40000:
  610. case 0x140000: /* 7450 MSS error and TEA */
  611. printk("Transfer error ack signal\n");
  612. break;
  613. case 0x20000:
  614. printk("Data parity error signal\n");
  615. break;
  616. case 0x10000:
  617. printk("Address parity error signal\n");
  618. break;
  619. case 0x20000000:
  620. printk("L1 Data Cache error\n");
  621. break;
  622. case 0x40000000:
  623. printk("L1 Instruction Cache error\n");
  624. break;
  625. case 0x00100000:
  626. printk("L2 data cache parity error\n");
  627. break;
  628. default:
  629. printk("Unknown values in msr\n");
  630. }
  631. return 0;
  632. }
  633. #endif /* everything else */
  634. void machine_check_exception(struct pt_regs *regs)
  635. {
  636. int recover = 0;
  637. bool nested = in_nmi();
  638. if (!nested)
  639. nmi_enter();
  640. /* 64s accounts the mce in machine_check_early when in HVMODE */
  641. if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !cpu_has_feature(CPU_FTR_HVMODE))
  642. __this_cpu_inc(irq_stat.mce_exceptions);
  643. add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
  644. /* See if any machine dependent calls. In theory, we would want
  645. * to call the CPU first, and call the ppc_md. one if the CPU
  646. * one returns a positive number. However there is existing code
  647. * that assumes the board gets a first chance, so let's keep it
  648. * that way for now and fix things later. --BenH.
  649. */
  650. if (ppc_md.machine_check_exception)
  651. recover = ppc_md.machine_check_exception(regs);
  652. else if (cur_cpu_spec->machine_check)
  653. recover = cur_cpu_spec->machine_check(regs);
  654. if (recover > 0)
  655. goto bail;
  656. if (debugger_fault_handler(regs))
  657. goto bail;
  658. if (check_io_access(regs))
  659. goto bail;
  660. if (!nested)
  661. nmi_exit();
  662. die("Machine check", regs, SIGBUS);
  663. /* Must die if the interrupt is not recoverable */
  664. if (!(regs->msr & MSR_RI))
  665. die("Unrecoverable Machine check", regs, SIGBUS);
  666. return;
  667. bail:
  668. if (!nested)
  669. nmi_exit();
  670. }
  671. void SMIException(struct pt_regs *regs)
  672. {
  673. die("System Management Interrupt", regs, SIGABRT);
  674. }
  675. #ifdef CONFIG_VSX
  676. static void p9_hmi_special_emu(struct pt_regs *regs)
  677. {
  678. unsigned int ra, rb, t, i, sel, instr, rc;
  679. const void __user *addr;
  680. u8 vbuf[16] __aligned(16), *vdst;
  681. unsigned long ea, msr, msr_mask;
  682. bool swap;
  683. if (__get_user_inatomic(instr, (unsigned int __user *)regs->nip))
  684. return;
  685. /*
  686. * lxvb16x opcode: 0x7c0006d8
  687. * lxvd2x opcode: 0x7c000698
  688. * lxvh8x opcode: 0x7c000658
  689. * lxvw4x opcode: 0x7c000618
  690. */
  691. if ((instr & 0xfc00073e) != 0x7c000618) {
  692. pr_devel("HMI vec emu: not vector CI %i:%s[%d] nip=%016lx"
  693. " instr=%08x\n",
  694. smp_processor_id(), current->comm, current->pid,
  695. regs->nip, instr);
  696. return;
  697. }
  698. /* Grab vector registers into the task struct */
  699. msr = regs->msr; /* Grab msr before we flush the bits */
  700. flush_vsx_to_thread(current);
  701. enable_kernel_altivec();
  702. /*
  703. * Is userspace running with a different endian (this is rare but
  704. * not impossible)
  705. */
  706. swap = (msr & MSR_LE) != (MSR_KERNEL & MSR_LE);
  707. /* Decode the instruction */
  708. ra = (instr >> 16) & 0x1f;
  709. rb = (instr >> 11) & 0x1f;
  710. t = (instr >> 21) & 0x1f;
  711. if (instr & 1)
  712. vdst = (u8 *)&current->thread.vr_state.vr[t];
  713. else
  714. vdst = (u8 *)&current->thread.fp_state.fpr[t][0];
  715. /* Grab the vector address */
  716. ea = regs->gpr[rb] + (ra ? regs->gpr[ra] : 0);
  717. if (is_32bit_task())
  718. ea &= 0xfffffffful;
  719. addr = (__force const void __user *)ea;
  720. /* Check it */
  721. if (!access_ok(VERIFY_READ, addr, 16)) {
  722. pr_devel("HMI vec emu: bad access %i:%s[%d] nip=%016lx"
  723. " instr=%08x addr=%016lx\n",
  724. smp_processor_id(), current->comm, current->pid,
  725. regs->nip, instr, (unsigned long)addr);
  726. return;
  727. }
  728. /* Read the vector */
  729. rc = 0;
  730. if ((unsigned long)addr & 0xfUL)
  731. /* unaligned case */
  732. rc = __copy_from_user_inatomic(vbuf, addr, 16);
  733. else
  734. __get_user_atomic_128_aligned(vbuf, addr, rc);
  735. if (rc) {
  736. pr_devel("HMI vec emu: page fault %i:%s[%d] nip=%016lx"
  737. " instr=%08x addr=%016lx\n",
  738. smp_processor_id(), current->comm, current->pid,
  739. regs->nip, instr, (unsigned long)addr);
  740. return;
  741. }
  742. pr_devel("HMI vec emu: emulated vector CI %i:%s[%d] nip=%016lx"
  743. " instr=%08x addr=%016lx\n",
  744. smp_processor_id(), current->comm, current->pid, regs->nip,
  745. instr, (unsigned long) addr);
  746. /* Grab instruction "selector" */
  747. sel = (instr >> 6) & 3;
  748. /*
  749. * Check to make sure the facility is actually enabled. This
  750. * could happen if we get a false positive hit.
  751. *
  752. * lxvd2x/lxvw4x always check MSR VSX sel = 0,2
  753. * lxvh8x/lxvb16x check MSR VSX or VEC depending on VSR used sel = 1,3
  754. */
  755. msr_mask = MSR_VSX;
  756. if ((sel & 1) && (instr & 1)) /* lxvh8x & lxvb16x + VSR >= 32 */
  757. msr_mask = MSR_VEC;
  758. if (!(msr & msr_mask)) {
  759. pr_devel("HMI vec emu: MSR fac clear %i:%s[%d] nip=%016lx"
  760. " instr=%08x msr:%016lx\n",
  761. smp_processor_id(), current->comm, current->pid,
  762. regs->nip, instr, msr);
  763. return;
  764. }
  765. /* Do logging here before we modify sel based on endian */
  766. switch (sel) {
  767. case 0: /* lxvw4x */
  768. PPC_WARN_EMULATED(lxvw4x, regs);
  769. break;
  770. case 1: /* lxvh8x */
  771. PPC_WARN_EMULATED(lxvh8x, regs);
  772. break;
  773. case 2: /* lxvd2x */
  774. PPC_WARN_EMULATED(lxvd2x, regs);
  775. break;
  776. case 3: /* lxvb16x */
  777. PPC_WARN_EMULATED(lxvb16x, regs);
  778. break;
  779. }
  780. #ifdef __LITTLE_ENDIAN__
  781. /*
  782. * An LE kernel stores the vector in the task struct as an LE
  783. * byte array (effectively swapping both the components and
  784. * the content of the components). Those instructions expect
  785. * the components to remain in ascending address order, so we
  786. * swap them back.
  787. *
  788. * If we are running a BE user space, the expectation is that
  789. * of a simple memcpy, so forcing the emulation to look like
  790. * a lxvb16x should do the trick.
  791. */
  792. if (swap)
  793. sel = 3;
  794. switch (sel) {
  795. case 0: /* lxvw4x */
  796. for (i = 0; i < 4; i++)
  797. ((u32 *)vdst)[i] = ((u32 *)vbuf)[3-i];
  798. break;
  799. case 1: /* lxvh8x */
  800. for (i = 0; i < 8; i++)
  801. ((u16 *)vdst)[i] = ((u16 *)vbuf)[7-i];
  802. break;
  803. case 2: /* lxvd2x */
  804. for (i = 0; i < 2; i++)
  805. ((u64 *)vdst)[i] = ((u64 *)vbuf)[1-i];
  806. break;
  807. case 3: /* lxvb16x */
  808. for (i = 0; i < 16; i++)
  809. vdst[i] = vbuf[15-i];
  810. break;
  811. }
  812. #else /* __LITTLE_ENDIAN__ */
  813. /* On a big endian kernel, a BE userspace only needs a memcpy */
  814. if (!swap)
  815. sel = 3;
  816. /* Otherwise, we need to swap the content of the components */
  817. switch (sel) {
  818. case 0: /* lxvw4x */
  819. for (i = 0; i < 4; i++)
  820. ((u32 *)vdst)[i] = cpu_to_le32(((u32 *)vbuf)[i]);
  821. break;
  822. case 1: /* lxvh8x */
  823. for (i = 0; i < 8; i++)
  824. ((u16 *)vdst)[i] = cpu_to_le16(((u16 *)vbuf)[i]);
  825. break;
  826. case 2: /* lxvd2x */
  827. for (i = 0; i < 2; i++)
  828. ((u64 *)vdst)[i] = cpu_to_le64(((u64 *)vbuf)[i]);
  829. break;
  830. case 3: /* lxvb16x */
  831. memcpy(vdst, vbuf, 16);
  832. break;
  833. }
  834. #endif /* !__LITTLE_ENDIAN__ */
  835. /* Go to next instruction */
  836. regs->nip += 4;
  837. }
  838. #endif /* CONFIG_VSX */
  839. void handle_hmi_exception(struct pt_regs *regs)
  840. {
  841. struct pt_regs *old_regs;
  842. old_regs = set_irq_regs(regs);
  843. irq_enter();
  844. #ifdef CONFIG_VSX
  845. /* Real mode flagged P9 special emu is needed */
  846. if (local_paca->hmi_p9_special_emu) {
  847. local_paca->hmi_p9_special_emu = 0;
  848. /*
  849. * We don't want to take page faults while doing the
  850. * emulation, we just replay the instruction if necessary.
  851. */
  852. pagefault_disable();
  853. p9_hmi_special_emu(regs);
  854. pagefault_enable();
  855. }
  856. #endif /* CONFIG_VSX */
  857. if (ppc_md.handle_hmi_exception)
  858. ppc_md.handle_hmi_exception(regs);
  859. irq_exit();
  860. set_irq_regs(old_regs);
  861. }
  862. void unknown_exception(struct pt_regs *regs)
  863. {
  864. enum ctx_state prev_state = exception_enter();
  865. printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
  866. regs->nip, regs->msr, regs->trap);
  867. _exception(SIGTRAP, regs, TRAP_UNK, 0);
  868. exception_exit(prev_state);
  869. }
  870. void instruction_breakpoint_exception(struct pt_regs *regs)
  871. {
  872. enum ctx_state prev_state = exception_enter();
  873. if (notify_die(DIE_IABR_MATCH, "iabr_match", regs, 5,
  874. 5, SIGTRAP) == NOTIFY_STOP)
  875. goto bail;
  876. if (debugger_iabr_match(regs))
  877. goto bail;
  878. _exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip);
  879. bail:
  880. exception_exit(prev_state);
  881. }
  882. void RunModeException(struct pt_regs *regs)
  883. {
  884. _exception(SIGTRAP, regs, TRAP_UNK, 0);
  885. }
  886. void single_step_exception(struct pt_regs *regs)
  887. {
  888. enum ctx_state prev_state = exception_enter();
  889. clear_single_step(regs);
  890. clear_br_trace(regs);
  891. if (kprobe_post_handler(regs))
  892. return;
  893. if (notify_die(DIE_SSTEP, "single_step", regs, 5,
  894. 5, SIGTRAP) == NOTIFY_STOP)
  895. goto bail;
  896. if (debugger_sstep(regs))
  897. goto bail;
  898. _exception(SIGTRAP, regs, TRAP_TRACE, regs->nip);
  899. bail:
  900. exception_exit(prev_state);
  901. }
  902. NOKPROBE_SYMBOL(single_step_exception);
  903. /*
  904. * After we have successfully emulated an instruction, we have to
  905. * check if the instruction was being single-stepped, and if so,
  906. * pretend we got a single-step exception. This was pointed out
  907. * by Kumar Gala. -- paulus
  908. */
  909. static void emulate_single_step(struct pt_regs *regs)
  910. {
  911. if (single_stepping(regs))
  912. single_step_exception(regs);
  913. }
  914. static inline int __parse_fpscr(unsigned long fpscr)
  915. {
  916. int ret = FPE_FLTUNK;
  917. /* Invalid operation */
  918. if ((fpscr & FPSCR_VE) && (fpscr & FPSCR_VX))
  919. ret = FPE_FLTINV;
  920. /* Overflow */
  921. else if ((fpscr & FPSCR_OE) && (fpscr & FPSCR_OX))
  922. ret = FPE_FLTOVF;
  923. /* Underflow */
  924. else if ((fpscr & FPSCR_UE) && (fpscr & FPSCR_UX))
  925. ret = FPE_FLTUND;
  926. /* Divide by zero */
  927. else if ((fpscr & FPSCR_ZE) && (fpscr & FPSCR_ZX))
  928. ret = FPE_FLTDIV;
  929. /* Inexact result */
  930. else if ((fpscr & FPSCR_XE) && (fpscr & FPSCR_XX))
  931. ret = FPE_FLTRES;
  932. return ret;
  933. }
  934. static void parse_fpe(struct pt_regs *regs)
  935. {
  936. int code = 0;
  937. flush_fp_to_thread(current);
  938. code = __parse_fpscr(current->thread.fp_state.fpscr);
  939. _exception(SIGFPE, regs, code, regs->nip);
  940. }
  941. /*
  942. * Illegal instruction emulation support. Originally written to
  943. * provide the PVR to user applications using the mfspr rd, PVR.
  944. * Return non-zero if we can't emulate, or -EFAULT if the associated
  945. * memory access caused an access fault. Return zero on success.
  946. *
  947. * There are a couple of ways to do this, either "decode" the instruction
  948. * or directly match lots of bits. In this case, matching lots of
  949. * bits is faster and easier.
  950. *
  951. */
  952. static int emulate_string_inst(struct pt_regs *regs, u32 instword)
  953. {
  954. u8 rT = (instword >> 21) & 0x1f;
  955. u8 rA = (instword >> 16) & 0x1f;
  956. u8 NB_RB = (instword >> 11) & 0x1f;
  957. u32 num_bytes;
  958. unsigned long EA;
  959. int pos = 0;
  960. /* Early out if we are an invalid form of lswx */
  961. if ((instword & PPC_INST_STRING_MASK) == PPC_INST_LSWX)
  962. if ((rT == rA) || (rT == NB_RB))
  963. return -EINVAL;
  964. EA = (rA == 0) ? 0 : regs->gpr[rA];
  965. switch (instword & PPC_INST_STRING_MASK) {
  966. case PPC_INST_LSWX:
  967. case PPC_INST_STSWX:
  968. EA += NB_RB;
  969. num_bytes = regs->xer & 0x7f;
  970. break;
  971. case PPC_INST_LSWI:
  972. case PPC_INST_STSWI:
  973. num_bytes = (NB_RB == 0) ? 32 : NB_RB;
  974. break;
  975. default:
  976. return -EINVAL;
  977. }
  978. while (num_bytes != 0)
  979. {
  980. u8 val;
  981. u32 shift = 8 * (3 - (pos & 0x3));
  982. /* if process is 32-bit, clear upper 32 bits of EA */
  983. if ((regs->msr & MSR_64BIT) == 0)
  984. EA &= 0xFFFFFFFF;
  985. switch ((instword & PPC_INST_STRING_MASK)) {
  986. case PPC_INST_LSWX:
  987. case PPC_INST_LSWI:
  988. if (get_user(val, (u8 __user *)EA))
  989. return -EFAULT;
  990. /* first time updating this reg,
  991. * zero it out */
  992. if (pos == 0)
  993. regs->gpr[rT] = 0;
  994. regs->gpr[rT] |= val << shift;
  995. break;
  996. case PPC_INST_STSWI:
  997. case PPC_INST_STSWX:
  998. val = regs->gpr[rT] >> shift;
  999. if (put_user(val, (u8 __user *)EA))
  1000. return -EFAULT;
  1001. break;
  1002. }
  1003. /* move EA to next address */
  1004. EA += 1;
  1005. num_bytes--;
  1006. /* manage our position within the register */
  1007. if (++pos == 4) {
  1008. pos = 0;
  1009. if (++rT == 32)
  1010. rT = 0;
  1011. }
  1012. }
  1013. return 0;
  1014. }
  1015. static int emulate_popcntb_inst(struct pt_regs *regs, u32 instword)
  1016. {
  1017. u32 ra,rs;
  1018. unsigned long tmp;
  1019. ra = (instword >> 16) & 0x1f;
  1020. rs = (instword >> 21) & 0x1f;
  1021. tmp = regs->gpr[rs];
  1022. tmp = tmp - ((tmp >> 1) & 0x5555555555555555ULL);
  1023. tmp = (tmp & 0x3333333333333333ULL) + ((tmp >> 2) & 0x3333333333333333ULL);
  1024. tmp = (tmp + (tmp >> 4)) & 0x0f0f0f0f0f0f0f0fULL;
  1025. regs->gpr[ra] = tmp;
  1026. return 0;
  1027. }
  1028. static int emulate_isel(struct pt_regs *regs, u32 instword)
  1029. {
  1030. u8 rT = (instword >> 21) & 0x1f;
  1031. u8 rA = (instword >> 16) & 0x1f;
  1032. u8 rB = (instword >> 11) & 0x1f;
  1033. u8 BC = (instword >> 6) & 0x1f;
  1034. u8 bit;
  1035. unsigned long tmp;
  1036. tmp = (rA == 0) ? 0 : regs->gpr[rA];
  1037. bit = (regs->ccr >> (31 - BC)) & 0x1;
  1038. regs->gpr[rT] = bit ? tmp : regs->gpr[rB];
  1039. return 0;
  1040. }
  1041. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1042. static inline bool tm_abort_check(struct pt_regs *regs, int cause)
  1043. {
  1044. /* If we're emulating a load/store in an active transaction, we cannot
  1045. * emulate it as the kernel operates in transaction suspended context.
  1046. * We need to abort the transaction. This creates a persistent TM
  1047. * abort so tell the user what caused it with a new code.
  1048. */
  1049. if (MSR_TM_TRANSACTIONAL(regs->msr)) {
  1050. tm_enable();
  1051. tm_abort(cause);
  1052. return true;
  1053. }
  1054. return false;
  1055. }
  1056. #else
  1057. static inline bool tm_abort_check(struct pt_regs *regs, int reason)
  1058. {
  1059. return false;
  1060. }
  1061. #endif
  1062. static int emulate_instruction(struct pt_regs *regs)
  1063. {
  1064. u32 instword;
  1065. u32 rd;
  1066. if (!user_mode(regs))
  1067. return -EINVAL;
  1068. CHECK_FULL_REGS(regs);
  1069. if (get_user(instword, (u32 __user *)(regs->nip)))
  1070. return -EFAULT;
  1071. /* Emulate the mfspr rD, PVR. */
  1072. if ((instword & PPC_INST_MFSPR_PVR_MASK) == PPC_INST_MFSPR_PVR) {
  1073. PPC_WARN_EMULATED(mfpvr, regs);
  1074. rd = (instword >> 21) & 0x1f;
  1075. regs->gpr[rd] = mfspr(SPRN_PVR);
  1076. return 0;
  1077. }
  1078. /* Emulating the dcba insn is just a no-op. */
  1079. if ((instword & PPC_INST_DCBA_MASK) == PPC_INST_DCBA) {
  1080. PPC_WARN_EMULATED(dcba, regs);
  1081. return 0;
  1082. }
  1083. /* Emulate the mcrxr insn. */
  1084. if ((instword & PPC_INST_MCRXR_MASK) == PPC_INST_MCRXR) {
  1085. int shift = (instword >> 21) & 0x1c;
  1086. unsigned long msk = 0xf0000000UL >> shift;
  1087. PPC_WARN_EMULATED(mcrxr, regs);
  1088. regs->ccr = (regs->ccr & ~msk) | ((regs->xer >> shift) & msk);
  1089. regs->xer &= ~0xf0000000UL;
  1090. return 0;
  1091. }
  1092. /* Emulate load/store string insn. */
  1093. if ((instword & PPC_INST_STRING_GEN_MASK) == PPC_INST_STRING) {
  1094. if (tm_abort_check(regs,
  1095. TM_CAUSE_EMULATE | TM_CAUSE_PERSISTENT))
  1096. return -EINVAL;
  1097. PPC_WARN_EMULATED(string, regs);
  1098. return emulate_string_inst(regs, instword);
  1099. }
  1100. /* Emulate the popcntb (Population Count Bytes) instruction. */
  1101. if ((instword & PPC_INST_POPCNTB_MASK) == PPC_INST_POPCNTB) {
  1102. PPC_WARN_EMULATED(popcntb, regs);
  1103. return emulate_popcntb_inst(regs, instword);
  1104. }
  1105. /* Emulate isel (Integer Select) instruction */
  1106. if ((instword & PPC_INST_ISEL_MASK) == PPC_INST_ISEL) {
  1107. PPC_WARN_EMULATED(isel, regs);
  1108. return emulate_isel(regs, instword);
  1109. }
  1110. /* Emulate sync instruction variants */
  1111. if ((instword & PPC_INST_SYNC_MASK) == PPC_INST_SYNC) {
  1112. PPC_WARN_EMULATED(sync, regs);
  1113. asm volatile("sync");
  1114. return 0;
  1115. }
  1116. #ifdef CONFIG_PPC64
  1117. /* Emulate the mfspr rD, DSCR. */
  1118. if ((((instword & PPC_INST_MFSPR_DSCR_USER_MASK) ==
  1119. PPC_INST_MFSPR_DSCR_USER) ||
  1120. ((instword & PPC_INST_MFSPR_DSCR_MASK) ==
  1121. PPC_INST_MFSPR_DSCR)) &&
  1122. cpu_has_feature(CPU_FTR_DSCR)) {
  1123. PPC_WARN_EMULATED(mfdscr, regs);
  1124. rd = (instword >> 21) & 0x1f;
  1125. regs->gpr[rd] = mfspr(SPRN_DSCR);
  1126. return 0;
  1127. }
  1128. /* Emulate the mtspr DSCR, rD. */
  1129. if ((((instword & PPC_INST_MTSPR_DSCR_USER_MASK) ==
  1130. PPC_INST_MTSPR_DSCR_USER) ||
  1131. ((instword & PPC_INST_MTSPR_DSCR_MASK) ==
  1132. PPC_INST_MTSPR_DSCR)) &&
  1133. cpu_has_feature(CPU_FTR_DSCR)) {
  1134. PPC_WARN_EMULATED(mtdscr, regs);
  1135. rd = (instword >> 21) & 0x1f;
  1136. current->thread.dscr = regs->gpr[rd];
  1137. current->thread.dscr_inherit = 1;
  1138. mtspr(SPRN_DSCR, current->thread.dscr);
  1139. return 0;
  1140. }
  1141. #endif
  1142. return -EINVAL;
  1143. }
  1144. int is_valid_bugaddr(unsigned long addr)
  1145. {
  1146. return is_kernel_addr(addr);
  1147. }
  1148. #ifdef CONFIG_MATH_EMULATION
  1149. static int emulate_math(struct pt_regs *regs)
  1150. {
  1151. int ret;
  1152. extern int do_mathemu(struct pt_regs *regs);
  1153. ret = do_mathemu(regs);
  1154. if (ret >= 0)
  1155. PPC_WARN_EMULATED(math, regs);
  1156. switch (ret) {
  1157. case 0:
  1158. emulate_single_step(regs);
  1159. return 0;
  1160. case 1: {
  1161. int code = 0;
  1162. code = __parse_fpscr(current->thread.fp_state.fpscr);
  1163. _exception(SIGFPE, regs, code, regs->nip);
  1164. return 0;
  1165. }
  1166. case -EFAULT:
  1167. _exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip);
  1168. return 0;
  1169. }
  1170. return -1;
  1171. }
  1172. #else
  1173. static inline int emulate_math(struct pt_regs *regs) { return -1; }
  1174. #endif
  1175. void program_check_exception(struct pt_regs *regs)
  1176. {
  1177. enum ctx_state prev_state = exception_enter();
  1178. unsigned int reason = get_reason(regs);
  1179. /* We can now get here via a FP Unavailable exception if the core
  1180. * has no FPU, in that case the reason flags will be 0 */
  1181. if (reason & REASON_FP) {
  1182. /* IEEE FP exception */
  1183. parse_fpe(regs);
  1184. goto bail;
  1185. }
  1186. if (reason & REASON_TRAP) {
  1187. unsigned long bugaddr;
  1188. /* Debugger is first in line to stop recursive faults in
  1189. * rcu_lock, notify_die, or atomic_notifier_call_chain */
  1190. if (debugger_bpt(regs))
  1191. goto bail;
  1192. if (kprobe_handler(regs))
  1193. goto bail;
  1194. /* trap exception */
  1195. if (notify_die(DIE_BPT, "breakpoint", regs, 5, 5, SIGTRAP)
  1196. == NOTIFY_STOP)
  1197. goto bail;
  1198. bugaddr = regs->nip;
  1199. /*
  1200. * Fixup bugaddr for BUG_ON() in real mode
  1201. */
  1202. if (!is_kernel_addr(bugaddr) && !(regs->msr & MSR_IR))
  1203. bugaddr += PAGE_OFFSET;
  1204. if (!(regs->msr & MSR_PR) && /* not user-mode */
  1205. report_bug(bugaddr, regs) == BUG_TRAP_TYPE_WARN) {
  1206. regs->nip += 4;
  1207. goto bail;
  1208. }
  1209. _exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip);
  1210. goto bail;
  1211. }
  1212. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1213. if (reason & REASON_TM) {
  1214. /* This is a TM "Bad Thing Exception" program check.
  1215. * This occurs when:
  1216. * - An rfid/hrfid/mtmsrd attempts to cause an illegal
  1217. * transition in TM states.
  1218. * - A trechkpt is attempted when transactional.
  1219. * - A treclaim is attempted when non transactional.
  1220. * - A tend is illegally attempted.
  1221. * - writing a TM SPR when transactional.
  1222. *
  1223. * If usermode caused this, it's done something illegal and
  1224. * gets a SIGILL slap on the wrist. We call it an illegal
  1225. * operand to distinguish from the instruction just being bad
  1226. * (e.g. executing a 'tend' on a CPU without TM!); it's an
  1227. * illegal /placement/ of a valid instruction.
  1228. */
  1229. if (user_mode(regs)) {
  1230. _exception(SIGILL, regs, ILL_ILLOPN, regs->nip);
  1231. goto bail;
  1232. } else {
  1233. printk(KERN_EMERG "Unexpected TM Bad Thing exception "
  1234. "at %lx (msr 0x%x)\n", regs->nip, reason);
  1235. die("Unrecoverable exception", regs, SIGABRT);
  1236. }
  1237. }
  1238. #endif
  1239. /*
  1240. * If we took the program check in the kernel skip down to sending a
  1241. * SIGILL. The subsequent cases all relate to emulating instructions
  1242. * which we should only do for userspace. We also do not want to enable
  1243. * interrupts for kernel faults because that might lead to further
  1244. * faults, and loose the context of the original exception.
  1245. */
  1246. if (!user_mode(regs))
  1247. goto sigill;
  1248. /* We restore the interrupt state now */
  1249. if (!arch_irq_disabled_regs(regs))
  1250. local_irq_enable();
  1251. /* (reason & REASON_ILLEGAL) would be the obvious thing here,
  1252. * but there seems to be a hardware bug on the 405GP (RevD)
  1253. * that means ESR is sometimes set incorrectly - either to
  1254. * ESR_DST (!?) or 0. In the process of chasing this with the
  1255. * hardware people - not sure if it can happen on any illegal
  1256. * instruction or only on FP instructions, whether there is a
  1257. * pattern to occurrences etc. -dgibson 31/Mar/2003
  1258. */
  1259. if (!emulate_math(regs))
  1260. goto bail;
  1261. /* Try to emulate it if we should. */
  1262. if (reason & (REASON_ILLEGAL | REASON_PRIVILEGED)) {
  1263. switch (emulate_instruction(regs)) {
  1264. case 0:
  1265. regs->nip += 4;
  1266. emulate_single_step(regs);
  1267. goto bail;
  1268. case -EFAULT:
  1269. _exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip);
  1270. goto bail;
  1271. }
  1272. }
  1273. sigill:
  1274. if (reason & REASON_PRIVILEGED)
  1275. _exception(SIGILL, regs, ILL_PRVOPC, regs->nip);
  1276. else
  1277. _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
  1278. bail:
  1279. exception_exit(prev_state);
  1280. }
  1281. NOKPROBE_SYMBOL(program_check_exception);
  1282. /*
  1283. * This occurs when running in hypervisor mode on POWER6 or later
  1284. * and an illegal instruction is encountered.
  1285. */
  1286. void emulation_assist_interrupt(struct pt_regs *regs)
  1287. {
  1288. regs->msr |= REASON_ILLEGAL;
  1289. program_check_exception(regs);
  1290. }
  1291. NOKPROBE_SYMBOL(emulation_assist_interrupt);
  1292. void alignment_exception(struct pt_regs *regs)
  1293. {
  1294. enum ctx_state prev_state = exception_enter();
  1295. int sig, code, fixed = 0;
  1296. /* We restore the interrupt state now */
  1297. if (!arch_irq_disabled_regs(regs))
  1298. local_irq_enable();
  1299. if (tm_abort_check(regs, TM_CAUSE_ALIGNMENT | TM_CAUSE_PERSISTENT))
  1300. goto bail;
  1301. /* we don't implement logging of alignment exceptions */
  1302. if (!(current->thread.align_ctl & PR_UNALIGN_SIGBUS))
  1303. fixed = fix_alignment(regs);
  1304. if (fixed == 1) {
  1305. regs->nip += 4; /* skip over emulated instruction */
  1306. emulate_single_step(regs);
  1307. goto bail;
  1308. }
  1309. /* Operand address was bad */
  1310. if (fixed == -EFAULT) {
  1311. sig = SIGSEGV;
  1312. code = SEGV_ACCERR;
  1313. } else {
  1314. sig = SIGBUS;
  1315. code = BUS_ADRALN;
  1316. }
  1317. if (user_mode(regs))
  1318. _exception(sig, regs, code, regs->dar);
  1319. else
  1320. bad_page_fault(regs, regs->dar, sig);
  1321. bail:
  1322. exception_exit(prev_state);
  1323. }
  1324. void StackOverflow(struct pt_regs *regs)
  1325. {
  1326. pr_crit("Kernel stack overflow in process %s[%d], r1=%lx\n",
  1327. current->comm, task_pid_nr(current), regs->gpr[1]);
  1328. debugger(regs);
  1329. show_regs(regs);
  1330. panic("kernel stack overflow");
  1331. }
  1332. void nonrecoverable_exception(struct pt_regs *regs)
  1333. {
  1334. printk(KERN_ERR "Non-recoverable exception at PC=%lx MSR=%lx\n",
  1335. regs->nip, regs->msr);
  1336. debugger(regs);
  1337. die("nonrecoverable exception", regs, SIGKILL);
  1338. }
  1339. void kernel_fp_unavailable_exception(struct pt_regs *regs)
  1340. {
  1341. enum ctx_state prev_state = exception_enter();
  1342. printk(KERN_EMERG "Unrecoverable FP Unavailable Exception "
  1343. "%lx at %lx\n", regs->trap, regs->nip);
  1344. die("Unrecoverable FP Unavailable Exception", regs, SIGABRT);
  1345. exception_exit(prev_state);
  1346. }
  1347. void altivec_unavailable_exception(struct pt_regs *regs)
  1348. {
  1349. enum ctx_state prev_state = exception_enter();
  1350. if (user_mode(regs)) {
  1351. /* A user program has executed an altivec instruction,
  1352. but this kernel doesn't support altivec. */
  1353. _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
  1354. goto bail;
  1355. }
  1356. printk(KERN_EMERG "Unrecoverable VMX/Altivec Unavailable Exception "
  1357. "%lx at %lx\n", regs->trap, regs->nip);
  1358. die("Unrecoverable VMX/Altivec Unavailable Exception", regs, SIGABRT);
  1359. bail:
  1360. exception_exit(prev_state);
  1361. }
  1362. void vsx_unavailable_exception(struct pt_regs *regs)
  1363. {
  1364. if (user_mode(regs)) {
  1365. /* A user program has executed an vsx instruction,
  1366. but this kernel doesn't support vsx. */
  1367. _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
  1368. return;
  1369. }
  1370. printk(KERN_EMERG "Unrecoverable VSX Unavailable Exception "
  1371. "%lx at %lx\n", regs->trap, regs->nip);
  1372. die("Unrecoverable VSX Unavailable Exception", regs, SIGABRT);
  1373. }
  1374. #ifdef CONFIG_PPC64
  1375. static void tm_unavailable(struct pt_regs *regs)
  1376. {
  1377. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1378. if (user_mode(regs)) {
  1379. current->thread.load_tm++;
  1380. regs->msr |= MSR_TM;
  1381. tm_enable();
  1382. tm_restore_sprs(&current->thread);
  1383. return;
  1384. }
  1385. #endif
  1386. pr_emerg("Unrecoverable TM Unavailable Exception "
  1387. "%lx at %lx\n", regs->trap, regs->nip);
  1388. die("Unrecoverable TM Unavailable Exception", regs, SIGABRT);
  1389. }
  1390. void facility_unavailable_exception(struct pt_regs *regs)
  1391. {
  1392. static char *facility_strings[] = {
  1393. [FSCR_FP_LG] = "FPU",
  1394. [FSCR_VECVSX_LG] = "VMX/VSX",
  1395. [FSCR_DSCR_LG] = "DSCR",
  1396. [FSCR_PM_LG] = "PMU SPRs",
  1397. [FSCR_BHRB_LG] = "BHRB",
  1398. [FSCR_TM_LG] = "TM",
  1399. [FSCR_EBB_LG] = "EBB",
  1400. [FSCR_TAR_LG] = "TAR",
  1401. [FSCR_MSGP_LG] = "MSGP",
  1402. [FSCR_SCV_LG] = "SCV",
  1403. };
  1404. char *facility = "unknown";
  1405. u64 value;
  1406. u32 instword, rd;
  1407. u8 status;
  1408. bool hv;
  1409. hv = (TRAP(regs) == 0xf80);
  1410. if (hv)
  1411. value = mfspr(SPRN_HFSCR);
  1412. else
  1413. value = mfspr(SPRN_FSCR);
  1414. status = value >> 56;
  1415. if ((hv || status >= 2) &&
  1416. (status < ARRAY_SIZE(facility_strings)) &&
  1417. facility_strings[status])
  1418. facility = facility_strings[status];
  1419. /* We should not have taken this interrupt in kernel */
  1420. if (!user_mode(regs)) {
  1421. pr_emerg("Facility '%s' unavailable (%d) exception in kernel mode at %lx\n",
  1422. facility, status, regs->nip);
  1423. die("Unexpected facility unavailable exception", regs, SIGABRT);
  1424. }
  1425. /* We restore the interrupt state now */
  1426. if (!arch_irq_disabled_regs(regs))
  1427. local_irq_enable();
  1428. if (status == FSCR_DSCR_LG) {
  1429. /*
  1430. * User is accessing the DSCR register using the problem
  1431. * state only SPR number (0x03) either through a mfspr or
  1432. * a mtspr instruction. If it is a write attempt through
  1433. * a mtspr, then we set the inherit bit. This also allows
  1434. * the user to write or read the register directly in the
  1435. * future by setting via the FSCR DSCR bit. But in case it
  1436. * is a read DSCR attempt through a mfspr instruction, we
  1437. * just emulate the instruction instead. This code path will
  1438. * always emulate all the mfspr instructions till the user
  1439. * has attempted at least one mtspr instruction. This way it
  1440. * preserves the same behaviour when the user is accessing
  1441. * the DSCR through privilege level only SPR number (0x11)
  1442. * which is emulated through illegal instruction exception.
  1443. * We always leave HFSCR DSCR set.
  1444. */
  1445. if (get_user(instword, (u32 __user *)(regs->nip))) {
  1446. pr_err("Failed to fetch the user instruction\n");
  1447. return;
  1448. }
  1449. /* Write into DSCR (mtspr 0x03, RS) */
  1450. if ((instword & PPC_INST_MTSPR_DSCR_USER_MASK)
  1451. == PPC_INST_MTSPR_DSCR_USER) {
  1452. rd = (instword >> 21) & 0x1f;
  1453. current->thread.dscr = regs->gpr[rd];
  1454. current->thread.dscr_inherit = 1;
  1455. current->thread.fscr |= FSCR_DSCR;
  1456. mtspr(SPRN_FSCR, current->thread.fscr);
  1457. }
  1458. /* Read from DSCR (mfspr RT, 0x03) */
  1459. if ((instword & PPC_INST_MFSPR_DSCR_USER_MASK)
  1460. == PPC_INST_MFSPR_DSCR_USER) {
  1461. if (emulate_instruction(regs)) {
  1462. pr_err("DSCR based mfspr emulation failed\n");
  1463. return;
  1464. }
  1465. regs->nip += 4;
  1466. emulate_single_step(regs);
  1467. }
  1468. return;
  1469. }
  1470. if (status == FSCR_TM_LG) {
  1471. /*
  1472. * If we're here then the hardware is TM aware because it
  1473. * generated an exception with FSRM_TM set.
  1474. *
  1475. * If cpu_has_feature(CPU_FTR_TM) is false, then either firmware
  1476. * told us not to do TM, or the kernel is not built with TM
  1477. * support.
  1478. *
  1479. * If both of those things are true, then userspace can spam the
  1480. * console by triggering the printk() below just by continually
  1481. * doing tbegin (or any TM instruction). So in that case just
  1482. * send the process a SIGILL immediately.
  1483. */
  1484. if (!cpu_has_feature(CPU_FTR_TM))
  1485. goto out;
  1486. tm_unavailable(regs);
  1487. return;
  1488. }
  1489. pr_err_ratelimited("%sFacility '%s' unavailable (%d), exception at 0x%lx, MSR=%lx\n",
  1490. hv ? "Hypervisor " : "", facility, status, regs->nip, regs->msr);
  1491. out:
  1492. _exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
  1493. }
  1494. #endif
  1495. #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  1496. void fp_unavailable_tm(struct pt_regs *regs)
  1497. {
  1498. /* Note: This does not handle any kind of FP laziness. */
  1499. TM_DEBUG("FP Unavailable trap whilst transactional at 0x%lx, MSR=%lx\n",
  1500. regs->nip, regs->msr);
  1501. /* We can only have got here if the task started using FP after
  1502. * beginning the transaction. So, the transactional regs are just a
  1503. * copy of the checkpointed ones. But, we still need to recheckpoint
  1504. * as we're enabling FP for the process; it will return, abort the
  1505. * transaction, and probably retry but now with FP enabled. So the
  1506. * checkpointed FP registers need to be loaded.
  1507. */
  1508. tm_reclaim_current(TM_CAUSE_FAC_UNAV);
  1509. /* Reclaim didn't save out any FPRs to transact_fprs. */
  1510. /* Enable FP for the task: */
  1511. current->thread.load_fp = 1;
  1512. /* This loads and recheckpoints the FP registers from
  1513. * thread.fpr[]. They will remain in registers after the
  1514. * checkpoint so we don't need to reload them after.
  1515. * If VMX is in use, the VRs now hold checkpointed values,
  1516. * so we don't want to load the VRs from the thread_struct.
  1517. */
  1518. tm_recheckpoint(&current->thread);
  1519. }
  1520. void altivec_unavailable_tm(struct pt_regs *regs)
  1521. {
  1522. /* See the comments in fp_unavailable_tm(). This function operates
  1523. * the same way.
  1524. */
  1525. TM_DEBUG("Vector Unavailable trap whilst transactional at 0x%lx,"
  1526. "MSR=%lx\n",
  1527. regs->nip, regs->msr);
  1528. tm_reclaim_current(TM_CAUSE_FAC_UNAV);
  1529. current->thread.load_vec = 1;
  1530. tm_recheckpoint(&current->thread);
  1531. current->thread.used_vr = 1;
  1532. }
  1533. void vsx_unavailable_tm(struct pt_regs *regs)
  1534. {
  1535. /* See the comments in fp_unavailable_tm(). This works similarly,
  1536. * though we're loading both FP and VEC registers in here.
  1537. *
  1538. * If FP isn't in use, load FP regs. If VEC isn't in use, load VEC
  1539. * regs. Either way, set MSR_VSX.
  1540. */
  1541. TM_DEBUG("VSX Unavailable trap whilst transactional at 0x%lx,"
  1542. "MSR=%lx\n",
  1543. regs->nip, regs->msr);
  1544. current->thread.used_vsr = 1;
  1545. /* This reclaims FP and/or VR regs if they're already enabled */
  1546. tm_reclaim_current(TM_CAUSE_FAC_UNAV);
  1547. current->thread.load_vec = 1;
  1548. current->thread.load_fp = 1;
  1549. tm_recheckpoint(&current->thread);
  1550. }
  1551. #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
  1552. void performance_monitor_exception(struct pt_regs *regs)
  1553. {
  1554. __this_cpu_inc(irq_stat.pmu_irqs);
  1555. perf_irq(regs);
  1556. }
  1557. #ifdef CONFIG_PPC_ADV_DEBUG_REGS
  1558. static void handle_debug(struct pt_regs *regs, unsigned long debug_status)
  1559. {
  1560. int changed = 0;
  1561. /*
  1562. * Determine the cause of the debug event, clear the
  1563. * event flags and send a trap to the handler. Torez
  1564. */
  1565. if (debug_status & (DBSR_DAC1R | DBSR_DAC1W)) {
  1566. dbcr_dac(current) &= ~(DBCR_DAC1R | DBCR_DAC1W);
  1567. #ifdef CONFIG_PPC_ADV_DEBUG_DAC_RANGE
  1568. current->thread.debug.dbcr2 &= ~DBCR2_DAC12MODE;
  1569. #endif
  1570. do_send_trap(regs, mfspr(SPRN_DAC1), debug_status,
  1571. 5);
  1572. changed |= 0x01;
  1573. } else if (debug_status & (DBSR_DAC2R | DBSR_DAC2W)) {
  1574. dbcr_dac(current) &= ~(DBCR_DAC2R | DBCR_DAC2W);
  1575. do_send_trap(regs, mfspr(SPRN_DAC2), debug_status,
  1576. 6);
  1577. changed |= 0x01;
  1578. } else if (debug_status & DBSR_IAC1) {
  1579. current->thread.debug.dbcr0 &= ~DBCR0_IAC1;
  1580. dbcr_iac_range(current) &= ~DBCR_IAC12MODE;
  1581. do_send_trap(regs, mfspr(SPRN_IAC1), debug_status,
  1582. 1);
  1583. changed |= 0x01;
  1584. } else if (debug_status & DBSR_IAC2) {
  1585. current->thread.debug.dbcr0 &= ~DBCR0_IAC2;
  1586. do_send_trap(regs, mfspr(SPRN_IAC2), debug_status,
  1587. 2);
  1588. changed |= 0x01;
  1589. } else if (debug_status & DBSR_IAC3) {
  1590. current->thread.debug.dbcr0 &= ~DBCR0_IAC3;
  1591. dbcr_iac_range(current) &= ~DBCR_IAC34MODE;
  1592. do_send_trap(regs, mfspr(SPRN_IAC3), debug_status,
  1593. 3);
  1594. changed |= 0x01;
  1595. } else if (debug_status & DBSR_IAC4) {
  1596. current->thread.debug.dbcr0 &= ~DBCR0_IAC4;
  1597. do_send_trap(regs, mfspr(SPRN_IAC4), debug_status,
  1598. 4);
  1599. changed |= 0x01;
  1600. }
  1601. /*
  1602. * At the point this routine was called, the MSR(DE) was turned off.
  1603. * Check all other debug flags and see if that bit needs to be turned
  1604. * back on or not.
  1605. */
  1606. if (DBCR_ACTIVE_EVENTS(current->thread.debug.dbcr0,
  1607. current->thread.debug.dbcr1))
  1608. regs->msr |= MSR_DE;
  1609. else
  1610. /* Make sure the IDM flag is off */
  1611. current->thread.debug.dbcr0 &= ~DBCR0_IDM;
  1612. if (changed & 0x01)
  1613. mtspr(SPRN_DBCR0, current->thread.debug.dbcr0);
  1614. }
  1615. void DebugException(struct pt_regs *regs, unsigned long debug_status)
  1616. {
  1617. current->thread.debug.dbsr = debug_status;
  1618. /* Hack alert: On BookE, Branch Taken stops on the branch itself, while
  1619. * on server, it stops on the target of the branch. In order to simulate
  1620. * the server behaviour, we thus restart right away with a single step
  1621. * instead of stopping here when hitting a BT
  1622. */
  1623. if (debug_status & DBSR_BT) {
  1624. regs->msr &= ~MSR_DE;
  1625. /* Disable BT */
  1626. mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) & ~DBCR0_BT);
  1627. /* Clear the BT event */
  1628. mtspr(SPRN_DBSR, DBSR_BT);
  1629. /* Do the single step trick only when coming from userspace */
  1630. if (user_mode(regs)) {
  1631. current->thread.debug.dbcr0 &= ~DBCR0_BT;
  1632. current->thread.debug.dbcr0 |= DBCR0_IDM | DBCR0_IC;
  1633. regs->msr |= MSR_DE;
  1634. return;
  1635. }
  1636. if (kprobe_post_handler(regs))
  1637. return;
  1638. if (notify_die(DIE_SSTEP, "block_step", regs, 5,
  1639. 5, SIGTRAP) == NOTIFY_STOP) {
  1640. return;
  1641. }
  1642. if (debugger_sstep(regs))
  1643. return;
  1644. } else if (debug_status & DBSR_IC) { /* Instruction complete */
  1645. regs->msr &= ~MSR_DE;
  1646. /* Disable instruction completion */
  1647. mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) & ~DBCR0_IC);
  1648. /* Clear the instruction completion event */
  1649. mtspr(SPRN_DBSR, DBSR_IC);
  1650. if (kprobe_post_handler(regs))
  1651. return;
  1652. if (notify_die(DIE_SSTEP, "single_step", regs, 5,
  1653. 5, SIGTRAP) == NOTIFY_STOP) {
  1654. return;
  1655. }
  1656. if (debugger_sstep(regs))
  1657. return;
  1658. if (user_mode(regs)) {
  1659. current->thread.debug.dbcr0 &= ~DBCR0_IC;
  1660. if (DBCR_ACTIVE_EVENTS(current->thread.debug.dbcr0,
  1661. current->thread.debug.dbcr1))
  1662. regs->msr |= MSR_DE;
  1663. else
  1664. /* Make sure the IDM bit is off */
  1665. current->thread.debug.dbcr0 &= ~DBCR0_IDM;
  1666. }
  1667. _exception(SIGTRAP, regs, TRAP_TRACE, regs->nip);
  1668. } else
  1669. handle_debug(regs, debug_status);
  1670. }
  1671. NOKPROBE_SYMBOL(DebugException);
  1672. #endif /* CONFIG_PPC_ADV_DEBUG_REGS */
  1673. #if !defined(CONFIG_TAU_INT)
  1674. void TAUException(struct pt_regs *regs)
  1675. {
  1676. printk("TAU trap at PC: %lx, MSR: %lx, vector=%lx %s\n",
  1677. regs->nip, regs->msr, regs->trap, print_tainted());
  1678. }
  1679. #endif /* CONFIG_INT_TAU */
  1680. #ifdef CONFIG_ALTIVEC
  1681. void altivec_assist_exception(struct pt_regs *regs)
  1682. {
  1683. int err;
  1684. if (!user_mode(regs)) {
  1685. printk(KERN_EMERG "VMX/Altivec assist exception in kernel mode"
  1686. " at %lx\n", regs->nip);
  1687. die("Kernel VMX/Altivec assist exception", regs, SIGILL);
  1688. }
  1689. flush_altivec_to_thread(current);
  1690. PPC_WARN_EMULATED(altivec, regs);
  1691. err = emulate_altivec(regs);
  1692. if (err == 0) {
  1693. regs->nip += 4; /* skip emulated instruction */
  1694. emulate_single_step(regs);
  1695. return;
  1696. }
  1697. if (err == -EFAULT) {
  1698. /* got an error reading the instruction */
  1699. _exception(SIGSEGV, regs, SEGV_ACCERR, regs->nip);
  1700. } else {
  1701. /* didn't recognize the instruction */
  1702. /* XXX quick hack for now: set the non-Java bit in the VSCR */
  1703. printk_ratelimited(KERN_ERR "Unrecognized altivec instruction "
  1704. "in %s at %lx\n", current->comm, regs->nip);
  1705. current->thread.vr_state.vscr.u[3] |= 0x10000;
  1706. }
  1707. }
  1708. #endif /* CONFIG_ALTIVEC */
  1709. #ifdef CONFIG_FSL_BOOKE
  1710. void CacheLockingException(struct pt_regs *regs, unsigned long address,
  1711. unsigned long error_code)
  1712. {
  1713. /* We treat cache locking instructions from the user
  1714. * as priv ops, in the future we could try to do
  1715. * something smarter
  1716. */
  1717. if (error_code & (ESR_DLK|ESR_ILK))
  1718. _exception(SIGILL, regs, ILL_PRVOPC, regs->nip);
  1719. return;
  1720. }
  1721. #endif /* CONFIG_FSL_BOOKE */
  1722. #ifdef CONFIG_SPE
  1723. void SPEFloatingPointException(struct pt_regs *regs)
  1724. {
  1725. extern int do_spe_mathemu(struct pt_regs *regs);
  1726. unsigned long spefscr;
  1727. int fpexc_mode;
  1728. int code = FPE_FLTUNK;
  1729. int err;
  1730. flush_spe_to_thread(current);
  1731. spefscr = current->thread.spefscr;
  1732. fpexc_mode = current->thread.fpexc_mode;
  1733. if ((spefscr & SPEFSCR_FOVF) && (fpexc_mode & PR_FP_EXC_OVF)) {
  1734. code = FPE_FLTOVF;
  1735. }
  1736. else if ((spefscr & SPEFSCR_FUNF) && (fpexc_mode & PR_FP_EXC_UND)) {
  1737. code = FPE_FLTUND;
  1738. }
  1739. else if ((spefscr & SPEFSCR_FDBZ) && (fpexc_mode & PR_FP_EXC_DIV))
  1740. code = FPE_FLTDIV;
  1741. else if ((spefscr & SPEFSCR_FINV) && (fpexc_mode & PR_FP_EXC_INV)) {
  1742. code = FPE_FLTINV;
  1743. }
  1744. else if ((spefscr & (SPEFSCR_FG | SPEFSCR_FX)) && (fpexc_mode & PR_FP_EXC_RES))
  1745. code = FPE_FLTRES;
  1746. err = do_spe_mathemu(regs);
  1747. if (err == 0) {
  1748. regs->nip += 4; /* skip emulated instruction */
  1749. emulate_single_step(regs);
  1750. return;
  1751. }
  1752. if (err == -EFAULT) {
  1753. /* got an error reading the instruction */
  1754. _exception(SIGSEGV, regs, SEGV_ACCERR, regs->nip);
  1755. } else if (err == -EINVAL) {
  1756. /* didn't recognize the instruction */
  1757. printk(KERN_ERR "unrecognized spe instruction "
  1758. "in %s at %lx\n", current->comm, regs->nip);
  1759. } else {
  1760. _exception(SIGFPE, regs, code, regs->nip);
  1761. }
  1762. return;
  1763. }
  1764. void SPEFloatingPointRoundException(struct pt_regs *regs)
  1765. {
  1766. extern int speround_handler(struct pt_regs *regs);
  1767. int err;
  1768. preempt_disable();
  1769. if (regs->msr & MSR_SPE)
  1770. giveup_spe(current);
  1771. preempt_enable();
  1772. regs->nip -= 4;
  1773. err = speround_handler(regs);
  1774. if (err == 0) {
  1775. regs->nip += 4; /* skip emulated instruction */
  1776. emulate_single_step(regs);
  1777. return;
  1778. }
  1779. if (err == -EFAULT) {
  1780. /* got an error reading the instruction */
  1781. _exception(SIGSEGV, regs, SEGV_ACCERR, regs->nip);
  1782. } else if (err == -EINVAL) {
  1783. /* didn't recognize the instruction */
  1784. printk(KERN_ERR "unrecognized spe instruction "
  1785. "in %s at %lx\n", current->comm, regs->nip);
  1786. } else {
  1787. _exception(SIGFPE, regs, FPE_FLTUNK, regs->nip);
  1788. return;
  1789. }
  1790. }
  1791. #endif
  1792. /*
  1793. * We enter here if we get an unrecoverable exception, that is, one
  1794. * that happened at a point where the RI (recoverable interrupt) bit
  1795. * in the MSR is 0. This indicates that SRR0/1 are live, and that
  1796. * we therefore lost state by taking this exception.
  1797. */
  1798. void unrecoverable_exception(struct pt_regs *regs)
  1799. {
  1800. printk(KERN_EMERG "Unrecoverable exception %lx at %lx\n",
  1801. regs->trap, regs->nip);
  1802. die("Unrecoverable exception", regs, SIGABRT);
  1803. }
  1804. NOKPROBE_SYMBOL(unrecoverable_exception);
  1805. #if defined(CONFIG_BOOKE_WDT) || defined(CONFIG_40x)
  1806. /*
  1807. * Default handler for a Watchdog exception,
  1808. * spins until a reboot occurs
  1809. */
  1810. void __attribute__ ((weak)) WatchdogHandler(struct pt_regs *regs)
  1811. {
  1812. /* Generic WatchdogHandler, implement your own */
  1813. mtspr(SPRN_TCR, mfspr(SPRN_TCR)&(~TCR_WIE));
  1814. return;
  1815. }
  1816. void WatchdogException(struct pt_regs *regs)
  1817. {
  1818. printk (KERN_EMERG "PowerPC Book-E Watchdog Exception\n");
  1819. WatchdogHandler(regs);
  1820. }
  1821. #endif
  1822. /*
  1823. * We enter here if we discover during exception entry that we are
  1824. * running in supervisor mode with a userspace value in the stack pointer.
  1825. */
  1826. void kernel_bad_stack(struct pt_regs *regs)
  1827. {
  1828. printk(KERN_EMERG "Bad kernel stack pointer %lx at %lx\n",
  1829. regs->gpr[1], regs->nip);
  1830. die("Bad kernel stack pointer", regs, SIGABRT);
  1831. }
  1832. NOKPROBE_SYMBOL(kernel_bad_stack);
  1833. void __init trap_init(void)
  1834. {
  1835. }
  1836. #ifdef CONFIG_PPC_EMULATED_STATS
  1837. #define WARN_EMULATED_SETUP(type) .type = { .name = #type }
  1838. struct ppc_emulated ppc_emulated = {
  1839. #ifdef CONFIG_ALTIVEC
  1840. WARN_EMULATED_SETUP(altivec),
  1841. #endif
  1842. WARN_EMULATED_SETUP(dcba),
  1843. WARN_EMULATED_SETUP(dcbz),
  1844. WARN_EMULATED_SETUP(fp_pair),
  1845. WARN_EMULATED_SETUP(isel),
  1846. WARN_EMULATED_SETUP(mcrxr),
  1847. WARN_EMULATED_SETUP(mfpvr),
  1848. WARN_EMULATED_SETUP(multiple),
  1849. WARN_EMULATED_SETUP(popcntb),
  1850. WARN_EMULATED_SETUP(spe),
  1851. WARN_EMULATED_SETUP(string),
  1852. WARN_EMULATED_SETUP(sync),
  1853. WARN_EMULATED_SETUP(unaligned),
  1854. #ifdef CONFIG_MATH_EMULATION
  1855. WARN_EMULATED_SETUP(math),
  1856. #endif
  1857. #ifdef CONFIG_VSX
  1858. WARN_EMULATED_SETUP(vsx),
  1859. #endif
  1860. #ifdef CONFIG_PPC64
  1861. WARN_EMULATED_SETUP(mfdscr),
  1862. WARN_EMULATED_SETUP(mtdscr),
  1863. WARN_EMULATED_SETUP(lq_stq),
  1864. WARN_EMULATED_SETUP(lxvw4x),
  1865. WARN_EMULATED_SETUP(lxvh8x),
  1866. WARN_EMULATED_SETUP(lxvd2x),
  1867. WARN_EMULATED_SETUP(lxvb16x),
  1868. #endif
  1869. };
  1870. u32 ppc_warn_emulated;
  1871. void ppc_warn_emulated_print(const char *type)
  1872. {
  1873. pr_warn_ratelimited("%s used emulated %s instruction\n", current->comm,
  1874. type);
  1875. }
  1876. static int __init ppc_warn_emulated_init(void)
  1877. {
  1878. struct dentry *dir, *d;
  1879. unsigned int i;
  1880. struct ppc_emulated_entry *entries = (void *)&ppc_emulated;
  1881. if (!powerpc_debugfs_root)
  1882. return -ENODEV;
  1883. dir = debugfs_create_dir("emulated_instructions",
  1884. powerpc_debugfs_root);
  1885. if (!dir)
  1886. return -ENOMEM;
  1887. d = debugfs_create_u32("do_warn", 0644, dir,
  1888. &ppc_warn_emulated);
  1889. if (!d)
  1890. goto fail;
  1891. for (i = 0; i < sizeof(ppc_emulated)/sizeof(*entries); i++) {
  1892. d = debugfs_create_u32(entries[i].name, 0644, dir,
  1893. (u32 *)&entries[i].val.counter);
  1894. if (!d)
  1895. goto fail;
  1896. }
  1897. return 0;
  1898. fail:
  1899. debugfs_remove_recursive(dir);
  1900. return -ENOMEM;
  1901. }
  1902. device_initcall(ppc_warn_emulated_init);
  1903. #endif /* CONFIG_PPC_EMULATED_STATS */