idle.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847
  1. /*
  2. * PowerNV cpuidle code
  3. *
  4. * Copyright 2015 IBM Corp.
  5. *
  6. * This program is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU General Public License
  8. * as published by the Free Software Foundation; either version
  9. * 2 of the License, or (at your option) any later version.
  10. */
  11. #include <linux/types.h>
  12. #include <linux/mm.h>
  13. #include <linux/slab.h>
  14. #include <linux/of.h>
  15. #include <linux/device.h>
  16. #include <linux/cpu.h>
  17. #include <asm/firmware.h>
  18. #include <asm/machdep.h>
  19. #include <asm/opal.h>
  20. #include <asm/cputhreads.h>
  21. #include <asm/cpuidle.h>
  22. #include <asm/code-patching.h>
  23. #include <asm/smp.h>
  24. #include <asm/runlatch.h>
  25. #include <asm/dbell.h>
  26. #include "powernv.h"
  27. #include "subcore.h"
  28. /* Power ISA 3.0 allows for stop states 0x0 - 0xF */
  29. #define MAX_STOP_STATE 0xF
  30. #define P9_STOP_SPR_MSR 2000
  31. #define P9_STOP_SPR_PSSCR 855
  32. static u32 supported_cpuidle_states;
  33. struct pnv_idle_states_t *pnv_idle_states;
  34. int nr_pnv_idle_states;
  35. /*
  36. * The default stop state that will be used by ppc_md.power_save
  37. * function on platforms that support stop instruction.
  38. */
  39. static u64 pnv_default_stop_val;
  40. static u64 pnv_default_stop_mask;
  41. static bool default_stop_found;
  42. /*
  43. * First deep stop state. Used to figure out when to save/restore
  44. * hypervisor context.
  45. */
  46. u64 pnv_first_deep_stop_state = MAX_STOP_STATE;
  47. /*
  48. * psscr value and mask of the deepest stop idle state.
  49. * Used when a cpu is offlined.
  50. */
  51. static u64 pnv_deepest_stop_psscr_val;
  52. static u64 pnv_deepest_stop_psscr_mask;
  53. static u64 pnv_deepest_stop_flag;
  54. static bool deepest_stop_found;
  55. static int pnv_save_sprs_for_deep_states(void)
  56. {
  57. int cpu;
  58. int rc;
  59. /*
  60. * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across
  61. * all cpus at boot. Get these reg values of current cpu and use the
  62. * same across all cpus.
  63. */
  64. uint64_t lpcr_val = mfspr(SPRN_LPCR);
  65. uint64_t hid0_val = mfspr(SPRN_HID0);
  66. uint64_t hid1_val = mfspr(SPRN_HID1);
  67. uint64_t hid4_val = mfspr(SPRN_HID4);
  68. uint64_t hid5_val = mfspr(SPRN_HID5);
  69. uint64_t hmeer_val = mfspr(SPRN_HMEER);
  70. uint64_t msr_val = MSR_IDLE;
  71. uint64_t psscr_val = pnv_deepest_stop_psscr_val;
  72. for_each_present_cpu(cpu) {
  73. uint64_t pir = get_hard_smp_processor_id(cpu);
  74. uint64_t hsprg0_val = (uint64_t)paca_ptrs[cpu];
  75. rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val);
  76. if (rc != 0)
  77. return rc;
  78. rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val);
  79. if (rc != 0)
  80. return rc;
  81. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  82. rc = opal_slw_set_reg(pir, P9_STOP_SPR_MSR, msr_val);
  83. if (rc)
  84. return rc;
  85. rc = opal_slw_set_reg(pir,
  86. P9_STOP_SPR_PSSCR, psscr_val);
  87. if (rc)
  88. return rc;
  89. }
  90. /* HIDs are per core registers */
  91. if (cpu_thread_in_core(cpu) == 0) {
  92. rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val);
  93. if (rc != 0)
  94. return rc;
  95. rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val);
  96. if (rc != 0)
  97. return rc;
  98. /* Only p8 needs to set extra HID regiters */
  99. if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
  100. rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val);
  101. if (rc != 0)
  102. return rc;
  103. rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val);
  104. if (rc != 0)
  105. return rc;
  106. rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val);
  107. if (rc != 0)
  108. return rc;
  109. }
  110. }
  111. }
  112. return 0;
  113. }
  114. static void pnv_alloc_idle_core_states(void)
  115. {
  116. int i, j;
  117. int nr_cores = cpu_nr_cores();
  118. u32 *core_idle_state;
  119. /*
  120. * core_idle_state - The lower 8 bits track the idle state of
  121. * each thread of the core.
  122. *
  123. * The most significant bit is the lock bit.
  124. *
  125. * Initially all the bits corresponding to threads_per_core
  126. * are set. They are cleared when the thread enters deep idle
  127. * state like sleep and winkle/stop.
  128. *
  129. * Initially the lock bit is cleared. The lock bit has 2
  130. * purposes:
  131. * a. While the first thread in the core waking up from
  132. * idle is restoring core state, it prevents other
  133. * threads in the core from switching to process
  134. * context.
  135. * b. While the last thread in the core is saving the
  136. * core state, it prevents a different thread from
  137. * waking up.
  138. */
  139. for (i = 0; i < nr_cores; i++) {
  140. int first_cpu = i * threads_per_core;
  141. int node = cpu_to_node(first_cpu);
  142. size_t paca_ptr_array_size;
  143. core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node);
  144. *core_idle_state = (1 << threads_per_core) - 1;
  145. paca_ptr_array_size = (threads_per_core *
  146. sizeof(struct paca_struct *));
  147. for (j = 0; j < threads_per_core; j++) {
  148. int cpu = first_cpu + j;
  149. paca_ptrs[cpu]->core_idle_state_ptr = core_idle_state;
  150. paca_ptrs[cpu]->thread_idle_state = PNV_THREAD_RUNNING;
  151. paca_ptrs[cpu]->thread_mask = 1 << j;
  152. }
  153. }
  154. update_subcore_sibling_mask();
  155. if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) {
  156. int rc = pnv_save_sprs_for_deep_states();
  157. if (likely(!rc))
  158. return;
  159. /*
  160. * The stop-api is unable to restore hypervisor
  161. * resources on wakeup from platform idle states which
  162. * lose full context. So disable such states.
  163. */
  164. supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT;
  165. pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n");
  166. pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n");
  167. if (cpu_has_feature(CPU_FTR_ARCH_300) &&
  168. (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) {
  169. /*
  170. * Use the default stop state for CPU-Hotplug
  171. * if available.
  172. */
  173. if (default_stop_found) {
  174. pnv_deepest_stop_psscr_val =
  175. pnv_default_stop_val;
  176. pnv_deepest_stop_psscr_mask =
  177. pnv_default_stop_mask;
  178. pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n",
  179. pnv_deepest_stop_psscr_val);
  180. } else { /* Fallback to snooze loop for CPU-Hotplug */
  181. deepest_stop_found = false;
  182. pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n");
  183. }
  184. }
  185. }
  186. }
  187. u32 pnv_get_supported_cpuidle_states(void)
  188. {
  189. return supported_cpuidle_states;
  190. }
  191. EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states);
  192. static void pnv_fastsleep_workaround_apply(void *info)
  193. {
  194. int rc;
  195. int *err = info;
  196. rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP,
  197. OPAL_CONFIG_IDLE_APPLY);
  198. if (rc)
  199. *err = 1;
  200. }
  201. /*
  202. * Used to store fastsleep workaround state
  203. * 0 - Workaround applied/undone at fastsleep entry/exit path (Default)
  204. * 1 - Workaround applied once, never undone.
  205. */
  206. static u8 fastsleep_workaround_applyonce;
  207. static ssize_t show_fastsleep_workaround_applyonce(struct device *dev,
  208. struct device_attribute *attr, char *buf)
  209. {
  210. return sprintf(buf, "%u\n", fastsleep_workaround_applyonce);
  211. }
  212. static ssize_t store_fastsleep_workaround_applyonce(struct device *dev,
  213. struct device_attribute *attr, const char *buf,
  214. size_t count)
  215. {
  216. cpumask_t primary_thread_mask;
  217. int err;
  218. u8 val;
  219. if (kstrtou8(buf, 0, &val) || val != 1)
  220. return -EINVAL;
  221. if (fastsleep_workaround_applyonce == 1)
  222. return count;
  223. /*
  224. * fastsleep_workaround_applyonce = 1 implies
  225. * fastsleep workaround needs to be left in 'applied' state on all
  226. * the cores. Do this by-
  227. * 1. Patching out the call to 'undo' workaround in fastsleep exit path
  228. * 2. Sending ipi to all the cores which have at least one online thread
  229. * 3. Patching out the call to 'apply' workaround in fastsleep entry
  230. * path
  231. * There is no need to send ipi to cores which have all threads
  232. * offlined, as last thread of the core entering fastsleep or deeper
  233. * state would have applied workaround.
  234. */
  235. err = patch_instruction(
  236. (unsigned int *)pnv_fastsleep_workaround_at_exit,
  237. PPC_INST_NOP);
  238. if (err) {
  239. pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit");
  240. goto fail;
  241. }
  242. get_online_cpus();
  243. primary_thread_mask = cpu_online_cores_map();
  244. on_each_cpu_mask(&primary_thread_mask,
  245. pnv_fastsleep_workaround_apply,
  246. &err, 1);
  247. put_online_cpus();
  248. if (err) {
  249. pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply");
  250. goto fail;
  251. }
  252. err = patch_instruction(
  253. (unsigned int *)pnv_fastsleep_workaround_at_entry,
  254. PPC_INST_NOP);
  255. if (err) {
  256. pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry");
  257. goto fail;
  258. }
  259. fastsleep_workaround_applyonce = 1;
  260. return count;
  261. fail:
  262. return -EIO;
  263. }
  264. static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600,
  265. show_fastsleep_workaround_applyonce,
  266. store_fastsleep_workaround_applyonce);
  267. static unsigned long __power7_idle_type(unsigned long type)
  268. {
  269. unsigned long srr1;
  270. if (!prep_irq_for_idle_irqsoff())
  271. return 0;
  272. __ppc64_runlatch_off();
  273. srr1 = power7_idle_insn(type);
  274. __ppc64_runlatch_on();
  275. fini_irq_for_idle_irqsoff();
  276. return srr1;
  277. }
  278. void power7_idle_type(unsigned long type)
  279. {
  280. unsigned long srr1;
  281. srr1 = __power7_idle_type(type);
  282. irq_set_pending_from_srr1(srr1);
  283. }
  284. void power7_idle(void)
  285. {
  286. if (!powersave_nap)
  287. return;
  288. power7_idle_type(PNV_THREAD_NAP);
  289. }
  290. static unsigned long __power9_idle_type(unsigned long stop_psscr_val,
  291. unsigned long stop_psscr_mask)
  292. {
  293. unsigned long psscr;
  294. unsigned long srr1;
  295. if (!prep_irq_for_idle_irqsoff())
  296. return 0;
  297. psscr = mfspr(SPRN_PSSCR);
  298. psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val;
  299. __ppc64_runlatch_off();
  300. srr1 = power9_idle_stop(psscr);
  301. __ppc64_runlatch_on();
  302. fini_irq_for_idle_irqsoff();
  303. return srr1;
  304. }
  305. void power9_idle_type(unsigned long stop_psscr_val,
  306. unsigned long stop_psscr_mask)
  307. {
  308. unsigned long srr1;
  309. srr1 = __power9_idle_type(stop_psscr_val, stop_psscr_mask);
  310. irq_set_pending_from_srr1(srr1);
  311. }
  312. /*
  313. * Used for ppc_md.power_save which needs a function with no parameters
  314. */
  315. void power9_idle(void)
  316. {
  317. power9_idle_type(pnv_default_stop_val, pnv_default_stop_mask);
  318. }
  319. #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
  320. /*
  321. * This is used in working around bugs in thread reconfiguration
  322. * on POWER9 (at least up to Nimbus DD2.2) relating to transactional
  323. * memory and the way that XER[SO] is checkpointed.
  324. * This function forces the core into SMT4 in order by asking
  325. * all other threads not to stop, and sending a message to any
  326. * that are in a stop state.
  327. * Must be called with preemption disabled.
  328. */
  329. void pnv_power9_force_smt4_catch(void)
  330. {
  331. int cpu, cpu0, thr;
  332. int awake_threads = 1; /* this thread is awake */
  333. int poke_threads = 0;
  334. int need_awake = threads_per_core;
  335. cpu = smp_processor_id();
  336. cpu0 = cpu & ~(threads_per_core - 1);
  337. for (thr = 0; thr < threads_per_core; ++thr) {
  338. if (cpu != cpu0 + thr)
  339. atomic_inc(&paca_ptrs[cpu0+thr]->dont_stop);
  340. }
  341. /* order setting dont_stop vs testing requested_psscr */
  342. mb();
  343. for (thr = 0; thr < threads_per_core; ++thr) {
  344. if (!paca_ptrs[cpu0+thr]->requested_psscr)
  345. ++awake_threads;
  346. else
  347. poke_threads |= (1 << thr);
  348. }
  349. /* If at least 3 threads are awake, the core is in SMT4 already */
  350. if (awake_threads < need_awake) {
  351. /* We have to wake some threads; we'll use msgsnd */
  352. for (thr = 0; thr < threads_per_core; ++thr) {
  353. if (poke_threads & (1 << thr)) {
  354. ppc_msgsnd_sync();
  355. ppc_msgsnd(PPC_DBELL_MSGTYPE, 0,
  356. paca_ptrs[cpu0+thr]->hw_cpu_id);
  357. }
  358. }
  359. /* now spin until at least 3 threads are awake */
  360. do {
  361. for (thr = 0; thr < threads_per_core; ++thr) {
  362. if ((poke_threads & (1 << thr)) &&
  363. !paca_ptrs[cpu0+thr]->requested_psscr) {
  364. ++awake_threads;
  365. poke_threads &= ~(1 << thr);
  366. }
  367. }
  368. } while (awake_threads < need_awake);
  369. }
  370. }
  371. EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_catch);
  372. void pnv_power9_force_smt4_release(void)
  373. {
  374. int cpu, cpu0, thr;
  375. cpu = smp_processor_id();
  376. cpu0 = cpu & ~(threads_per_core - 1);
  377. /* clear all the dont_stop flags */
  378. for (thr = 0; thr < threads_per_core; ++thr) {
  379. if (cpu != cpu0 + thr)
  380. atomic_dec(&paca_ptrs[cpu0+thr]->dont_stop);
  381. }
  382. }
  383. EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_release);
  384. #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
  385. #ifdef CONFIG_HOTPLUG_CPU
  386. void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val)
  387. {
  388. u64 pir = get_hard_smp_processor_id(cpu);
  389. mtspr(SPRN_LPCR, lpcr_val);
  390. /*
  391. * Program the LPCR via stop-api only if the deepest stop state
  392. * can lose hypervisor context.
  393. */
  394. if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT)
  395. opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val);
  396. }
  397. /*
  398. * pnv_cpu_offline: A function that puts the CPU into the deepest
  399. * available platform idle state on a CPU-Offline.
  400. * interrupts hard disabled and no lazy irq pending.
  401. */
  402. unsigned long pnv_cpu_offline(unsigned int cpu)
  403. {
  404. unsigned long srr1;
  405. u32 idle_states = pnv_get_supported_cpuidle_states();
  406. __ppc64_runlatch_off();
  407. if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) {
  408. unsigned long psscr;
  409. psscr = mfspr(SPRN_PSSCR);
  410. psscr = (psscr & ~pnv_deepest_stop_psscr_mask) |
  411. pnv_deepest_stop_psscr_val;
  412. srr1 = power9_offline_stop(psscr);
  413. } else if ((idle_states & OPAL_PM_WINKLE_ENABLED) &&
  414. (idle_states & OPAL_PM_LOSE_FULL_CONTEXT)) {
  415. srr1 = power7_idle_insn(PNV_THREAD_WINKLE);
  416. } else if ((idle_states & OPAL_PM_SLEEP_ENABLED) ||
  417. (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
  418. srr1 = power7_idle_insn(PNV_THREAD_SLEEP);
  419. } else if (idle_states & OPAL_PM_NAP_ENABLED) {
  420. srr1 = power7_idle_insn(PNV_THREAD_NAP);
  421. } else {
  422. /* This is the fallback method. We emulate snooze */
  423. while (!generic_check_cpu_restart(cpu)) {
  424. HMT_low();
  425. HMT_very_low();
  426. }
  427. srr1 = 0;
  428. HMT_medium();
  429. }
  430. __ppc64_runlatch_on();
  431. return srr1;
  432. }
  433. #endif
  434. /*
  435. * Power ISA 3.0 idle initialization.
  436. *
  437. * POWER ISA 3.0 defines a new SPR Processor stop Status and Control
  438. * Register (PSSCR) to control idle behavior.
  439. *
  440. * PSSCR layout:
  441. * ----------------------------------------------------------
  442. * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL |
  443. * ----------------------------------------------------------
  444. * 0 4 41 42 43 44 48 54 56 60
  445. *
  446. * PSSCR key fields:
  447. * Bits 0:3 - Power-Saving Level Status (PLS). This field indicates the
  448. * lowest power-saving state the thread entered since stop instruction was
  449. * last executed.
  450. *
  451. * Bit 41 - Status Disable(SD)
  452. * 0 - Shows PLS entries
  453. * 1 - PLS entries are all 0
  454. *
  455. * Bit 42 - Enable State Loss
  456. * 0 - No state is lost irrespective of other fields
  457. * 1 - Allows state loss
  458. *
  459. * Bit 43 - Exit Criterion
  460. * 0 - Exit from power-save mode on any interrupt
  461. * 1 - Exit from power-save mode controlled by LPCR's PECE bits
  462. *
  463. * Bits 44:47 - Power-Saving Level Limit
  464. * This limits the power-saving level that can be entered into.
  465. *
  466. * Bits 60:63 - Requested Level
  467. * Used to specify which power-saving level must be entered on executing
  468. * stop instruction
  469. */
  470. int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags)
  471. {
  472. int err = 0;
  473. /*
  474. * psscr_mask == 0xf indicates an older firmware.
  475. * Set remaining fields of psscr to the default values.
  476. * See NOTE above definition of PSSCR_HV_DEFAULT_VAL
  477. */
  478. if (*psscr_mask == 0xf) {
  479. *psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL;
  480. *psscr_mask = PSSCR_HV_DEFAULT_MASK;
  481. return err;
  482. }
  483. /*
  484. * New firmware is expected to set the psscr_val bits correctly.
  485. * Validate that the following invariants are correctly maintained by
  486. * the new firmware.
  487. * - ESL bit value matches the EC bit value.
  488. * - ESL bit is set for all the deep stop states.
  489. */
  490. if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) {
  491. err = ERR_EC_ESL_MISMATCH;
  492. } else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) &&
  493. GET_PSSCR_ESL(*psscr_val) == 0) {
  494. err = ERR_DEEP_STATE_ESL_MISMATCH;
  495. }
  496. return err;
  497. }
  498. /*
  499. * pnv_arch300_idle_init: Initializes the default idle state, first
  500. * deep idle state and deepest idle state on
  501. * ISA 3.0 CPUs.
  502. *
  503. * @np: /ibm,opal/power-mgt device node
  504. * @flags: cpu-idle-state-flags array
  505. * @dt_idle_states: Number of idle state entries
  506. * Returns 0 on success
  507. */
  508. static int __init pnv_power9_idle_init(void)
  509. {
  510. u64 max_residency_ns = 0;
  511. int i;
  512. /*
  513. * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask},
  514. * and the pnv_default_stop_{val,mask}.
  515. *
  516. * pnv_first_deep_stop_state should be set to the first stop
  517. * level to cause hypervisor state loss.
  518. *
  519. * pnv_deepest_stop_{val,mask} should be set to values corresponding to
  520. * the deepest stop state.
  521. *
  522. * pnv_default_stop_{val,mask} should be set to values corresponding to
  523. * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state.
  524. */
  525. pnv_first_deep_stop_state = MAX_STOP_STATE;
  526. for (i = 0; i < nr_pnv_idle_states; i++) {
  527. int err;
  528. struct pnv_idle_states_t *state = &pnv_idle_states[i];
  529. u64 psscr_rl = state->psscr_val & PSSCR_RL_MASK;
  530. if ((state->flags & OPAL_PM_LOSE_FULL_CONTEXT) &&
  531. pnv_first_deep_stop_state > psscr_rl)
  532. pnv_first_deep_stop_state = psscr_rl;
  533. err = validate_psscr_val_mask(&state->psscr_val,
  534. &state->psscr_mask,
  535. state->flags);
  536. if (err) {
  537. report_invalid_psscr_val(state->psscr_val, err);
  538. continue;
  539. }
  540. state->valid = true;
  541. if (max_residency_ns < state->residency_ns) {
  542. max_residency_ns = state->residency_ns;
  543. pnv_deepest_stop_psscr_val = state->psscr_val;
  544. pnv_deepest_stop_psscr_mask = state->psscr_mask;
  545. pnv_deepest_stop_flag = state->flags;
  546. deepest_stop_found = true;
  547. }
  548. if (!default_stop_found &&
  549. (state->flags & OPAL_PM_STOP_INST_FAST)) {
  550. pnv_default_stop_val = state->psscr_val;
  551. pnv_default_stop_mask = state->psscr_mask;
  552. default_stop_found = true;
  553. }
  554. }
  555. if (unlikely(!default_stop_found)) {
  556. pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n");
  557. } else {
  558. ppc_md.power_save = power9_idle;
  559. pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n",
  560. pnv_default_stop_val, pnv_default_stop_mask);
  561. }
  562. if (unlikely(!deepest_stop_found)) {
  563. pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait");
  564. } else {
  565. pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n",
  566. pnv_deepest_stop_psscr_val,
  567. pnv_deepest_stop_psscr_mask);
  568. }
  569. pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n",
  570. pnv_first_deep_stop_state);
  571. return 0;
  572. }
  573. /*
  574. * Probe device tree for supported idle states
  575. */
  576. static void __init pnv_probe_idle_states(void)
  577. {
  578. int i;
  579. if (nr_pnv_idle_states < 0) {
  580. pr_warn("cpuidle-powernv: no idle states found in the DT\n");
  581. return;
  582. }
  583. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  584. if (pnv_power9_idle_init())
  585. return;
  586. }
  587. for (i = 0; i < nr_pnv_idle_states; i++)
  588. supported_cpuidle_states |= pnv_idle_states[i].flags;
  589. }
  590. /*
  591. * This function parses device-tree and populates all the information
  592. * into pnv_idle_states structure. It also sets up nr_pnv_idle_states
  593. * which is the number of cpuidle states discovered through device-tree.
  594. */
  595. static int pnv_parse_cpuidle_dt(void)
  596. {
  597. struct device_node *np;
  598. int nr_idle_states, i;
  599. int rc = 0;
  600. u32 *temp_u32;
  601. u64 *temp_u64;
  602. const char **temp_string;
  603. np = of_find_node_by_path("/ibm,opal/power-mgt");
  604. if (!np) {
  605. pr_warn("opal: PowerMgmt Node not found\n");
  606. return -ENODEV;
  607. }
  608. nr_idle_states = of_property_count_u32_elems(np,
  609. "ibm,cpu-idle-state-flags");
  610. pnv_idle_states = kcalloc(nr_idle_states, sizeof(*pnv_idle_states),
  611. GFP_KERNEL);
  612. temp_u32 = kcalloc(nr_idle_states, sizeof(u32), GFP_KERNEL);
  613. temp_u64 = kcalloc(nr_idle_states, sizeof(u64), GFP_KERNEL);
  614. temp_string = kcalloc(nr_idle_states, sizeof(char *), GFP_KERNEL);
  615. if (!(pnv_idle_states && temp_u32 && temp_u64 && temp_string)) {
  616. pr_err("Could not allocate memory for dt parsing\n");
  617. rc = -ENOMEM;
  618. goto out;
  619. }
  620. /* Read flags */
  621. if (of_property_read_u32_array(np, "ibm,cpu-idle-state-flags",
  622. temp_u32, nr_idle_states)) {
  623. pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n");
  624. rc = -EINVAL;
  625. goto out;
  626. }
  627. for (i = 0; i < nr_idle_states; i++)
  628. pnv_idle_states[i].flags = temp_u32[i];
  629. /* Read latencies */
  630. if (of_property_read_u32_array(np, "ibm,cpu-idle-state-latencies-ns",
  631. temp_u32, nr_idle_states)) {
  632. pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n");
  633. rc = -EINVAL;
  634. goto out;
  635. }
  636. for (i = 0; i < nr_idle_states; i++)
  637. pnv_idle_states[i].latency_ns = temp_u32[i];
  638. /* Read residencies */
  639. if (of_property_read_u32_array(np, "ibm,cpu-idle-state-residency-ns",
  640. temp_u32, nr_idle_states)) {
  641. pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n");
  642. rc = -EINVAL;
  643. goto out;
  644. }
  645. for (i = 0; i < nr_idle_states; i++)
  646. pnv_idle_states[i].residency_ns = temp_u32[i];
  647. /* For power9 */
  648. if (cpu_has_feature(CPU_FTR_ARCH_300)) {
  649. /* Read pm_crtl_val */
  650. if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr",
  651. temp_u64, nr_idle_states)) {
  652. pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n");
  653. rc = -EINVAL;
  654. goto out;
  655. }
  656. for (i = 0; i < nr_idle_states; i++)
  657. pnv_idle_states[i].psscr_val = temp_u64[i];
  658. /* Read pm_crtl_mask */
  659. if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr-mask",
  660. temp_u64, nr_idle_states)) {
  661. pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n");
  662. rc = -EINVAL;
  663. goto out;
  664. }
  665. for (i = 0; i < nr_idle_states; i++)
  666. pnv_idle_states[i].psscr_mask = temp_u64[i];
  667. }
  668. /*
  669. * power8 specific properties ibm,cpu-idle-state-pmicr-mask and
  670. * ibm,cpu-idle-state-pmicr-val were never used and there is no
  671. * plan to use it in near future. Hence, not parsing these properties
  672. */
  673. if (of_property_read_string_array(np, "ibm,cpu-idle-state-names",
  674. temp_string, nr_idle_states) < 0) {
  675. pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n");
  676. rc = -EINVAL;
  677. goto out;
  678. }
  679. for (i = 0; i < nr_idle_states; i++)
  680. strlcpy(pnv_idle_states[i].name, temp_string[i],
  681. PNV_IDLE_NAME_LEN);
  682. nr_pnv_idle_states = nr_idle_states;
  683. rc = 0;
  684. out:
  685. kfree(temp_u32);
  686. kfree(temp_u64);
  687. kfree(temp_string);
  688. return rc;
  689. }
  690. static int __init pnv_init_idle_states(void)
  691. {
  692. int rc = 0;
  693. supported_cpuidle_states = 0;
  694. /* In case we error out nr_pnv_idle_states will be zero */
  695. nr_pnv_idle_states = 0;
  696. if (cpuidle_disable != IDLE_NO_OVERRIDE)
  697. goto out;
  698. rc = pnv_parse_cpuidle_dt();
  699. if (rc)
  700. return rc;
  701. pnv_probe_idle_states();
  702. if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
  703. patch_instruction(
  704. (unsigned int *)pnv_fastsleep_workaround_at_entry,
  705. PPC_INST_NOP);
  706. patch_instruction(
  707. (unsigned int *)pnv_fastsleep_workaround_at_exit,
  708. PPC_INST_NOP);
  709. } else {
  710. /*
  711. * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that
  712. * workaround is needed to use fastsleep. Provide sysfs
  713. * control to choose how this workaround has to be applied.
  714. */
  715. device_create_file(cpu_subsys.dev_root,
  716. &dev_attr_fastsleep_workaround_applyonce);
  717. }
  718. pnv_alloc_idle_core_states();
  719. if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED)
  720. ppc_md.power_save = power7_idle;
  721. out:
  722. return 0;
  723. }
  724. machine_subsys_initcall(powernv, pnv_init_idle_states);