tick-broadcast.c 27 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018
  1. /*
  2. * linux/kernel/time/tick-broadcast.c
  3. *
  4. * This file contains functions which emulate a local clock-event
  5. * device via a broadcast event source.
  6. *
  7. * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
  8. * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  9. * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  10. *
  11. * This code is licenced under the GPL version 2. For details see
  12. * kernel-base/COPYING.
  13. */
  14. #include <linux/cpu.h>
  15. #include <linux/err.h>
  16. #include <linux/hrtimer.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/percpu.h>
  19. #include <linux/profile.h>
  20. #include <linux/sched.h>
  21. #include <linux/smp.h>
  22. #include <linux/module.h>
  23. #include "tick-internal.h"
  24. /*
  25. * Broadcast support for broken x86 hardware, where the local apic
  26. * timer stops in C3 state.
  27. */
  28. static struct tick_device tick_broadcast_device;
  29. static cpumask_var_t tick_broadcast_mask __cpumask_var_read_mostly;
  30. static cpumask_var_t tick_broadcast_on __cpumask_var_read_mostly;
  31. static cpumask_var_t tmpmask __cpumask_var_read_mostly;
  32. static int tick_broadcast_forced;
  33. static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
  34. #ifdef CONFIG_TICK_ONESHOT
  35. static void tick_broadcast_setup_oneshot(struct clock_event_device *bc);
  36. static void tick_broadcast_clear_oneshot(int cpu);
  37. static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
  38. #else
  39. static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) { BUG(); }
  40. static inline void tick_broadcast_clear_oneshot(int cpu) { }
  41. static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }
  42. #endif
  43. /*
  44. * Debugging: see timer_list.c
  45. */
  46. struct tick_device *tick_get_broadcast_device(void)
  47. {
  48. return &tick_broadcast_device;
  49. }
  50. struct cpumask *tick_get_broadcast_mask(void)
  51. {
  52. return tick_broadcast_mask;
  53. }
  54. /*
  55. * Start the device in periodic mode
  56. */
  57. static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  58. {
  59. if (bc)
  60. tick_setup_periodic(bc, 1);
  61. }
  62. /*
  63. * Check, if the device can be utilized as broadcast device:
  64. */
  65. static bool tick_check_broadcast_device(struct clock_event_device *curdev,
  66. struct clock_event_device *newdev)
  67. {
  68. if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
  69. (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
  70. (newdev->features & CLOCK_EVT_FEAT_C3STOP))
  71. return false;
  72. if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&
  73. !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
  74. return false;
  75. return !curdev || newdev->rating > curdev->rating;
  76. }
  77. /*
  78. * Conditionally install/replace broadcast device
  79. */
  80. void tick_install_broadcast_device(struct clock_event_device *dev)
  81. {
  82. struct clock_event_device *cur = tick_broadcast_device.evtdev;
  83. if (!tick_check_broadcast_device(cur, dev))
  84. return;
  85. if (!try_module_get(dev->owner))
  86. return;
  87. clockevents_exchange_device(cur, dev);
  88. if (cur)
  89. cur->event_handler = clockevents_handle_noop;
  90. tick_broadcast_device.evtdev = dev;
  91. if (!cpumask_empty(tick_broadcast_mask))
  92. tick_broadcast_start_periodic(dev);
  93. /*
  94. * Inform all cpus about this. We might be in a situation
  95. * where we did not switch to oneshot mode because the per cpu
  96. * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
  97. * of a oneshot capable broadcast device. Without that
  98. * notification the systems stays stuck in periodic mode
  99. * forever.
  100. */
  101. if (dev->features & CLOCK_EVT_FEAT_ONESHOT)
  102. tick_clock_notify();
  103. }
  104. /*
  105. * Check, if the device is the broadcast device
  106. */
  107. int tick_is_broadcast_device(struct clock_event_device *dev)
  108. {
  109. return (dev && tick_broadcast_device.evtdev == dev);
  110. }
  111. int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq)
  112. {
  113. int ret = -ENODEV;
  114. if (tick_is_broadcast_device(dev)) {
  115. raw_spin_lock(&tick_broadcast_lock);
  116. ret = __clockevents_update_freq(dev, freq);
  117. raw_spin_unlock(&tick_broadcast_lock);
  118. }
  119. return ret;
  120. }
  121. static void err_broadcast(const struct cpumask *mask)
  122. {
  123. pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
  124. }
  125. static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
  126. {
  127. if (!dev->broadcast)
  128. dev->broadcast = tick_broadcast;
  129. if (!dev->broadcast) {
  130. pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
  131. dev->name);
  132. dev->broadcast = err_broadcast;
  133. }
  134. }
  135. /*
  136. * Check, if the device is disfunctional and a place holder, which
  137. * needs to be handled by the broadcast device.
  138. */
  139. int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
  140. {
  141. struct clock_event_device *bc = tick_broadcast_device.evtdev;
  142. unsigned long flags;
  143. int ret = 0;
  144. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  145. /*
  146. * Devices might be registered with both periodic and oneshot
  147. * mode disabled. This signals, that the device needs to be
  148. * operated from the broadcast device and is a placeholder for
  149. * the cpu local device.
  150. */
  151. if (!tick_device_is_functional(dev)) {
  152. dev->event_handler = tick_handle_periodic;
  153. tick_device_setup_broadcast_func(dev);
  154. cpumask_set_cpu(cpu, tick_broadcast_mask);
  155. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
  156. tick_broadcast_start_periodic(bc);
  157. else
  158. tick_broadcast_setup_oneshot(bc);
  159. ret = 1;
  160. } else {
  161. /*
  162. * Clear the broadcast bit for this cpu if the
  163. * device is not power state affected.
  164. */
  165. if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
  166. cpumask_clear_cpu(cpu, tick_broadcast_mask);
  167. else
  168. tick_device_setup_broadcast_func(dev);
  169. /*
  170. * Clear the broadcast bit if the CPU is not in
  171. * periodic broadcast on state.
  172. */
  173. if (!cpumask_test_cpu(cpu, tick_broadcast_on))
  174. cpumask_clear_cpu(cpu, tick_broadcast_mask);
  175. switch (tick_broadcast_device.mode) {
  176. case TICKDEV_MODE_ONESHOT:
  177. /*
  178. * If the system is in oneshot mode we can
  179. * unconditionally clear the oneshot mask bit,
  180. * because the CPU is running and therefore
  181. * not in an idle state which causes the power
  182. * state affected device to stop. Let the
  183. * caller initialize the device.
  184. */
  185. tick_broadcast_clear_oneshot(cpu);
  186. ret = 0;
  187. break;
  188. case TICKDEV_MODE_PERIODIC:
  189. /*
  190. * If the system is in periodic mode, check
  191. * whether the broadcast device can be
  192. * switched off now.
  193. */
  194. if (cpumask_empty(tick_broadcast_mask) && bc)
  195. clockevents_shutdown(bc);
  196. /*
  197. * If we kept the cpu in the broadcast mask,
  198. * tell the caller to leave the per cpu device
  199. * in shutdown state. The periodic interrupt
  200. * is delivered by the broadcast device, if
  201. * the broadcast device exists and is not
  202. * hrtimer based.
  203. */
  204. if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))
  205. ret = cpumask_test_cpu(cpu, tick_broadcast_mask);
  206. break;
  207. default:
  208. break;
  209. }
  210. }
  211. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  212. return ret;
  213. }
  214. #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
  215. int tick_receive_broadcast(void)
  216. {
  217. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  218. struct clock_event_device *evt = td->evtdev;
  219. if (!evt)
  220. return -ENODEV;
  221. if (!evt->event_handler)
  222. return -EINVAL;
  223. evt->event_handler(evt);
  224. return 0;
  225. }
  226. #endif
  227. /*
  228. * Broadcast the event to the cpus, which are set in the mask (mangled).
  229. */
  230. static bool tick_do_broadcast(struct cpumask *mask)
  231. {
  232. int cpu = smp_processor_id();
  233. struct tick_device *td;
  234. bool local = false;
  235. /*
  236. * Check, if the current cpu is in the mask
  237. */
  238. if (cpumask_test_cpu(cpu, mask)) {
  239. struct clock_event_device *bc = tick_broadcast_device.evtdev;
  240. cpumask_clear_cpu(cpu, mask);
  241. /*
  242. * We only run the local handler, if the broadcast
  243. * device is not hrtimer based. Otherwise we run into
  244. * a hrtimer recursion.
  245. *
  246. * local timer_interrupt()
  247. * local_handler()
  248. * expire_hrtimers()
  249. * bc_handler()
  250. * local_handler()
  251. * expire_hrtimers()
  252. */
  253. local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);
  254. }
  255. if (!cpumask_empty(mask)) {
  256. /*
  257. * It might be necessary to actually check whether the devices
  258. * have different broadcast functions. For now, just use the
  259. * one of the first device. This works as long as we have this
  260. * misfeature only on x86 (lapic)
  261. */
  262. td = &per_cpu(tick_cpu_device, cpumask_first(mask));
  263. td->evtdev->broadcast(mask);
  264. }
  265. return local;
  266. }
  267. /*
  268. * Periodic broadcast:
  269. * - invoke the broadcast handlers
  270. */
  271. static bool tick_do_periodic_broadcast(void)
  272. {
  273. cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
  274. return tick_do_broadcast(tmpmask);
  275. }
  276. /*
  277. * Event handler for periodic broadcast ticks
  278. */
  279. static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
  280. {
  281. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  282. bool bc_local;
  283. raw_spin_lock(&tick_broadcast_lock);
  284. /* Handle spurious interrupts gracefully */
  285. if (clockevent_state_shutdown(tick_broadcast_device.evtdev)) {
  286. raw_spin_unlock(&tick_broadcast_lock);
  287. return;
  288. }
  289. bc_local = tick_do_periodic_broadcast();
  290. if (clockevent_state_oneshot(dev)) {
  291. ktime_t next = ktime_add(dev->next_event, tick_period);
  292. clockevents_program_event(dev, next, true);
  293. }
  294. raw_spin_unlock(&tick_broadcast_lock);
  295. /*
  296. * We run the handler of the local cpu after dropping
  297. * tick_broadcast_lock because the handler might deadlock when
  298. * trying to switch to oneshot mode.
  299. */
  300. if (bc_local)
  301. td->evtdev->event_handler(td->evtdev);
  302. }
  303. /**
  304. * tick_broadcast_control - Enable/disable or force broadcast mode
  305. * @mode: The selected broadcast mode
  306. *
  307. * Called when the system enters a state where affected tick devices
  308. * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
  309. */
  310. void tick_broadcast_control(enum tick_broadcast_mode mode)
  311. {
  312. struct clock_event_device *bc, *dev;
  313. struct tick_device *td;
  314. int cpu, bc_stopped;
  315. unsigned long flags;
  316. /* Protects also the local clockevent device. */
  317. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  318. td = this_cpu_ptr(&tick_cpu_device);
  319. dev = td->evtdev;
  320. /*
  321. * Is the device not affected by the powerstate ?
  322. */
  323. if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
  324. goto out;
  325. if (!tick_device_is_functional(dev))
  326. goto out;
  327. cpu = smp_processor_id();
  328. bc = tick_broadcast_device.evtdev;
  329. bc_stopped = cpumask_empty(tick_broadcast_mask);
  330. switch (mode) {
  331. case TICK_BROADCAST_FORCE:
  332. tick_broadcast_forced = 1;
  333. case TICK_BROADCAST_ON:
  334. cpumask_set_cpu(cpu, tick_broadcast_on);
  335. if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
  336. /*
  337. * Only shutdown the cpu local device, if:
  338. *
  339. * - the broadcast device exists
  340. * - the broadcast device is not a hrtimer based one
  341. * - the broadcast device is in periodic mode to
  342. * avoid a hickup during switch to oneshot mode
  343. */
  344. if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&
  345. tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
  346. clockevents_shutdown(dev);
  347. }
  348. break;
  349. case TICK_BROADCAST_OFF:
  350. if (tick_broadcast_forced)
  351. break;
  352. cpumask_clear_cpu(cpu, tick_broadcast_on);
  353. if (!tick_device_is_functional(dev))
  354. break;
  355. if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
  356. if (tick_broadcast_device.mode ==
  357. TICKDEV_MODE_PERIODIC)
  358. tick_setup_periodic(dev, 0);
  359. }
  360. break;
  361. }
  362. if (bc) {
  363. if (cpumask_empty(tick_broadcast_mask)) {
  364. if (!bc_stopped)
  365. clockevents_shutdown(bc);
  366. } else if (bc_stopped) {
  367. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
  368. tick_broadcast_start_periodic(bc);
  369. else
  370. tick_broadcast_setup_oneshot(bc);
  371. }
  372. }
  373. out:
  374. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  375. }
  376. EXPORT_SYMBOL_GPL(tick_broadcast_control);
  377. /*
  378. * Set the periodic handler depending on broadcast on/off
  379. */
  380. void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
  381. {
  382. if (!broadcast)
  383. dev->event_handler = tick_handle_periodic;
  384. else
  385. dev->event_handler = tick_handle_periodic_broadcast;
  386. }
  387. #ifdef CONFIG_HOTPLUG_CPU
  388. /*
  389. * Remove a CPU from broadcasting
  390. */
  391. void tick_shutdown_broadcast(unsigned int cpu)
  392. {
  393. struct clock_event_device *bc;
  394. unsigned long flags;
  395. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  396. bc = tick_broadcast_device.evtdev;
  397. cpumask_clear_cpu(cpu, tick_broadcast_mask);
  398. cpumask_clear_cpu(cpu, tick_broadcast_on);
  399. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
  400. if (bc && cpumask_empty(tick_broadcast_mask))
  401. clockevents_shutdown(bc);
  402. }
  403. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  404. }
  405. #endif
  406. void tick_suspend_broadcast(void)
  407. {
  408. struct clock_event_device *bc;
  409. unsigned long flags;
  410. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  411. bc = tick_broadcast_device.evtdev;
  412. if (bc)
  413. clockevents_shutdown(bc);
  414. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  415. }
  416. /*
  417. * This is called from tick_resume_local() on a resuming CPU. That's
  418. * called from the core resume function, tick_unfreeze() and the magic XEN
  419. * resume hackery.
  420. *
  421. * In none of these cases the broadcast device mode can change and the
  422. * bit of the resuming CPU in the broadcast mask is safe as well.
  423. */
  424. bool tick_resume_check_broadcast(void)
  425. {
  426. if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT)
  427. return false;
  428. else
  429. return cpumask_test_cpu(smp_processor_id(), tick_broadcast_mask);
  430. }
  431. void tick_resume_broadcast(void)
  432. {
  433. struct clock_event_device *bc;
  434. unsigned long flags;
  435. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  436. bc = tick_broadcast_device.evtdev;
  437. if (bc) {
  438. clockevents_tick_resume(bc);
  439. switch (tick_broadcast_device.mode) {
  440. case TICKDEV_MODE_PERIODIC:
  441. if (!cpumask_empty(tick_broadcast_mask))
  442. tick_broadcast_start_periodic(bc);
  443. break;
  444. case TICKDEV_MODE_ONESHOT:
  445. if (!cpumask_empty(tick_broadcast_mask))
  446. tick_resume_broadcast_oneshot(bc);
  447. break;
  448. }
  449. }
  450. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  451. }
  452. #ifdef CONFIG_TICK_ONESHOT
  453. static cpumask_var_t tick_broadcast_oneshot_mask __cpumask_var_read_mostly;
  454. static cpumask_var_t tick_broadcast_pending_mask __cpumask_var_read_mostly;
  455. static cpumask_var_t tick_broadcast_force_mask __cpumask_var_read_mostly;
  456. /*
  457. * Exposed for debugging: see timer_list.c
  458. */
  459. struct cpumask *tick_get_broadcast_oneshot_mask(void)
  460. {
  461. return tick_broadcast_oneshot_mask;
  462. }
  463. /*
  464. * Called before going idle with interrupts disabled. Checks whether a
  465. * broadcast event from the other core is about to happen. We detected
  466. * that in tick_broadcast_oneshot_control(). The callsite can use this
  467. * to avoid a deep idle transition as we are about to get the
  468. * broadcast IPI right away.
  469. */
  470. int tick_check_broadcast_expired(void)
  471. {
  472. return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
  473. }
  474. /*
  475. * Set broadcast interrupt affinity
  476. */
  477. static void tick_broadcast_set_affinity(struct clock_event_device *bc,
  478. const struct cpumask *cpumask)
  479. {
  480. if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
  481. return;
  482. if (cpumask_equal(bc->cpumask, cpumask))
  483. return;
  484. bc->cpumask = cpumask;
  485. irq_set_affinity(bc->irq, bc->cpumask);
  486. }
  487. static void tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
  488. ktime_t expires)
  489. {
  490. if (!clockevent_state_oneshot(bc))
  491. clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
  492. clockevents_program_event(bc, expires, 1);
  493. tick_broadcast_set_affinity(bc, cpumask_of(cpu));
  494. }
  495. static void tick_resume_broadcast_oneshot(struct clock_event_device *bc)
  496. {
  497. clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
  498. }
  499. /*
  500. * Called from irq_enter() when idle was interrupted to reenable the
  501. * per cpu device.
  502. */
  503. void tick_check_oneshot_broadcast_this_cpu(void)
  504. {
  505. if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
  506. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  507. /*
  508. * We might be in the middle of switching over from
  509. * periodic to oneshot. If the CPU has not yet
  510. * switched over, leave the device alone.
  511. */
  512. if (td->mode == TICKDEV_MODE_ONESHOT) {
  513. clockevents_switch_state(td->evtdev,
  514. CLOCK_EVT_STATE_ONESHOT);
  515. }
  516. }
  517. }
  518. /*
  519. * Handle oneshot mode broadcasting
  520. */
  521. static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
  522. {
  523. struct tick_device *td;
  524. ktime_t now, next_event;
  525. int cpu, next_cpu = 0;
  526. bool bc_local;
  527. raw_spin_lock(&tick_broadcast_lock);
  528. dev->next_event = KTIME_MAX;
  529. next_event = KTIME_MAX;
  530. cpumask_clear(tmpmask);
  531. now = ktime_get();
  532. /* Find all expired events */
  533. for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
  534. /*
  535. * Required for !SMP because for_each_cpu() reports
  536. * unconditionally CPU0 as set on UP kernels.
  537. */
  538. if (!IS_ENABLED(CONFIG_SMP) &&
  539. cpumask_empty(tick_broadcast_oneshot_mask))
  540. break;
  541. td = &per_cpu(tick_cpu_device, cpu);
  542. if (td->evtdev->next_event <= now) {
  543. cpumask_set_cpu(cpu, tmpmask);
  544. /*
  545. * Mark the remote cpu in the pending mask, so
  546. * it can avoid reprogramming the cpu local
  547. * timer in tick_broadcast_oneshot_control().
  548. */
  549. cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
  550. } else if (td->evtdev->next_event < next_event) {
  551. next_event = td->evtdev->next_event;
  552. next_cpu = cpu;
  553. }
  554. }
  555. /*
  556. * Remove the current cpu from the pending mask. The event is
  557. * delivered immediately in tick_do_broadcast() !
  558. */
  559. cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
  560. /* Take care of enforced broadcast requests */
  561. cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
  562. cpumask_clear(tick_broadcast_force_mask);
  563. /*
  564. * Sanity check. Catch the case where we try to broadcast to
  565. * offline cpus.
  566. */
  567. if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
  568. cpumask_and(tmpmask, tmpmask, cpu_online_mask);
  569. /*
  570. * Wakeup the cpus which have an expired event.
  571. */
  572. bc_local = tick_do_broadcast(tmpmask);
  573. /*
  574. * Two reasons for reprogram:
  575. *
  576. * - The global event did not expire any CPU local
  577. * events. This happens in dyntick mode, as the maximum PIT
  578. * delta is quite small.
  579. *
  580. * - There are pending events on sleeping CPUs which were not
  581. * in the event mask
  582. */
  583. if (next_event != KTIME_MAX)
  584. tick_broadcast_set_event(dev, next_cpu, next_event);
  585. raw_spin_unlock(&tick_broadcast_lock);
  586. if (bc_local) {
  587. td = this_cpu_ptr(&tick_cpu_device);
  588. td->evtdev->event_handler(td->evtdev);
  589. }
  590. }
  591. static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu)
  592. {
  593. if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))
  594. return 0;
  595. if (bc->next_event == KTIME_MAX)
  596. return 0;
  597. return bc->bound_on == cpu ? -EBUSY : 0;
  598. }
  599. static void broadcast_shutdown_local(struct clock_event_device *bc,
  600. struct clock_event_device *dev)
  601. {
  602. /*
  603. * For hrtimer based broadcasting we cannot shutdown the cpu
  604. * local device if our own event is the first one to expire or
  605. * if we own the broadcast timer.
  606. */
  607. if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {
  608. if (broadcast_needs_cpu(bc, smp_processor_id()))
  609. return;
  610. if (dev->next_event < bc->next_event)
  611. return;
  612. }
  613. clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
  614. }
  615. int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
  616. {
  617. struct clock_event_device *bc, *dev;
  618. int cpu, ret = 0;
  619. ktime_t now;
  620. /*
  621. * If there is no broadcast device, tell the caller not to go
  622. * into deep idle.
  623. */
  624. if (!tick_broadcast_device.evtdev)
  625. return -EBUSY;
  626. dev = this_cpu_ptr(&tick_cpu_device)->evtdev;
  627. raw_spin_lock(&tick_broadcast_lock);
  628. bc = tick_broadcast_device.evtdev;
  629. cpu = smp_processor_id();
  630. if (state == TICK_BROADCAST_ENTER) {
  631. /*
  632. * If the current CPU owns the hrtimer broadcast
  633. * mechanism, it cannot go deep idle and we do not add
  634. * the CPU to the broadcast mask. We don't have to go
  635. * through the EXIT path as the local timer is not
  636. * shutdown.
  637. */
  638. ret = broadcast_needs_cpu(bc, cpu);
  639. if (ret)
  640. goto out;
  641. /*
  642. * If the broadcast device is in periodic mode, we
  643. * return.
  644. */
  645. if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
  646. /* If it is a hrtimer based broadcast, return busy */
  647. if (bc->features & CLOCK_EVT_FEAT_HRTIMER)
  648. ret = -EBUSY;
  649. goto out;
  650. }
  651. if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
  652. WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
  653. /* Conditionally shut down the local timer. */
  654. broadcast_shutdown_local(bc, dev);
  655. /*
  656. * We only reprogram the broadcast timer if we
  657. * did not mark ourself in the force mask and
  658. * if the cpu local event is earlier than the
  659. * broadcast event. If the current CPU is in
  660. * the force mask, then we are going to be
  661. * woken by the IPI right away; we return
  662. * busy, so the CPU does not try to go deep
  663. * idle.
  664. */
  665. if (cpumask_test_cpu(cpu, tick_broadcast_force_mask)) {
  666. ret = -EBUSY;
  667. } else if (dev->next_event < bc->next_event) {
  668. tick_broadcast_set_event(bc, cpu, dev->next_event);
  669. /*
  670. * In case of hrtimer broadcasts the
  671. * programming might have moved the
  672. * timer to this cpu. If yes, remove
  673. * us from the broadcast mask and
  674. * return busy.
  675. */
  676. ret = broadcast_needs_cpu(bc, cpu);
  677. if (ret) {
  678. cpumask_clear_cpu(cpu,
  679. tick_broadcast_oneshot_mask);
  680. }
  681. }
  682. }
  683. } else {
  684. if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
  685. clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
  686. /*
  687. * The cpu which was handling the broadcast
  688. * timer marked this cpu in the broadcast
  689. * pending mask and fired the broadcast
  690. * IPI. So we are going to handle the expired
  691. * event anyway via the broadcast IPI
  692. * handler. No need to reprogram the timer
  693. * with an already expired event.
  694. */
  695. if (cpumask_test_and_clear_cpu(cpu,
  696. tick_broadcast_pending_mask))
  697. goto out;
  698. /*
  699. * Bail out if there is no next event.
  700. */
  701. if (dev->next_event == KTIME_MAX)
  702. goto out;
  703. /*
  704. * If the pending bit is not set, then we are
  705. * either the CPU handling the broadcast
  706. * interrupt or we got woken by something else.
  707. *
  708. * We are not longer in the broadcast mask, so
  709. * if the cpu local expiry time is already
  710. * reached, we would reprogram the cpu local
  711. * timer with an already expired event.
  712. *
  713. * This can lead to a ping-pong when we return
  714. * to idle and therefor rearm the broadcast
  715. * timer before the cpu local timer was able
  716. * to fire. This happens because the forced
  717. * reprogramming makes sure that the event
  718. * will happen in the future and depending on
  719. * the min_delta setting this might be far
  720. * enough out that the ping-pong starts.
  721. *
  722. * If the cpu local next_event has expired
  723. * then we know that the broadcast timer
  724. * next_event has expired as well and
  725. * broadcast is about to be handled. So we
  726. * avoid reprogramming and enforce that the
  727. * broadcast handler, which did not run yet,
  728. * will invoke the cpu local handler.
  729. *
  730. * We cannot call the handler directly from
  731. * here, because we might be in a NOHZ phase
  732. * and we did not go through the irq_enter()
  733. * nohz fixups.
  734. */
  735. now = ktime_get();
  736. if (dev->next_event <= now) {
  737. cpumask_set_cpu(cpu, tick_broadcast_force_mask);
  738. goto out;
  739. }
  740. /*
  741. * We got woken by something else. Reprogram
  742. * the cpu local timer device.
  743. */
  744. tick_program_event(dev->next_event, 1);
  745. }
  746. }
  747. out:
  748. raw_spin_unlock(&tick_broadcast_lock);
  749. return ret;
  750. }
  751. /*
  752. * Reset the one shot broadcast for a cpu
  753. *
  754. * Called with tick_broadcast_lock held
  755. */
  756. static void tick_broadcast_clear_oneshot(int cpu)
  757. {
  758. cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
  759. cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
  760. }
  761. static void tick_broadcast_init_next_event(struct cpumask *mask,
  762. ktime_t expires)
  763. {
  764. struct tick_device *td;
  765. int cpu;
  766. for_each_cpu(cpu, mask) {
  767. td = &per_cpu(tick_cpu_device, cpu);
  768. if (td->evtdev)
  769. td->evtdev->next_event = expires;
  770. }
  771. }
  772. /**
  773. * tick_broadcast_setup_oneshot - setup the broadcast device
  774. */
  775. static void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
  776. {
  777. int cpu = smp_processor_id();
  778. if (!bc)
  779. return;
  780. /* Set it up only once ! */
  781. if (bc->event_handler != tick_handle_oneshot_broadcast) {
  782. int was_periodic = clockevent_state_periodic(bc);
  783. bc->event_handler = tick_handle_oneshot_broadcast;
  784. /*
  785. * We must be careful here. There might be other CPUs
  786. * waiting for periodic broadcast. We need to set the
  787. * oneshot_mask bits for those and program the
  788. * broadcast device to fire.
  789. */
  790. cpumask_copy(tmpmask, tick_broadcast_mask);
  791. cpumask_clear_cpu(cpu, tmpmask);
  792. cpumask_or(tick_broadcast_oneshot_mask,
  793. tick_broadcast_oneshot_mask, tmpmask);
  794. if (was_periodic && !cpumask_empty(tmpmask)) {
  795. clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
  796. tick_broadcast_init_next_event(tmpmask,
  797. tick_next_period);
  798. tick_broadcast_set_event(bc, cpu, tick_next_period);
  799. } else
  800. bc->next_event = KTIME_MAX;
  801. } else {
  802. /*
  803. * The first cpu which switches to oneshot mode sets
  804. * the bit for all other cpus which are in the general
  805. * (periodic) broadcast mask. So the bit is set and
  806. * would prevent the first broadcast enter after this
  807. * to program the bc device.
  808. */
  809. tick_broadcast_clear_oneshot(cpu);
  810. }
  811. }
  812. /*
  813. * Select oneshot operating mode for the broadcast device
  814. */
  815. void tick_broadcast_switch_to_oneshot(void)
  816. {
  817. struct clock_event_device *bc;
  818. unsigned long flags;
  819. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  820. tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
  821. bc = tick_broadcast_device.evtdev;
  822. if (bc)
  823. tick_broadcast_setup_oneshot(bc);
  824. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  825. }
  826. #ifdef CONFIG_HOTPLUG_CPU
  827. void hotplug_cpu__broadcast_tick_pull(int deadcpu)
  828. {
  829. struct clock_event_device *bc;
  830. unsigned long flags;
  831. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  832. bc = tick_broadcast_device.evtdev;
  833. if (bc && broadcast_needs_cpu(bc, deadcpu)) {
  834. /* This moves the broadcast assignment to this CPU: */
  835. clockevents_program_event(bc, bc->next_event, 1);
  836. }
  837. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  838. }
  839. /*
  840. * Remove a dead CPU from broadcasting
  841. */
  842. void tick_shutdown_broadcast_oneshot(unsigned int cpu)
  843. {
  844. unsigned long flags;
  845. raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
  846. /*
  847. * Clear the broadcast masks for the dead cpu, but do not stop
  848. * the broadcast device!
  849. */
  850. cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
  851. cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
  852. cpumask_clear_cpu(cpu, tick_broadcast_force_mask);
  853. raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
  854. }
  855. #endif
  856. /*
  857. * Check, whether the broadcast device is in one shot mode
  858. */
  859. int tick_broadcast_oneshot_active(void)
  860. {
  861. return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
  862. }
  863. /*
  864. * Check whether the broadcast device supports oneshot.
  865. */
  866. bool tick_broadcast_oneshot_available(void)
  867. {
  868. struct clock_event_device *bc = tick_broadcast_device.evtdev;
  869. return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
  870. }
  871. #else
  872. int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
  873. {
  874. struct clock_event_device *bc = tick_broadcast_device.evtdev;
  875. if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))
  876. return -EBUSY;
  877. return 0;
  878. }
  879. #endif
  880. void __init tick_broadcast_init(void)
  881. {
  882. zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
  883. zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT);
  884. zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
  885. #ifdef CONFIG_TICK_ONESHOT
  886. zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
  887. zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
  888. zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
  889. #endif
  890. }